Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web into sharing_scenario

9 files changed, 109 insertions, 36 deletions

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 6982ecf19..bf2918734 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -36,4 +36,14 @@ export namespace Networking {
         return response.json();
     }
 
+    export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string): Promise<Upload.FileResponse<T>[]> {
+        const parameters = {
+            method: 'POST',
+            body: JSON.stringify({ videoId }),
+            json: true
+        };
+        const response = await fetch("/uploadYoutubeVideo", parameters);
+        return response.json();
+    }
+
 }
\ No newline at end of file
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 906603d78..6fb6f70b3 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -1397,19 +1397,15 @@ export namespace DocUtils {
         return optionsCollection;
     }
 
-    export async function uploadFilesToDocs(files: File[], options: DocumentOptions) {
-        const generatedDocuments: Doc[] = [];
-        for (const { source: { name, type }, result } of await Networking.UploadFilesToServer(files)) {
-            if (result instanceof Error) {
-                alert(`Upload failed: ${result.message}`);
-                return [];
-            }
-            const full = { ...options, _width: 400, title: name };
-            const pathname = Utils.prepend(result.accessPaths.agnostic.client);
-            const doc = await DocUtils.DocumentFromType(type, pathname, full);
-            if (!doc) {
-                continue;
-            }
+    async function processFileupload(generatedDocuments: Doc[], name: string, type: string, result: Error | Upload.FileInformation, options: DocumentOptions) {
+        if (result instanceof Error) {
+            alert(`Upload failed: ${result.message}`);
+            return;
+        }
+        const full = { ...options, _width: 400, title: name };
+        const pathname = Utils.prepend(result.accessPaths.agnostic.client);
+        const doc = await DocUtils.DocumentFromType(type, pathname, full);
+        if (doc) {
             const proto = Doc.GetProto(doc);
             proto.text = result.rawText;
             proto.fileUpload = basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "");
@@ -1426,6 +1422,20 @@ export namespace DocUtils {
             }
             generatedDocuments.push(doc);
         }
+    }
+
+    export async function uploadYoutubeVideo(videoId: string, options: DocumentOptions) {
+        const generatedDocuments: Doc[] = [];
+        for (const { source: { name, type }, result } of await Networking.UploadYoutubeToServer(videoId)) {
+            processFileupload(generatedDocuments, name, type, result, options);
+        }
+        return generatedDocuments;
+    }
+    export async function uploadFilesToDocs(files: File[], options: DocumentOptions) {
+        const generatedDocuments: Doc[] = [];
+        for (const { source: { name, type }, result } of await Networking.UploadFilesToServer(files)) {
+            processFileupload(generatedDocuments, name, type, result, options);
+        }
         return generatedDocuments;
     }
 }
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index a5d62acb4..8d549bd56 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -347,16 +347,11 @@ export function CollectionSubView<T, X>(schemaCtor: (doc: Doc) => T, moreProps?:
 
             if (uriList || text) {
                 if ((uriList || text).includes("www.youtube.com/watch") || text.includes("www.youtube.com/embed")) {
-                    const url = (uriList || text).replace("youtube.com/watch?v=", "youtube.com/embed/").split("&")[0];
-                    console.log("Video URI = ", uriList);
-                    console.log("Add:" + addDocument(Docs.Create.VideoDocument(url, {
-                        ...options,
-                        title: url,
-                        _width: 400,
-                        _height: 315,
-                        _nativeWidth: 600,
-                        _nativeHeight: 472.5
-                    })));
+
+                    const batch = UndoManager.StartBatch("youtube upload");
+                    const generatedDocuments: Doc[] = [];
+                    this.slowLoadDocuments((uriList || text).split("v=")[1], options, generatedDocuments, text, completed, e.clientX, e.clientY, addDocument).then(batch.end);
+
                     return;
                 }
                 // let matches: RegExpExecArray | null;
@@ -444,10 +439,14 @@ export function CollectionSubView<T, X>(schemaCtor: (doc: Doc) => T, moreProps?:
             }
             this.slowLoadDocuments(files, options, generatedDocuments, text, completed, e.clientX, e.clientY, addDocument).then(batch.end);
         }
-        slowLoadDocuments = async (files: File[], options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: (() => void) | undefined, clientX: number, clientY: number, addDocument: (doc: Doc | Doc[]) => boolean) => {
+        slowLoadDocuments = async (files: (File[] | string), options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: (() => void) | undefined, clientX: number, clientY: number, addDocument: (doc: Doc | Doc[]) => boolean) => {
             const disposer = OverlayView.Instance.addElement(
                 <ReactLoading type={"spinningBubbles"} color={"green"} height={250} width={250} />, { x: clientX - 125, y: clientY - 125 });
-            generatedDocuments.push(...await DocUtils.uploadFilesToDocs(files, options));
+            if (typeof files === "string") {
+                generatedDocuments.push(...await DocUtils.uploadYoutubeVideo(files, options));
+            } else {
+                generatedDocuments.push(...await DocUtils.uploadFilesToDocs(files, options));
+            }
             if (generatedDocuments.length) {
                 const set = generatedDocuments.length > 1 && generatedDocuments.map(d => DocUtils.iconify(d));
                 if (set) {
diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx
index 85ae66fdc..fb60265e3 100644
--- a/src/client/views/collections/CollectionView.tsx
+++ b/src/client/views/collections/CollectionView.tsx
@@ -1,8 +1,8 @@
-import { computed, observable } from 'mobx';
+import { computed, observable, runInAction } from 'mobx';
 import { observer } from "mobx-react";
 import * as React from 'react';
 import 'react-image-lightbox-with-rotate/style.css'; // This only needs to be imported once in your app
-import { Doc, DocListCast } from '../../../fields/Doc';
+import { Doc, DocListCast, StrListCast } from '../../../fields/Doc';
 import { documentSchema } from '../../../fields/documentSchemas';
 import { Id } from '../../../fields/FieldSymbols';
 import { ObjectField } from '../../../fields/ObjectField';
@@ -34,6 +34,7 @@ import { CollectionStackingView } from './CollectionStackingView';
 import { SubCollectionViewProps } from './CollectionSubView';
 import { CollectionTimeView } from './CollectionTimeView';
 import { CollectionTreeView } from "./CollectionTreeView";
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import './CollectionView.scss';
 export const COLLECTION_BORDER_WIDTH = 2;
 const path = require('path');
@@ -231,6 +232,13 @@ export class CollectionView extends ViewBoxAnnotatableComponent<ViewBoxAnnotatab
     childLayoutTemplate = () => this.props.childLayoutTemplate?.() || Cast(this.rootDoc.childLayoutTemplate, Doc, null);
     @computed get childLayoutString() { return StrCast(this.rootDoc.childLayoutString); }
 
+    /**
+    * Shows the filter icon if it's a user-created collection which isn't a dashboard and has some docFilters applied on it or on the current dashboard.
+    */
+    @computed get showFilterIcon() {
+        return this.props.Document.viewType !== CollectionViewType.Docking && !Doc.IsSystem(this.props.Document) && ((StrListCast(this.props.Document._docFilters).length || StrListCast(this.props.Document._docRangeFilters).length || StrListCast(CurrentUserUtils.ActiveDashboard._docFilters).length || StrListCast(CurrentUserUtils.ActiveDashboard._docRangeFilters).length));
+    }
+
     render() {
         TraceMobx();
         const props: SubCollectionViewProps = {
@@ -250,6 +258,12 @@ export class CollectionView extends ViewBoxAnnotatableComponent<ViewBoxAnnotatab
             style={{ pointerEvents: this.props.layerProvider?.(this.rootDoc) === false ? "none" : undefined }}>
             {this.showIsTagged()}
             {this.collectionViewType !== undefined ? this.SubView(this.collectionViewType, props) : (null)}
+            {this.showFilterIcon ?
+                <FontAwesomeIcon icon={"filter"} size="lg"
+                    style={{ position: 'absolute', top: '1%', right: '1%', cursor: "pointer", padding: 1, color: '#18c718bd', zIndex: 1 }}
+                    onPointerDown={e => { runInAction(() => CurrentUserUtils.propertiesWidth = 250); e.stopPropagation(); }}
+                />
+                : (null)}
         </div>);
     }
 }
diff --git a/src/client/views/collections/TreeView.scss b/src/client/views/collections/TreeView.scss
index 0239ae863..3f6fc8b0c 100644
--- a/src/client/views/collections/TreeView.scss
+++ b/src/client/views/collections/TreeView.scss
@@ -118,6 +118,7 @@
         align-items: center;
         margin-left: 0.25rem;
         opacity: 0.75;
+        cursor: pointer;
 
         >svg {
             margin-left: 0.25rem;
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index a14ba036f..0f9dbe23b 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -48,6 +48,7 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso
 import "./CollectionFreeFormView.scss";
 import { MarqueeView } from "./MarqueeView";
 import React = require("react");
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 
 export const panZoomSchema = createSchema({
     _panX: "number",
@@ -833,10 +834,10 @@ export class CollectionFreeFormView extends CollectionSubView<PanZoomDocument, P
             const measuredDocs = docs.filter(doc => doc && this.childDataProvider(doc, "")).map(doc => this.childDataProvider(doc, ""));
             if (measuredDocs.length) {
                 const ranges = measuredDocs.reduce(({ xrange, yrange }, { x, y, width, height }) =>  // computes range of content
-                    ({
-                        xrange: { min: Math.min(xrange.min, x), max: Math.max(xrange.max, x + width) },
-                        yrange: { min: Math.min(yrange.min, y), max: Math.max(yrange.max, y + height) }
-                    })
+                ({
+                    xrange: { min: Math.min(xrange.min, x), max: Math.max(xrange.max, x + width) },
+                    yrange: { min: Math.min(yrange.min, y), max: Math.max(yrange.max, y + height) }
+                })
                     , {
                         xrange: { min: Number.MAX_VALUE, max: -Number.MAX_VALUE },
                         yrange: { min: Number.MAX_VALUE, max: -Number.MAX_VALUE }
@@ -1494,6 +1495,7 @@ export class CollectionFreeFormView extends CollectionSubView<PanZoomDocument, P
                 this.placeholder : this.marqueeView}
             {this.props.noOverlay ? (null) : <CollectionFreeFormOverlayView elements={this.elementFunc} />}
 
+
             <div className={"pullpane-indicator"}
                 style={{
                     display: this._pullDirection ? "block" : "none",
diff --git a/src/client/views/nodes/FilterBox.tsx b/src/client/views/nodes/FilterBox.tsx
index c97de3402..c892a9f6c 100644
--- a/src/client/views/nodes/FilterBox.tsx
+++ b/src/client/views/nodes/FilterBox.tsx
@@ -194,12 +194,12 @@ export class FilterBox extends ViewBoxBaseComponent<FieldViewProps, FilterBoxDoc
      * Responds to clicking the check box in the flyout menu
      */
     facetClick = (facetHeader: string) => {
-        const targetDoc = FilterBox.targetDoc;
+        const { targetDoc } = FilterBox;
         const found = this.activeAttributes.findIndex(doc => doc.title === facetHeader);
         if (found !== -1) {
             this.removeFilter(facetHeader);
         } else {
-            const allCollectionDocs = DocListCast((targetDoc.data as any)[0].data);
+            const allCollectionDocs = DocListCast((targetDoc.data as any)?.[0].data);
             const facetValues = this.gatherFieldValues(targetDoc, facetHeader);
 
             let nonNumbers = 0;
@@ -357,9 +357,7 @@ export class FilterBox extends ViewBoxBaseComponent<FieldViewProps, FilterBoxDoc
     render() {
         const facetCollection = this.props.Document;
 
-        // TODO uncomment the line below when the treeview x works
-        // const options = this._allFacets.filter(facet => this.currentFacets.indexOf(facet) === -1).map(facet => ({ value: facet, label: facet }));
-        const options = this._allFacets.map(facet => ({ value: facet, label: facet }));
+        const options = this._allFacets.filter(facet => this.currentFacets.indexOf(facet) === -1).map(facet => ({ value: facet, label: facet }));
         return this.props.dontRegisterView ? (null) : <div className="filterBox-treeView" style={{ width: "100%" }}>
 
             <div className="filterBox-title">
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index d6950d46a..02f6462aa 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -16,6 +16,7 @@ const imageDataUri = require('image-data-uri');
 import { isWebUri } from "valid-url";
 import { Opt } from "../../fields/Doc";
 import { SolrManager } from "./SearchManager";
+import { StringDecoder } from "string_decoder";
 
 export enum Directory {
     parsed_files = "parsed_files",
@@ -66,6 +67,26 @@ export default class UploadManager extends ApiManager {
 
         register({
             method: Method.POST,
+            subscription: "/uploadYoutubeVideo",
+            secureHandler: async ({ req, res }) => {
+                //req.readableBuffer.head.data
+                return new Promise<void>(async resolve => {
+                    req.addListener("data", async (args) => {
+                        console.log(args);
+                        const payload = String.fromCharCode.apply(String, args);
+                        const videoId = JSON.parse(payload).videoId;
+                        const results: Upload.FileResponse[] = [];
+                        const result = await DashUploadUtils.uploadYoutube(videoId);
+                        result && !(result.result instanceof Error) && results.push(result);
+                        _success(res, results);
+                        resolve();
+                    });
+                });
+            }
+        });
+
+        register({
+            method: Method.POST,
             subscription: new RouteSubscriber("youtubeScreenshot"),
             secureHandler: async ({ req, res }) => {
                 const { id, timecode } = req.body;
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index ff6b2381c..555e3bf3b 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -15,6 +15,7 @@ import { clientPathToFile, Directory, pathToDirectory, serverPathToFile } from '
 import { resolvedServerUrl } from "./server_Initialization";
 import { AcceptableMedia, Upload } from './SharedMediaTypes';
 import request = require('request-promise');
+const { exec } = require("child_process");
 const parse = require('pdf-parse');
 const ffmpeg = require("fluent-ffmpeg");
 const requestImageSize = require("../client/util/request-image-size");
@@ -57,6 +58,23 @@ export namespace DashUploadUtils {
 
     const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia;
 
+    export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
+        console.log("UPLOAD " + videoId);
+        return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
+            exec('/usr/local/bin/youtube-dl -o ' + (videoId + ".mp4") + ' https://www.youtube.com/watch?v=' + videoId + ' -f `/usr/local/bin/youtube-dl https://www.youtube.com/watch?v=' + videoId + ' -F | grep "(best)" | sed -e "s/ .*//"`',
+                (error: any, stdout: any, stderr: any) => {
+                    if (error) console.log(`error: ${error.message}`);
+                    else if (stderr) console.log(`stderr: ${stderr}`);
+                    else {
+                        console.log(`stdout: ${stdout}`);
+                        const data = { size: 0, path: videoId + ".mp4", name: videoId, type: "video/mp4" };
+                        const file = { ...data, toJSON: () => data };
+                        res(MoveParsedFile(file, Directory.videos));
+                    }
+                });
+        });
+    }
+
     export async function upload(file: File): Promise<Upload.FileResponse> {
         const { type, path, name } = file;
         const types = type.split("/");