added downloading of youtube videos if https://youtube-dl.org/ is installed on the server's path

5 files changed, 83 insertions, 25 deletions

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 6982ecf19..bf2918734 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -36,4 +36,14 @@ export namespace Networking {
         return response.json();
     }
 
+    export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string): Promise<Upload.FileResponse<T>[]> {
+        const parameters = {
+            method: 'POST',
+            body: JSON.stringify({ videoId }),
+            json: true
+        };
+        const response = await fetch("/uploadYoutubeVideo", parameters);
+        return response.json();
+    }
+
 }
\ No newline at end of file
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 219890945..24682cbd0 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -1386,19 +1386,15 @@ export namespace DocUtils {
         return optionsCollection;
     }
 
-    export async function uploadFilesToDocs(files: File[], options: DocumentOptions) {
-        const generatedDocuments: Doc[] = [];
-        for (const { source: { name, type }, result } of await Networking.UploadFilesToServer(files)) {
-            if (result instanceof Error) {
-                alert(`Upload failed: ${result.message}`);
-                return [];
-            }
-            const full = { ...options, _width: 400, title: name };
-            const pathname = Utils.prepend(result.accessPaths.agnostic.client);
-            const doc = await DocUtils.DocumentFromType(type, pathname, full);
-            if (!doc) {
-                continue;
-            }
+    async function processFileupload(generatedDocuments: Doc[], name: string, type: string, result: Error | Upload.FileInformation, options: DocumentOptions) {
+        if (result instanceof Error) {
+            alert(`Upload failed: ${result.message}`);
+            return;
+        }
+        const full = { ...options, _width: 400, title: name };
+        const pathname = Utils.prepend(result.accessPaths.agnostic.client);
+        const doc = await DocUtils.DocumentFromType(type, pathname, full);
+        if (doc) {
             const proto = Doc.GetProto(doc);
             proto.text = result.rawText;
             proto.fileUpload = basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "");
@@ -1415,6 +1411,20 @@ export namespace DocUtils {
             }
             generatedDocuments.push(doc);
         }
+    }
+
+    export async function uploadYoutubeVideo(videoId: string, options: DocumentOptions) {
+        const generatedDocuments: Doc[] = [];
+        for (const { source: { name, type }, result } of await Networking.UploadYoutubeToServer(videoId)) {
+            processFileupload(generatedDocuments, name, type, result, options);
+        }
+        return generatedDocuments;
+    }
+    export async function uploadFilesToDocs(files: File[], options: DocumentOptions) {
+        const generatedDocuments: Doc[] = [];
+        for (const { source: { name, type }, result } of await Networking.UploadFilesToServer(files)) {
+            processFileupload(generatedDocuments, name, type, result, options);
+        }
         return generatedDocuments;
     }
 }
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index a5d62acb4..8d549bd56 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -347,16 +347,11 @@ export function CollectionSubView<T, X>(schemaCtor: (doc: Doc) => T, moreProps?:
 
             if (uriList || text) {
                 if ((uriList || text).includes("www.youtube.com/watch") || text.includes("www.youtube.com/embed")) {
-                    const url = (uriList || text).replace("youtube.com/watch?v=", "youtube.com/embed/").split("&")[0];
-                    console.log("Video URI = ", uriList);
-                    console.log("Add:" + addDocument(Docs.Create.VideoDocument(url, {
-                        ...options,
-                        title: url,
-                        _width: 400,
-                        _height: 315,
-                        _nativeWidth: 600,
-                        _nativeHeight: 472.5
-                    })));
+
+                    const batch = UndoManager.StartBatch("youtube upload");
+                    const generatedDocuments: Doc[] = [];
+                    this.slowLoadDocuments((uriList || text).split("v=")[1], options, generatedDocuments, text, completed, e.clientX, e.clientY, addDocument).then(batch.end);
+
                     return;
                 }
                 // let matches: RegExpExecArray | null;
@@ -444,10 +439,14 @@ export function CollectionSubView<T, X>(schemaCtor: (doc: Doc) => T, moreProps?:
             }
             this.slowLoadDocuments(files, options, generatedDocuments, text, completed, e.clientX, e.clientY, addDocument).then(batch.end);
         }
-        slowLoadDocuments = async (files: File[], options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: (() => void) | undefined, clientX: number, clientY: number, addDocument: (doc: Doc | Doc[]) => boolean) => {
+        slowLoadDocuments = async (files: (File[] | string), options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: (() => void) | undefined, clientX: number, clientY: number, addDocument: (doc: Doc | Doc[]) => boolean) => {
             const disposer = OverlayView.Instance.addElement(
                 <ReactLoading type={"spinningBubbles"} color={"green"} height={250} width={250} />, { x: clientX - 125, y: clientY - 125 });
-            generatedDocuments.push(...await DocUtils.uploadFilesToDocs(files, options));
+            if (typeof files === "string") {
+                generatedDocuments.push(...await DocUtils.uploadYoutubeVideo(files, options));
+            } else {
+                generatedDocuments.push(...await DocUtils.uploadFilesToDocs(files, options));
+            }
             if (generatedDocuments.length) {
                 const set = generatedDocuments.length > 1 && generatedDocuments.map(d => DocUtils.iconify(d));
                 if (set) {
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index d6950d46a..02f6462aa 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -16,6 +16,7 @@ const imageDataUri = require('image-data-uri');
 import { isWebUri } from "valid-url";
 import { Opt } from "../../fields/Doc";
 import { SolrManager } from "./SearchManager";
+import { StringDecoder } from "string_decoder";
 
 export enum Directory {
     parsed_files = "parsed_files",
@@ -66,6 +67,26 @@ export default class UploadManager extends ApiManager {
 
         register({
             method: Method.POST,
+            subscription: "/uploadYoutubeVideo",
+            secureHandler: async ({ req, res }) => {
+                //req.readableBuffer.head.data
+                return new Promise<void>(async resolve => {
+                    req.addListener("data", async (args) => {
+                        console.log(args);
+                        const payload = String.fromCharCode.apply(String, args);
+                        const videoId = JSON.parse(payload).videoId;
+                        const results: Upload.FileResponse[] = [];
+                        const result = await DashUploadUtils.uploadYoutube(videoId);
+                        result && !(result.result instanceof Error) && results.push(result);
+                        _success(res, results);
+                        resolve();
+                    });
+                });
+            }
+        });
+
+        register({
+            method: Method.POST,
             subscription: new RouteSubscriber("youtubeScreenshot"),
             secureHandler: async ({ req, res }) => {
                 const { id, timecode } = req.body;
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index ff6b2381c..555e3bf3b 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -15,6 +15,7 @@ import { clientPathToFile, Directory, pathToDirectory, serverPathToFile } from '
 import { resolvedServerUrl } from "./server_Initialization";
 import { AcceptableMedia, Upload } from './SharedMediaTypes';
 import request = require('request-promise');
+const { exec } = require("child_process");
 const parse = require('pdf-parse');
 const ffmpeg = require("fluent-ffmpeg");
 const requestImageSize = require("../client/util/request-image-size");
@@ -57,6 +58,23 @@ export namespace DashUploadUtils {
 
     const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia;
 
+    export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
+        console.log("UPLOAD " + videoId);
+        return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
+            exec('/usr/local/bin/youtube-dl -o ' + (videoId + ".mp4") + ' https://www.youtube.com/watch?v=' + videoId + ' -f `/usr/local/bin/youtube-dl https://www.youtube.com/watch?v=' + videoId + ' -F | grep "(best)" | sed -e "s/ .*//"`',
+                (error: any, stdout: any, stderr: any) => {
+                    if (error) console.log(`error: ${error.message}`);
+                    else if (stderr) console.log(`stderr: ${stderr}`);
+                    else {
+                        console.log(`stdout: ${stdout}`);
+                        const data = { size: 0, path: videoId + ".mp4", name: videoId, type: "video/mp4" };
+                        const file = { ...data, toJSON: () => data };
+                        res(MoveParsedFile(file, Directory.videos));
+                    }
+                });
+        });
+    }
+
     export async function upload(file: File): Promise<Upload.FileResponse> {
         const { type, path, name } = file;
         const types = type.split("/");