Merge pull request #345 from browngraphicslab/audio_refactor

Audio refactor

5 files changed, 188 insertions, 164 deletions

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 50a759c9d..42e33ece0 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -19,7 +19,8 @@ export enum Directory {
     videos = "videos",
     pdfs = "pdfs",
     text = "text",
-    pdf_thumbnails = "pdf_thumbnails"
+    pdf_thumbnails = "pdf_thumbnails",
+    audio = "audio"
 }
 
 export function serverPathToFile(directory: Directory, filename: string) {
@@ -61,9 +62,18 @@ export default class UploadManager extends ApiManager {
         });
 
         register({
+            method: Method.GET,
+            subscription: "/hello",
+            secureHandler: ({ req, res }) => {
+                res.send("<h1>world!</h1>");
+            }
+        });
+
+        register({
             method: Method.POST,
             subscription: "/uploadRemoteImage",
             secureHandler: async ({ req, res }) => {
+
                 const { sources } = req.body;
                 if (Array.isArray(sources)) {
                     const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source)));
@@ -77,6 +87,7 @@ export default class UploadManager extends ApiManager {
             method: Method.POST,
             subscription: "/uploadDoc",
             secureHandler: ({ req, res }) => {
+
                 const form = new formidable.IncomingForm();
                 form.keepExtensions = true;
                 // let path = req.body.path;
@@ -181,6 +192,7 @@ export default class UploadManager extends ApiManager {
             method: Method.POST,
             subscription: "/inspectImage",
             secureHandler: async ({ req, res }) => {
+
                 const { source } = req.body;
                 if (typeof source === "string") {
                     return res.send(await DashUploadUtils.InspectImage(source));
diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts
index d18529cf2..ad8119bf4 100644
--- a/src/server/ApiManagers/UtilManager.ts
+++ b/src/server/ApiManagers/UtilManager.ts
@@ -3,11 +3,11 @@ import { Method } from "../RouteManager";
 import { exec } from 'child_process';
 import RouteSubscriber from "../RouteSubscriber";
 import { red } from "colors";
-import { IBM_Recommender } from "../../client/apis/IBM_Recommender";
-import { Recommender } from "../Recommender";
+// import { IBM_Recommender } from "../../client/apis/IBM_Recommender";
+// import { Recommender } from "../Recommender";
 
-const recommender = new Recommender();
-recommender.testModel();
+// const recommender = new Recommender();
+// recommender.testModel();
 import executeImport from "../../scraping/buxton/final/BuxtonImporter";
 
 export default class UtilManager extends ApiManager {
@@ -27,25 +27,25 @@ export default class UtilManager extends ApiManager {
             }
         });
 
-        register({
-            method: Method.POST,
-            subscription: "/IBMAnalysis",
-            secureHandler: async ({ req, res }) => res.send(await IBM_Recommender.analyze(req.body))
-        });
+        // register({
+        //     method: Method.POST,
+        //     subscription: "/IBMAnalysis",
+        //     secureHandler: async ({ req, res }) => res.send(await IBM_Recommender.analyze(req.body))
+        // });
 
-        register({
-            method: Method.POST,
-            subscription: "/recommender",
-            secureHandler: async ({ req, res }) => {
-                const keyphrases = req.body.keyphrases;
-                const wordvecs = await recommender.vectorize(keyphrases);
-                let embedding: Float32Array = new Float32Array();
-                if (wordvecs && wordvecs.dataSync()) {
-                    embedding = wordvecs.dataSync() as Float32Array;
-                }
-                res.send(embedding);
-            }
-        });
+        // register({
+        //     method: Method.POST,
+        //     subscription: "/recommender",
+        //     secureHandler: async ({ req, res }) => {
+        //         const keyphrases = req.body.keyphrases;
+        //         const wordvecs = await recommender.vectorize(keyphrases);
+        //         let embedding: Float32Array = new Float32Array();
+        //         if (wordvecs && wordvecs.dataSync()) {
+        //             embedding = wordvecs.dataSync() as Float32Array;
+        //         }
+        //         res.send(embedding);
+        //     }
+        // });
 
 
         register({
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index cc3dd75a4..cf78af60a 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -53,7 +53,7 @@ export namespace DashUploadUtils {
     const size = "content-length";
     const type = "content-type";
 
-    const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia;
+    const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptibleMedia;
 
     export async function upload(file: File): Promise<Upload.FileResponse> {
         const { type, path, name } = file;
@@ -76,14 +76,22 @@ export namespace DashUploadUtils {
                 if (applicationFormats.includes(format)) {
                     return UploadPdf(file);
                 }
-            default: //  "blob":
-                return MoveParsedFile(file, Directory.videos);
+            case "audio":
+                if (audioFormats.includes(format)) {
+                    return MoveParsedFile(file, Directory.audio);
+                }
         }
 
         console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`));
         return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) };
     }
 
+    async function UploadAudio(file: File) {
+        const { path: sourcePath } = file;
+
+        return MoveParsedFile(file, Directory.audio);
+    }
+
     async function UploadPdf(file: File) {
         const { path: sourcePath } = file;
         const dataBuffer = readFileSync(sourcePath);
@@ -94,6 +102,7 @@ export namespace DashUploadUtils {
             const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
             writeStream.write(result.text, error => error ? reject(error) : resolve());
         });
+        console.log(MoveParsedFile(file, Directory.pdfs));
         return MoveParsedFile(file, Directory.pdfs);
     }
 
@@ -197,8 +206,10 @@ export namespace DashUploadUtils {
                         accessPaths: {
                             agnostic: getAccessPaths(destination, name)
                         }
+
                     }
-                });
+                }
+                );
             });
         });
     }
diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts
index 1d2cb3858..aacdb4053 100644
--- a/src/server/Recommender.ts
+++ b/src/server/Recommender.ts
@@ -1,137 +1,137 @@
-//import { Doc } from "../new_fields/Doc";
-//import { StrCast } from "../new_fields/Types";
-//import { List } from "../new_fields/List";
-//import { CognitiveServices } from "../client/cognitive_services/CognitiveServices";
-
-// var w2v = require('word2vec');
-var assert = require('assert');
-var arxivapi = require('arxiv-api-node');
-import requestPromise = require("request-promise");
-import * as use from '@tensorflow-models/universal-sentence-encoder';
-import { Tensor } from "@tensorflow/tfjs-core/dist/tensor";
-require('@tensorflow/tfjs-node');
-
-//http://gnuwin32.sourceforge.net/packages/make.htm
-
-export class Recommender {
-
-    private _model: any;
-    static Instance: Recommender;
-    private dimension: number = 0;
-    private choice: string = ""; // Tensorflow or Word2Vec
-
-    constructor() {
-        console.log("creating recommender...");
-        Recommender.Instance = this;
-    }
-
-    /***
-     * Loads pre-trained model from TF
-     */
-
-    public async loadTFModel() {
-        let self = this;
-        return new Promise(res => {
-            use.load().then(model => {
-                self.choice = "TF";
-                self._model = model;
-                self.dimension = 512;
-                res(model);
-            });
-        }
-
-        );
-    }
-
-    /***
-     * Loads pre-trained model from word2vec
-     */
-
-    // private loadModel(): Promise<any> {
-    //     let self = this;
-    //     return new Promise(res => {
-    //         w2v.loadModel("./node_modules/word2vec/examples/fixtures/vectors.txt", function (err: any, model: any) {
-    //             self.choice = "WV";
-    //             self._model = model;
-    //             self.dimension = model.size;
-    //             res(model);
-    //         });
-    //     });
-    // }
-
-    /***
-     * Testing
-     */
-
-    public async testModel() {
-        if (!this._model) {
-            await this.loadTFModel();
-        }
-        if (this._model) {
-            if (this.choice === "WV") {
-                let similarity = this._model.similarity('father', 'mother');
-                console.log(similarity);
-            }
-            else if (this.choice === "TF") {
-                const model = this._model as use.UniversalSentenceEncoder;
-                // Embed an array of sentences.
-                const sentences = [
-                    'Hello.',
-                    'How are you?'
-                ];
-                const embeddings = await this.vectorize(sentences);
-                if (embeddings) embeddings.print(true /*verbose*/);
-                // model.embed(sentences).then(embeddings => {
-                //     // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
-                //     // So in this example `embeddings` has the shape [2, 512].
-                //     embeddings.print(true /* verbose */);
-                // });
-            }
-        }
-        else {
-            console.log("model not found :(");
-        }
-    }
-
-    /***
-     * Uses model to convert words to vectors
-     */
-
-    public async vectorize(text: string[]): Promise<Tensor | undefined> {
-        if (!this._model) {
-            await this.loadTFModel();
-        }
-        if (this._model) {
-            if (this.choice === "WV") {
-                let word_vecs = this._model.getVectors(text);
-                return word_vecs;
-            }
-            else if (this.choice === "TF") {
-                const model = this._model as use.UniversalSentenceEncoder;
-                return new Promise<Tensor>(res => {
-                    model.embed(text).then(embeddings => {
-                        res(embeddings);
-                    });
-                });
-
-            }
-        }
-    }
-
-    // public async trainModel() {
-    //     console.log("phrasing...");
-    //     w2v.word2vec("./node_modules/word2vec/examples/eng_news-typical_2016_1M-sentences.txt", './node_modules/word2vec/examples/my_phrases.txt', {
-    //         cbow: 1,
-    //         size: 200,
-    //         window: 8,
-    //         negative: 25,
-    //         hs: 0,
-    //         sample: 1e-4,
-    //         threads: 20,
-    //         iter: 200,
-    //         minCount: 2
-    //     });
-    //     console.log("phrased!!!");
-    // }
-
-}
+// //import { Doc } from "../new_fields/Doc";
+// //import { StrCast } from "../new_fields/Types";
+// //import { List } from "../new_fields/List";
+// //import { CognitiveServices } from "../client/cognitive_services/CognitiveServices";
+
+// // var w2v = require('word2vec');
+// var assert = require('assert');
+// var arxivapi = require('arxiv-api-node');
+// import requestPromise = require("request-promise");
+// import * as use from '@tensorflow-models/universal-sentence-encoder';
+// import { Tensor } from "@tensorflow/tfjs-core/dist/tensor";
+// require('@tensorflow/tfjs-node');
+
+// //http://gnuwin32.sourceforge.net/packages/make.htm
+
+// export class Recommender {
+
+//     private _model: any;
+//     static Instance: Recommender;
+//     private dimension: number = 0;
+//     private choice: string = ""; // Tensorflow or Word2Vec
+
+//     constructor() {
+//         console.log("creating recommender...");
+//         Recommender.Instance = this;
+//     }
+
+//     /***
+//      * Loads pre-trained model from TF
+//      */
+
+//     public async loadTFModel() {
+//         let self = this;
+//         return new Promise(res => {
+//             use.load().then(model => {
+//                 self.choice = "TF";
+//                 self._model = model;
+//                 self.dimension = 512;
+//                 res(model);
+//             });
+//         }
+
+//         );
+//     }
+
+//     /***
+//      * Loads pre-trained model from word2vec
+//      */
+
+//     // private loadModel(): Promise<any> {
+//     //     let self = this;
+//     //     return new Promise(res => {
+//     //         w2v.loadModel("./node_modules/word2vec/examples/fixtures/vectors.txt", function (err: any, model: any) {
+//     //             self.choice = "WV";
+//     //             self._model = model;
+//     //             self.dimension = model.size;
+//     //             res(model);
+//     //         });
+//     //     });
+//     // }
+
+//     /***
+//      * Testing
+//      */
+
+//     public async testModel() {
+//         if (!this._model) {
+//             await this.loadTFModel();
+//         }
+//         if (this._model) {
+//             if (this.choice === "WV") {
+//                 let similarity = this._model.similarity('father', 'mother');
+//                 console.log(similarity);
+//             }
+//             else if (this.choice === "TF") {
+//                 const model = this._model as use.UniversalSentenceEncoder;
+//                 // Embed an array of sentences.
+//                 const sentences = [
+//                     'Hello.',
+//                     'How are you?'
+//                 ];
+//                 const embeddings = await this.vectorize(sentences);
+//                 if (embeddings) embeddings.print(true /*verbose*/);
+//                 // model.embed(sentences).then(embeddings => {
+//                 //     // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
+//                 //     // So in this example `embeddings` has the shape [2, 512].
+//                 //     embeddings.print(true /* verbose */);
+//                 // });
+//             }
+//         }
+//         else {
+//             console.log("model not found :(");
+//         }
+//     }
+
+//     /***
+//      * Uses model to convert words to vectors
+//      */
+
+//     public async vectorize(text: string[]): Promise<Tensor | undefined> {
+//         if (!this._model) {
+//             await this.loadTFModel();
+//         }
+//         if (this._model) {
+//             if (this.choice === "WV") {
+//                 let word_vecs = this._model.getVectors(text);
+//                 return word_vecs;
+//             }
+//             else if (this.choice === "TF") {
+//                 const model = this._model as use.UniversalSentenceEncoder;
+//                 return new Promise<Tensor>(res => {
+//                     model.embed(text).then(embeddings => {
+//                         res(embeddings);
+//                     });
+//                 });
+
+//             }
+//         }
+//     }
+
+//     // public async trainModel() {
+//     //     console.log("phrasing...");
+//     //     w2v.word2vec("./node_modules/word2vec/examples/eng_news-typical_2016_1M-sentences.txt", './node_modules/word2vec/examples/my_phrases.txt', {
+//     //         cbow: 1,
+//     //         size: 200,
+//     //         window: 8,
+//     //         negative: 25,
+//     //         hs: 0,
+//     //         sample: 1e-4,
+//     //         threads: 20,
+//     //         iter: 200,
+//     //         minCount: 2
+//     //     });
+//     //     console.log("phrased!!!");
+//     // }
+
+// }
diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts
index 185e787cc..3d3683912 100644
--- a/src/server/SharedMediaTypes.ts
+++ b/src/server/SharedMediaTypes.ts
@@ -10,6 +10,7 @@ export namespace AcceptibleMedia {
     export const imageFormats = [...pngs, ...jpgs, ...gifs, ...webps, ...tiffs];
     export const videoFormats = [".mov", ".mp4"];
     export const applicationFormats = [".pdf"];
+    export const audioFormats = [".wav", ".mp3", ".flac", ".au", ".aiff", ".m4a", ".webm;codecs=opus"];
 }
 
 export namespace Upload {