aboutsummaryrefslogtreecommitdiff
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/ApiManagers/UploadManager.ts14
-rw-r--r--src/server/ApiManagers/UtilManager.ts44
-rw-r--r--src/server/DashUploadUtils.ts19
-rw-r--r--src/server/Recommender.ts274
-rw-r--r--src/server/SharedMediaTypes.ts1
5 files changed, 188 insertions, 164 deletions
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 50a759c9d..42e33ece0 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -19,7 +19,8 @@ export enum Directory {
videos = "videos",
pdfs = "pdfs",
text = "text",
- pdf_thumbnails = "pdf_thumbnails"
+ pdf_thumbnails = "pdf_thumbnails",
+ audio = "audio"
}
export function serverPathToFile(directory: Directory, filename: string) {
@@ -61,9 +62,18 @@ export default class UploadManager extends ApiManager {
});
register({
+ method: Method.GET,
+ subscription: "/hello",
+ secureHandler: ({ req, res }) => {
+ res.send("<h1>world!</h1>");
+ }
+ });
+
+ register({
method: Method.POST,
subscription: "/uploadRemoteImage",
secureHandler: async ({ req, res }) => {
+
const { sources } = req.body;
if (Array.isArray(sources)) {
const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source)));
@@ -77,6 +87,7 @@ export default class UploadManager extends ApiManager {
method: Method.POST,
subscription: "/uploadDoc",
secureHandler: ({ req, res }) => {
+
const form = new formidable.IncomingForm();
form.keepExtensions = true;
// let path = req.body.path;
@@ -181,6 +192,7 @@ export default class UploadManager extends ApiManager {
method: Method.POST,
subscription: "/inspectImage",
secureHandler: async ({ req, res }) => {
+
const { source } = req.body;
if (typeof source === "string") {
return res.send(await DashUploadUtils.InspectImage(source));
diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts
index d18529cf2..ad8119bf4 100644
--- a/src/server/ApiManagers/UtilManager.ts
+++ b/src/server/ApiManagers/UtilManager.ts
@@ -3,11 +3,11 @@ import { Method } from "../RouteManager";
import { exec } from 'child_process';
import RouteSubscriber from "../RouteSubscriber";
import { red } from "colors";
-import { IBM_Recommender } from "../../client/apis/IBM_Recommender";
-import { Recommender } from "../Recommender";
+// import { IBM_Recommender } from "../../client/apis/IBM_Recommender";
+// import { Recommender } from "../Recommender";
-const recommender = new Recommender();
-recommender.testModel();
+// const recommender = new Recommender();
+// recommender.testModel();
import executeImport from "../../scraping/buxton/final/BuxtonImporter";
export default class UtilManager extends ApiManager {
@@ -27,25 +27,25 @@ export default class UtilManager extends ApiManager {
}
});
- register({
- method: Method.POST,
- subscription: "/IBMAnalysis",
- secureHandler: async ({ req, res }) => res.send(await IBM_Recommender.analyze(req.body))
- });
+ // register({
+ // method: Method.POST,
+ // subscription: "/IBMAnalysis",
+ // secureHandler: async ({ req, res }) => res.send(await IBM_Recommender.analyze(req.body))
+ // });
- register({
- method: Method.POST,
- subscription: "/recommender",
- secureHandler: async ({ req, res }) => {
- const keyphrases = req.body.keyphrases;
- const wordvecs = await recommender.vectorize(keyphrases);
- let embedding: Float32Array = new Float32Array();
- if (wordvecs && wordvecs.dataSync()) {
- embedding = wordvecs.dataSync() as Float32Array;
- }
- res.send(embedding);
- }
- });
+ // register({
+ // method: Method.POST,
+ // subscription: "/recommender",
+ // secureHandler: async ({ req, res }) => {
+ // const keyphrases = req.body.keyphrases;
+ // const wordvecs = await recommender.vectorize(keyphrases);
+ // let embedding: Float32Array = new Float32Array();
+ // if (wordvecs && wordvecs.dataSync()) {
+ // embedding = wordvecs.dataSync() as Float32Array;
+ // }
+ // res.send(embedding);
+ // }
+ // });
register({
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index cc3dd75a4..cf78af60a 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -53,7 +53,7 @@ export namespace DashUploadUtils {
const size = "content-length";
const type = "content-type";
- const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia;
+ const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptibleMedia;
export async function upload(file: File): Promise<Upload.FileResponse> {
const { type, path, name } = file;
@@ -76,14 +76,22 @@ export namespace DashUploadUtils {
if (applicationFormats.includes(format)) {
return UploadPdf(file);
}
- default: // "blob":
- return MoveParsedFile(file, Directory.videos);
+ case "audio":
+ if (audioFormats.includes(format)) {
+ return MoveParsedFile(file, Directory.audio);
+ }
}
console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`));
return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) };
}
+ async function UploadAudio(file: File) {
+ const { path: sourcePath } = file;
+
+ return MoveParsedFile(file, Directory.audio);
+ }
+
async function UploadPdf(file: File) {
const { path: sourcePath } = file;
const dataBuffer = readFileSync(sourcePath);
@@ -94,6 +102,7 @@ export namespace DashUploadUtils {
const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
writeStream.write(result.text, error => error ? reject(error) : resolve());
});
+ console.log(MoveParsedFile(file, Directory.pdfs));
return MoveParsedFile(file, Directory.pdfs);
}
@@ -197,8 +206,10 @@ export namespace DashUploadUtils {
accessPaths: {
agnostic: getAccessPaths(destination, name)
}
+
}
- });
+ }
+ );
});
});
}
diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts
index 1d2cb3858..aacdb4053 100644
--- a/src/server/Recommender.ts
+++ b/src/server/Recommender.ts
@@ -1,137 +1,137 @@
-//import { Doc } from "../new_fields/Doc";
-//import { StrCast } from "../new_fields/Types";
-//import { List } from "../new_fields/List";
-//import { CognitiveServices } from "../client/cognitive_services/CognitiveServices";
-
-// var w2v = require('word2vec');
-var assert = require('assert');
-var arxivapi = require('arxiv-api-node');
-import requestPromise = require("request-promise");
-import * as use from '@tensorflow-models/universal-sentence-encoder';
-import { Tensor } from "@tensorflow/tfjs-core/dist/tensor";
-require('@tensorflow/tfjs-node');
-
-//http://gnuwin32.sourceforge.net/packages/make.htm
-
-export class Recommender {
-
- private _model: any;
- static Instance: Recommender;
- private dimension: number = 0;
- private choice: string = ""; // Tensorflow or Word2Vec
-
- constructor() {
- console.log("creating recommender...");
- Recommender.Instance = this;
- }
-
- /***
- * Loads pre-trained model from TF
- */
-
- public async loadTFModel() {
- let self = this;
- return new Promise(res => {
- use.load().then(model => {
- self.choice = "TF";
- self._model = model;
- self.dimension = 512;
- res(model);
- });
- }
-
- );
- }
-
- /***
- * Loads pre-trained model from word2vec
- */
-
- // private loadModel(): Promise<any> {
- // let self = this;
- // return new Promise(res => {
- // w2v.loadModel("./node_modules/word2vec/examples/fixtures/vectors.txt", function (err: any, model: any) {
- // self.choice = "WV";
- // self._model = model;
- // self.dimension = model.size;
- // res(model);
- // });
- // });
- // }
-
- /***
- * Testing
- */
-
- public async testModel() {
- if (!this._model) {
- await this.loadTFModel();
- }
- if (this._model) {
- if (this.choice === "WV") {
- let similarity = this._model.similarity('father', 'mother');
- console.log(similarity);
- }
- else if (this.choice === "TF") {
- const model = this._model as use.UniversalSentenceEncoder;
- // Embed an array of sentences.
- const sentences = [
- 'Hello.',
- 'How are you?'
- ];
- const embeddings = await this.vectorize(sentences);
- if (embeddings) embeddings.print(true /*verbose*/);
- // model.embed(sentences).then(embeddings => {
- // // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
- // // So in this example `embeddings` has the shape [2, 512].
- // embeddings.print(true /* verbose */);
- // });
- }
- }
- else {
- console.log("model not found :(");
- }
- }
-
- /***
- * Uses model to convert words to vectors
- */
-
- public async vectorize(text: string[]): Promise<Tensor | undefined> {
- if (!this._model) {
- await this.loadTFModel();
- }
- if (this._model) {
- if (this.choice === "WV") {
- let word_vecs = this._model.getVectors(text);
- return word_vecs;
- }
- else if (this.choice === "TF") {
- const model = this._model as use.UniversalSentenceEncoder;
- return new Promise<Tensor>(res => {
- model.embed(text).then(embeddings => {
- res(embeddings);
- });
- });
-
- }
- }
- }
-
- // public async trainModel() {
- // console.log("phrasing...");
- // w2v.word2vec("./node_modules/word2vec/examples/eng_news-typical_2016_1M-sentences.txt", './node_modules/word2vec/examples/my_phrases.txt', {
- // cbow: 1,
- // size: 200,
- // window: 8,
- // negative: 25,
- // hs: 0,
- // sample: 1e-4,
- // threads: 20,
- // iter: 200,
- // minCount: 2
- // });
- // console.log("phrased!!!");
- // }
-
-}
+// //import { Doc } from "../new_fields/Doc";
+// //import { StrCast } from "../new_fields/Types";
+// //import { List } from "../new_fields/List";
+// //import { CognitiveServices } from "../client/cognitive_services/CognitiveServices";
+
+// // var w2v = require('word2vec');
+// var assert = require('assert');
+// var arxivapi = require('arxiv-api-node');
+// import requestPromise = require("request-promise");
+// import * as use from '@tensorflow-models/universal-sentence-encoder';
+// import { Tensor } from "@tensorflow/tfjs-core/dist/tensor";
+// require('@tensorflow/tfjs-node');
+
+// //http://gnuwin32.sourceforge.net/packages/make.htm
+
+// export class Recommender {
+
+// private _model: any;
+// static Instance: Recommender;
+// private dimension: number = 0;
+// private choice: string = ""; // Tensorflow or Word2Vec
+
+// constructor() {
+// console.log("creating recommender...");
+// Recommender.Instance = this;
+// }
+
+// /***
+// * Loads pre-trained model from TF
+// */
+
+// public async loadTFModel() {
+// let self = this;
+// return new Promise(res => {
+// use.load().then(model => {
+// self.choice = "TF";
+// self._model = model;
+// self.dimension = 512;
+// res(model);
+// });
+// }
+
+// );
+// }
+
+// /***
+// * Loads pre-trained model from word2vec
+// */
+
+// // private loadModel(): Promise<any> {
+// // let self = this;
+// // return new Promise(res => {
+// // w2v.loadModel("./node_modules/word2vec/examples/fixtures/vectors.txt", function (err: any, model: any) {
+// // self.choice = "WV";
+// // self._model = model;
+// // self.dimension = model.size;
+// // res(model);
+// // });
+// // });
+// // }
+
+// /***
+// * Testing
+// */
+
+// public async testModel() {
+// if (!this._model) {
+// await this.loadTFModel();
+// }
+// if (this._model) {
+// if (this.choice === "WV") {
+// let similarity = this._model.similarity('father', 'mother');
+// console.log(similarity);
+// }
+// else if (this.choice === "TF") {
+// const model = this._model as use.UniversalSentenceEncoder;
+// // Embed an array of sentences.
+// const sentences = [
+// 'Hello.',
+// 'How are you?'
+// ];
+// const embeddings = await this.vectorize(sentences);
+// if (embeddings) embeddings.print(true /*verbose*/);
+// // model.embed(sentences).then(embeddings => {
+// // // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence.
+// // // So in this example `embeddings` has the shape [2, 512].
+// // embeddings.print(true /* verbose */);
+// // });
+// }
+// }
+// else {
+// console.log("model not found :(");
+// }
+// }
+
+// /***
+// * Uses model to convert words to vectors
+// */
+
+// public async vectorize(text: string[]): Promise<Tensor | undefined> {
+// if (!this._model) {
+// await this.loadTFModel();
+// }
+// if (this._model) {
+// if (this.choice === "WV") {
+// let word_vecs = this._model.getVectors(text);
+// return word_vecs;
+// }
+// else if (this.choice === "TF") {
+// const model = this._model as use.UniversalSentenceEncoder;
+// return new Promise<Tensor>(res => {
+// model.embed(text).then(embeddings => {
+// res(embeddings);
+// });
+// });
+
+// }
+// }
+// }
+
+// // public async trainModel() {
+// // console.log("phrasing...");
+// // w2v.word2vec("./node_modules/word2vec/examples/eng_news-typical_2016_1M-sentences.txt", './node_modules/word2vec/examples/my_phrases.txt', {
+// // cbow: 1,
+// // size: 200,
+// // window: 8,
+// // negative: 25,
+// // hs: 0,
+// // sample: 1e-4,
+// // threads: 20,
+// // iter: 200,
+// // minCount: 2
+// // });
+// // console.log("phrased!!!");
+// // }
+
+// }
diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts
index 185e787cc..3d3683912 100644
--- a/src/server/SharedMediaTypes.ts
+++ b/src/server/SharedMediaTypes.ts
@@ -10,6 +10,7 @@ export namespace AcceptibleMedia {
export const imageFormats = [...pngs, ...jpgs, ...gifs, ...webps, ...tiffs];
export const videoFormats = [".mov", ".mp4"];
export const applicationFormats = [".pdf"];
+ export const audioFormats = [".wav", ".mp3", ".flac", ".au", ".aiff", ".m4a", ".webm;codecs=opus"];
}
export namespace Upload {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-25 21:31:00 +0000