merged physics with master

4 files changed, 144 insertions, 122 deletions

diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts
new file mode 100644
index 000000000..0d43130d1
--- /dev/null
+++ b/src/server/ApiManagers/DataVizManager.ts
@@ -0,0 +1,26 @@
+import { csvParser, csvToString } from "../DataVizUtils";
+import { Method, _success } from "../RouteManager";
+import ApiManager, { Registration } from "./ApiManager";
+import { Directory, serverPathToFile } from "./UploadManager";
+import * as path from 'path';
+
+export default class DataVizManager extends ApiManager {
+    protected initialize(register: Registration): void {
+        register({
+            method: Method.GET,
+            subscription: "/csvData",
+            secureHandler: async ({ req, res }) => {
+                const uri = req.query.uri as string;
+
+                return new Promise<void>(resolve => {
+                    const name = path.basename(uri);
+                    const sPath = serverPathToFile(Directory.csv, name);
+                    const parsedCsv = csvParser(csvToString(sPath));
+                    _success(res, parsedCsv);
+                    resolve();
+                });
+            }
+        });
+    }
+    
+}
\ No newline at end of file
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
index a74e13a62..186f0bcd3 100644
--- a/src/server/ApiManagers/SearchManager.ts
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -1,94 +1,89 @@
-import { exec } from "child_process";
-import { cyan, green, red, yellow } from "colors";
+import { exec } from 'child_process';
+import { cyan, green, red, yellow } from 'colors';
 import * as path from 'path';
-import { log_execution } from "../ActionUtilities";
-import { Database } from "../database";
-import { Method } from "../RouteManager";
-import RouteSubscriber from "../RouteSubscriber";
-import { Search } from "../Search";
-import ApiManager, { Registration } from "./ApiManager";
-import { Directory, pathToDirectory } from "./UploadManager";
+import { log_execution } from '../ActionUtilities';
+import { Database } from '../database';
+import { Method } from '../RouteManager';
+import RouteSubscriber from '../RouteSubscriber';
+import { Search } from '../Search';
+import ApiManager, { Registration } from './ApiManager';
+import { Directory, pathToDirectory } from './UploadManager';
 const findInFiles = require('find-in-files');
 
 export class SearchManager extends ApiManager {
-
     protected initialize(register: Registration): void {
-
         register({
             method: Method.GET,
-            subscription: new RouteSubscriber("solr").add("action"),
+            subscription: new RouteSubscriber('solr').add('action'),
             secureHandler: async ({ req, res }) => {
                 const { action } = req.params;
                 switch (action) {
-                    case "start":
-                    case "stop":
-                        const status = req.params.action === "start";
+                    case 'start':
+                    case 'stop':
+                        const status = req.params.action === 'start';
                         SolrManager.SetRunning(status);
                         break;
-                    case "update":
+                    case 'update':
                         await SolrManager.update();
                         break;
                     default:
                         console.log(yellow(`${action} is an unknown solr operation.`));
                 }
-                res.redirect("/home");
-            }
+                res.redirect('/home');
+            },
         });
 
         register({
             method: Method.GET,
-            subscription: "/textsearch",
+            subscription: '/textsearch',
             secureHandler: async ({ req, res }) => {
                 const q = req.query.q;
                 if (q === undefined) {
                     res.send([]);
                     return;
                 }
-                const resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] };
+                const resObj: { ids: string[]; numFound: number; lines: string[] } = { ids: [], numFound: 0, lines: [] };
                 let results: any;
                 const dir = pathToDirectory(Directory.text);
                 try {
                     const regex = new RegExp(q.toString());
-                    results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, dir, ".txt$");
+                    results = await findInFiles.find({ term: q, flags: 'ig' }, dir, '.txt$');
                     for (const result in results) {
-                        resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, ""));
+                        resObj.ids.push(path.basename(result, '.txt').replace(/upload_/, ''));
                         resObj.lines.push(results[result].line);
                         resObj.numFound++;
                     }
                     res.send(resObj);
                 } catch (e) {
-                    console.log(red("textsearch:bad RegExp" + q.toString()));
+                    console.log(red('textsearch:bad RegExp' + q.toString()));
                     res.send([]);
                     return;
                 }
-            }
+            },
         });
 
         register({
             method: Method.GET,
-            subscription: "/dashsearch",
+            subscription: '/dashsearch',
             secureHandler: async ({ req, res }) => {
                 const solrQuery: any = {};
-                ["q", "fq", "start", "rows", "sort", "hl.maxAnalyzedChars", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]);
+                ['q', 'fq', 'start', 'rows', 'sort', 'hl.maxAnalyzedChars', 'hl', 'hl.fl'].forEach(key => (solrQuery[key] = req.query[key]));
                 if (solrQuery.q === undefined) {
                     res.send([]);
                     return;
                 }
                 const results = await Search.search(solrQuery);
                 res.send(results);
-            }
+            },
         });
-
     }
-
 }
 
 export namespace SolrManager {
-
     export function SetRunning(status: boolean) {
-        const args = status ? "start" : "stop -p 8983";
+        const args = status ? 'start' : 'stop -p 8983';
         console.log(`solr management: trying to ${args}`);
-        exec(`solr ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => {
+        exec(`solr ${args}`, { cwd: './solr-8.3.1/bin' }, (error, stdout, stderr) => {
             if (error) {
                 console.log(red(`solr management error: unable to ${args} server`));
                 console.log(red(error.message));
@@ -97,39 +92,39 @@ export namespace SolrManager {
             console.log(yellow(stderr));
         });
         if (status) {
-            console.log(cyan("Start script is executing: please allow 15 seconds for solr to start on port 8983."));
+            console.log(cyan('Start script is executing: please allow 15 seconds for solr to start on port 8983.'));
         }
     }
 
     export async function update() {
-        console.log(green("Beginning update..."));
+        console.log(green('Beginning update...'));
         await log_execution<void>({
-            startMessage: "Clearing existing Solr information...",
-            endMessage: "Solr information successfully cleared",
+            startMessage: 'Clearing existing Solr information...',
+            endMessage: 'Solr information successfully cleared',
             action: Search.clear,
-            color: cyan
+            color: cyan,
         });
         const cursor = await log_execution({
-            startMessage: "Connecting to and querying for all documents from database...",
+            startMessage: 'Connecting to and querying for all documents from database...',
             endMessage: ({ result, error }) => {
                 const success = error === null && result !== undefined;
                 if (!success) {
-                    console.log(red("Unable to connect to the database."));
+                    console.log(red('Unable to connect to the database.'));
                     process.exit(0);
                 }
-                return "Connection successful and query complete";
+                return 'Connection successful and query complete';
             },
             action: () => Database.Instance.query({}),
-            color: yellow
+            color: yellow,
         });
         const updates: any[] = [];
         let numDocs = 0;
         function updateDoc(doc: any) {
             numDocs++;
-            if ((numDocs % 50) === 0) {
+            if (numDocs % 50 === 0) {
                 console.log(`Batch of 50 complete, total of ${numDocs}`);
             }
-            if (doc.__type !== "Doc") {
+            if (doc.__type !== 'Doc') {
                 return;
             }
             const fields = doc.fields;
@@ -143,8 +138,8 @@ export namespace SolrManager {
                 const term = ToSearchTerm(value);
                 if (term !== undefined) {
                     const { suffix, value } = term;
-                    if (key.endsWith('lastModified')) {
-                        update["lastModified" + suffix] = value;
+                    if (key.endsWith('modificationDate')) {
+                        update['modificationDate' + suffix] = value;
                     }
                     update[key + suffix] = value;
                     dynfield = true;
@@ -157,51 +152,54 @@ export namespace SolrManager {
         await cursor?.forEach(updateDoc);
         const result = await log_execution({
             startMessage: `Dispatching updates for ${updates.length} documents`,
-            endMessage: "Dispatched updates complete",
+            endMessage: 'Dispatched updates complete',
             action: () => Search.updateDocuments(updates),
-            color: cyan
+            color: cyan,
         });
         try {
             if (result) {
                 const { status } = JSON.parse(result).responseHeader;
-                console.log(status ? red(`Failed with status code (${status})`) : green("Success!"));
+                console.log(status ? red(`Failed with status code (${status})`) : green('Success!'));
             } else {
-                console.log(red("Solr is likely not running!"));
+                console.log(red('Solr is likely not running!'));
             }
         } catch (e) {
-            console.log(red("Error:"));
+            console.log(red('Error:'));
             console.log(e);
-            console.log("\n");
+            console.log('\n');
         }
         await cursor?.close();
     }
 
-    const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
-        "number": "_n",
-        "string": "_t",
-        "boolean": "_b",
-        "image": ["_t", "url"],
-        "video": ["_t", "url"],
-        "pdf": ["_t", "url"],
-        "audio": ["_t", "url"],
-        "web": ["_t", "url"],
-        "map": ["_t", "url"],
-        "date": ["_d", value => new Date(value.date).toISOString()],
-        "proxy": ["_i", "fieldId"],
-        "prefetch_proxy": ["_i", "fieldId"],
-        "list": ["_l", list => {
-            const results = [];
-            for (const value of list.fields) {
-                const term = ToSearchTerm(value);
-                if (term) {
-                    results.push(term.value);
+    const suffixMap: { [type: string]: string | [string, string | ((json: any) => any)] } = {
+        number: '_n',
+        string: '_t',
+        boolean: '_b',
+        image: ['_t', 'url'],
+        video: ['_t', 'url'],
+        pdf: ['_t', 'url'],
+        audio: ['_t', 'url'],
+        web: ['_t', 'url'],
+        map: ['_t', 'url'],
+        date: ['_d', value => new Date(value.date).toISOString()],
+        proxy: ['_i', 'fieldId'],
+        prefetch_proxy: ['_i', 'fieldId'],
+        list: [
+            '_l',
+            list => {
+                const results = [];
+                for (const value of list.fields) {
+                    const term = ToSearchTerm(value);
+                    if (term) {
+                        results.push(term.value);
+                    }
                 }
-            }
-            return results.length ? results : null;
-        }]
+                return results.length ? results : null;
+            },
+        ],
     };
 
-    function ToSearchTerm(val: any): { suffix: string, value: any } | undefined {
+    function ToSearchTerm(val: any): { suffix: string; value: any } | undefined {
         if (val === null || val === undefined) {
             return;
         }
@@ -213,7 +211,7 @@ export namespace SolrManager {
 
         if (Array.isArray(suffix)) {
             const accessor = suffix[1];
-            if (typeof accessor === "function") {
+            if (typeof accessor === 'function') {
                 val = accessor(val);
             } else {
                 val = val[accessor];
@@ -223,5 +221,4 @@ export namespace SolrManager {
 
         return { suffix, value: val };
     }
-
-}
\ No newline at end of file
+}
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index fe4c475c9..74c06b4a6 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -1,19 +1,19 @@
-import ApiManager, { Registration } from './ApiManager';
-import { Method, _success } from '../RouteManager';
 import * as formidable from 'formidable';
-import v4 = require('uuid/v4');
-const AdmZip = require('adm-zip');
-import { extname, basename, dirname } from 'path';
 import { createReadStream, createWriteStream, unlink, writeFile } from 'fs';
-import { publicDirectory, filesDirectory } from '..';
-import { Database } from '../database';
-import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils';
+import { basename, dirname, extname, normalize } from 'path';
 import * as sharp from 'sharp';
-import { AcceptableMedia, Upload } from '../SharedMediaTypes';
-import { normalize } from 'path';
+import { filesDirectory, publicDirectory } from '..';
+import { retrocycle } from '../../decycler/decycler';
+import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils';
+import { Database } from '../database';
+import { Method, _success } from '../RouteManager';
 import RouteSubscriber from '../RouteSubscriber';
-const imageDataUri = require('image-data-uri');
+import { AcceptableMedia, Upload } from '../SharedMediaTypes';
+import ApiManager, { Registration } from './ApiManager';
 import { SolrManager } from './SearchManager';
+import v4 = require('uuid/v4');
+const AdmZip = require('adm-zip');
+const imageDataUri = require('image-data-uri');
 const fs = require('fs');
 
 export enum Directory {
@@ -180,13 +180,9 @@ export default class UploadManager extends ApiManager {
                 const ids: { [id: string]: string } = {};
                 let remap = true;
                 const getId = (id: string): string => {
-                    if (!remap) return id;
-                    if (id.endsWith('Proto')) return id;
-                    if (id in ids) {
-                        return ids[id];
-                    } else {
-                        return (ids[id] = v4());
-                    }
+                    if (!remap || id.endsWith('Proto')) return id;
+                    if (id in ids) return ids[id];
+                    return (ids[id] = v4());
                 };
                 const mapFn = (doc: any) => {
                     if (doc.id) {
@@ -228,48 +224,50 @@ export default class UploadManager extends ApiManager {
                     form.parse(req, async (_err, fields, files) => {
                         remap = fields.remap !== 'false';
                         let id: string = '';
+                        let docids: string[] = [];
+                        let linkids: string[] = [];
                         try {
                             for (const name in files) {
                                 const f = files[name];
                                 const path_2 = Array.isArray(f) ? '' : f.path;
                                 const zip = new AdmZip(path_2);
                                 zip.getEntries().forEach((entry: any) => {
-                                    if (!entry.entryName.startsWith('files/')) return;
-                                    let directory = dirname(entry.entryName) + '/';
-                                    const extension = extname(entry.entryName);
-                                    const base = basename(entry.entryName).split('.')[0];
+                                    let entryName = entry.entryName.replace(/%%%/g, '/');
+                                    if (!entryName.startsWith('files/')) {
+                                        return;
+                                    }
+                                    const extension = extname(entryName);
+                                    const pathname = publicDirectory + '/' + entry.entryName;
+                                    const targetname = publicDirectory + '/' + entryName;
                                     try {
                                         zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
-                                        directory = '/' + directory;
-
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_o' + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_s' + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_m' + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_l' + extension));
+                                        createReadStream(pathname).pipe(createWriteStream(targetname));
+                                        if (extension !== '.pdf') {
+                                            createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_s' + extension)));
+                                            createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_m' + extension)));
+                                            createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_l' + extension)));
+                                        }
+                                        unlink(pathname, () => {});
                                     } catch (e) {
                                         console.log(e);
                                     }
                                 });
-                                const json = zip.getEntry('doc.json');
+                                const json = zip.getEntry('docs.json');
                                 try {
-                                    const data = JSON.parse(json.getData().toString('utf8'));
-                                    const datadocs = data.docs;
+                                    const data = JSON.parse(json.getData().toString('utf8'), retrocycle());
+                                    const { docs, links } = data;
                                     id = getId(data.id);
-                                    const docs = Object.keys(datadocs).map(key => datadocs[key]);
-                                    docs.forEach(mapFn);
+                                    const rdocs = Object.keys(docs).map(key => docs[key]);
+                                    const ldocs = Object.keys(links).map(key => links[key]);
+                                    [...rdocs, ...ldocs].forEach(mapFn);
+                                    docids = rdocs.map(doc => doc.id);
+                                    linkids = ldocs.map(link => link.id);
                                     await Promise.all(
-                                        docs.map(
-                                            (doc: any) =>
+                                        [...rdocs, ...ldocs].map(
+                                            doc =>
                                                 new Promise<void>(res => {
-                                                    Database.Instance.replace(
-                                                        doc.id,
-                                                        doc,
-                                                        (err, r) => {
-                                                            err && console.log(err);
-                                                            res();
-                                                        },
-                                                        true
-                                                    );
+                                                    // overwrite mongo doc with json doc contents
+                                                    Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true);
                                                 })
                                         )
                                     );
@@ -279,7 +277,7 @@ export default class UploadManager extends ApiManager {
                                 unlink(path_2, () => {});
                             }
                             SolrManager.update();
-                            res.send(JSON.stringify(id || 'error'));
+                            res.send(JSON.stringify({ id, docids, linkids } || 'error'));
                         } catch (e) {
                             console.log(e);
                         }
diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts
index 53e55c1c3..c3dadd821 100644
--- a/src/server/ApiManagers/UserManager.ts
+++ b/src/server/ApiManagers/UserManager.ts
@@ -5,6 +5,7 @@ import { msToTime } from '../ActionUtilities';
 import * as bcrypt from 'bcrypt-nodejs';
 import { Opt } from '../../fields/Doc';
 import { WebSocket } from '../websocket';
+import { DashStats } from '../DashStats';
 
 export const timeMap: { [id: string]: number } = {};
 interface ActivityUnit {