Merge branch 'master' of https://github.com/brown-dash/Dash-Web

6 files changed, 358 insertions, 27 deletions

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 6e28268a9..74c06b4a6 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -180,13 +180,9 @@ export default class UploadManager extends ApiManager {
                 const ids: { [id: string]: string } = {};
                 let remap = true;
                 const getId = (id: string): string => {
-                    if (!remap) return id;
-                    if (id.endsWith('Proto')) return id;
-                    if (id in ids) {
-                        return ids[id];
-                    } else {
-                        return (ids[id] = v4());
-                    }
+                    if (!remap || id.endsWith('Proto')) return id;
+                    if (id in ids) return ids[id];
+                    return (ids[id] = v4());
                 };
                 const mapFn = (doc: any) => {
                     if (doc.id) {
@@ -229,6 +225,7 @@ export default class UploadManager extends ApiManager {
                         remap = fields.remap !== 'false';
                         let id: string = '';
                         let docids: string[] = [];
+                        let linkids: string[] = [];
                         try {
                             for (const name in files) {
                                 const f = files[name];
@@ -255,27 +252,22 @@ export default class UploadManager extends ApiManager {
                                         console.log(e);
                                     }
                                 });
-                                const json = zip.getEntry('doc.json');
+                                const json = zip.getEntry('docs.json');
                                 try {
                                     const data = JSON.parse(json.getData().toString('utf8'), retrocycle());
-                                    const datadocs = data.docs;
+                                    const { docs, links } = data;
                                     id = getId(data.id);
-                                    const docs = Object.keys(datadocs).map(key => datadocs[key]);
-                                    docs.forEach(mapFn);
-                                    docids = docs.map(doc => doc.id);
+                                    const rdocs = Object.keys(docs).map(key => docs[key]);
+                                    const ldocs = Object.keys(links).map(key => links[key]);
+                                    [...rdocs, ...ldocs].forEach(mapFn);
+                                    docids = rdocs.map(doc => doc.id);
+                                    linkids = ldocs.map(link => link.id);
                                     await Promise.all(
-                                        docs.map(
-                                            (doc: any) =>
+                                        [...rdocs, ...ldocs].map(
+                                            doc =>
                                                 new Promise<void>(res => {
-                                                    Database.Instance.replace(
-                                                        doc.id,
-                                                        doc,
-                                                        (err, r) => {
-                                                            err && console.log(err);
-                                                            res();
-                                                        },
-                                                        true
-                                                    );
+                                                    // overwrite mongo doc with json doc contents
+                                                    Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true);
                                                 })
                                         )
                                     );
@@ -285,7 +277,7 @@ export default class UploadManager extends ApiManager {
                                 unlink(path_2, () => {});
                             }
                             SolrManager.update();
-                            res.send(JSON.stringify({ id, docids } || 'error'));
+                            res.send(JSON.stringify({ id, docids, linkids } || 'error'));
                         } catch (e) {
                             console.log(e);
                         }
diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts
index 53e55c1c3..c3dadd821 100644
--- a/src/server/ApiManagers/UserManager.ts
+++ b/src/server/ApiManagers/UserManager.ts
@@ -5,6 +5,7 @@ import { msToTime } from '../ActionUtilities';
 import * as bcrypt from 'bcrypt-nodejs';
 import { Opt } from '../../fields/Doc';
 import { WebSocket } from '../websocket';
+import { DashStats } from '../DashStats';
 
 export const timeMap: { [id: string]: number } = {};
 interface ActivityUnit {
diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts
new file mode 100644
index 000000000..8d341db63
--- /dev/null
+++ b/src/server/DashStats.ts
@@ -0,0 +1,293 @@
+import { cyan, magenta } from 'colors';
+import { Response } from 'express';
+import SocketIO from 'socket.io';
+import { timeMap } from './ApiManagers/UserManager';
+import { WebSocket } from './websocket';
+const fs = require('fs');
+
+/**
+ * DashStats focuses on tracking user data for each session.
+ * 
+ * This includes time connected, number of operations, and
+ * the rate of their operations
+ */
+export namespace DashStats {
+    export const SAMPLING_INTERVAL = 1000; // in milliseconds (ms) - Time interval to update the frontend. 
+    export const RATE_INTERVAL = 10; // in seconds (s) - Used to calculate rate
+
+    const statsCSVFilename = './src/server/stats/userLoginStats.csv';
+    const columns = ['USERNAME', 'ACTION', 'TIME'];
+
+    /**
+     * UserStats holds the stats associated with a particular user. 
+     */
+    interface UserStats {
+        socketId: string;
+        username: string;
+        time: string;
+        operations: number;
+        rate: number;
+    }
+
+    /**
+     * UserLastOperations is the queue object for each user 
+     * storing their past operations. 
+     */
+    interface UserLastOperations {
+        sampleOperations: number; // stores how many operations total are in this rate section (10 sec, for example)
+        lastSampleOperations: number; // stores how many total operations were recorded at the last sample
+        previousOperationsQueue: number[]; // stores the operations to calculate rate. 
+    }
+
+    /**
+     * StatsDataBundle represents an object that will be sent to the frontend view
+     * on each websocket update. 
+     */
+    interface StatsDataBundle {
+        connectedUsers: UserStats[];
+    }
+
+    /**
+     * CSVStore represents how objects will be stored in the CSV
+     */
+    interface CSVStore {
+        USERNAME: string;
+        ACTION: string;
+        TIME: string;
+    }
+
+    /**
+     * ServerTraffic describes the current traffic going to the backend. 
+     */
+    enum ServerTraffic {
+        NOT_BUSY,
+        BUSY,
+        VERY_BUSY
+    }
+
+    // These values can be changed after further testing how many
+    // users correspond to each traffic level in Dash. 
+    const BUSY_SERVER_BOUND = 2;
+    const VERY_BUSY_SERVER_BOUND = 3;
+
+    const serverTrafficMessages = [
+        "Not Busy",
+        "Busy",
+        "Very Busy"
+    ]
+
+    // lastUserOperations maps each username to a UserLastOperations
+    // structure 
+    export const lastUserOperations = new Map<string, UserLastOperations>();
+
+    /**
+     * handleStats is called when the /stats route is called, providing a JSON
+     * object with relevant stats. In this case, we return the number of
+     * current connections and 
+     * @param res Response object from Express
+     */
+    export function handleStats(res: Response) {
+        let current = getCurrentStats();
+        const results: CSVStore[] = [];
+        res.json({
+                currentConnections: current.length,
+                socketMap: current,
+            });
+    }
+
+    /**
+     * getUpdatedStatesBundle() sends an updated copy of the current stats to the 
+     * frontend /statsview route via websockets. 
+     * 
+     * @returns a StatsDataBundle that is sent to the frontend view on each websocket update
+     */
+    export function getUpdatedStatsBundle(): StatsDataBundle {
+        let current = getCurrentStats();
+
+        return {
+            connectedUsers: current,
+        }
+    }
+
+    /**
+     * handleStatsView() is called when the /statsview route is called. This 
+     * will use pug to render a frontend view of the current stats
+     * 
+     * @param res 
+     */
+    export function handleStatsView(res: Response) {
+        let current = getCurrentStats();
+
+        let connectedUsers = current.map((socketPair) => {
+            return socketPair.time + " - " + socketPair.username + " Operations: " + socketPair.operations;
+        })
+
+        let serverTraffic = ServerTraffic.NOT_BUSY;
+        if(current.length < BUSY_SERVER_BOUND) {
+            serverTraffic = ServerTraffic.NOT_BUSY;
+        } else if(current.length >= BUSY_SERVER_BOUND && current.length < VERY_BUSY_SERVER_BOUND) {
+            serverTraffic = ServerTraffic.BUSY;
+        } else {
+            serverTraffic = ServerTraffic.VERY_BUSY;
+        }
+        
+        res.render("stats.pug", {
+            title: "Dash Stats",
+            numConnections: connectedUsers.length,
+            serverTraffic: serverTraffic,
+            serverTrafficMessage : serverTrafficMessages[serverTraffic],
+            connectedUsers: connectedUsers
+        });
+    }
+
+    /**
+     * logUserLogin() writes a login event to the CSV file. 
+     * 
+     * @param username the username in the format of "username@domain.com logged in"
+     * @param socket the websocket associated with the current connection
+     */
+    export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) {
+        if (!(username === undefined)) {
+            let currentDate = new Date();
+            console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`));
+
+            let toWrite: CSVStore = {
+                USERNAME : username,
+                ACTION : "loggedIn",
+                TIME : currentDate.toISOString()
+            }
+
+            let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"});
+            statsFile.write(convertToCSV(toWrite));
+            statsFile.end();
+            console.log(cyan(convertToCSV(toWrite)));
+        }
+    }
+
+    /**
+     * logUserLogout() writes a logout event to the CSV file. 
+     * 
+     * @param username the username in the format of "username@domain.com logged in"
+     * @param socket the websocket associated with the current connection. 
+     */
+    export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) {
+        if (!(username === undefined)) {
+            let currentDate = new Date();
+
+            let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"});
+            let toWrite: CSVStore = {
+                USERNAME : username,
+                ACTION : "loggedOut",
+                TIME : currentDate.toISOString()
+            }
+            statsFile.write(convertToCSV(toWrite));
+            statsFile.end();
+        }
+    }
+
+    /**
+     * getLastOperationsOrDefault() is a helper method that will attempt
+     * to query the lastUserOperations map for a specified username. If the
+     * username is not in the map, an empty UserLastOperations object is returned. 
+     * @param username 
+     * @returns the user's UserLastOperations structure or an empty
+     * UserLastOperations object (All values set to 0) if the username is not found.
+     */
+    function getLastOperationsOrDefault(username: string): UserLastOperations {
+        if(lastUserOperations.get(username) === undefined) {
+            let initializeOperationsQueue = [];
+            for(let i = 0; i < RATE_INTERVAL; i++) {
+                initializeOperationsQueue.push(0);
+            }
+            return {
+                sampleOperations: 0,
+                lastSampleOperations: 0,
+                previousOperationsQueue: initializeOperationsQueue
+            }
+        }
+        return lastUserOperations.get(username)!;
+    }
+
+    /**
+     * updateLastOperations updates a specific user's UserLastOperations information
+     * for the current sampling cycle. The method removes old/outdated counts for
+     * operations from the queue and adds new data for the current sampling 
+     * cycle to the queue, updating the total count as it goes. 
+     * @param lastOperationData the old UserLastOperations data that must be updated
+     * @param currentOperations the total number of operations measured for this sampling cycle.
+     * @returns the udpated UserLastOperations structure. 
+     */
+    function updateLastOperations(lastOperationData: UserLastOperations, currentOperations: number): UserLastOperations {
+        // create a copy of the UserLastOperations to modify
+        let newLastOperationData: UserLastOperations = {
+            sampleOperations: lastOperationData.sampleOperations,
+            lastSampleOperations: lastOperationData.lastSampleOperations,
+            previousOperationsQueue: lastOperationData.previousOperationsQueue.slice()
+        }
+
+        let newSampleOperations = newLastOperationData.sampleOperations;
+        newSampleOperations -= newLastOperationData.previousOperationsQueue.shift()!; // removes and returns the first element of the queue
+        let operationsThisCycle = currentOperations - lastOperationData.lastSampleOperations;
+        newSampleOperations += operationsThisCycle; // add the operations this cycle to find out what our count for the interval should be (e.g operations in the last 10 seconds)
+
+        // update values for the copy object
+        newLastOperationData.sampleOperations = newSampleOperations;
+
+        newLastOperationData.previousOperationsQueue.push(operationsThisCycle);
+        newLastOperationData.lastSampleOperations = currentOperations;
+
+        return newLastOperationData;
+    }
+
+    /**
+     * getUserOperationsOrDefault() is a helper method to get the user's total
+     * operations for the CURRENT sampling interval. The method will return 0 
+     * if the username is not in the userOperations map.
+     * @param username the username to search the map for
+     * @returns the total number of operations recorded up to this sampling cycle. 
+     */
+    function getUserOperationsOrDefault(username: string): number {
+        return WebSocket.userOperations.get(username) === undefined ? 0 : WebSocket.userOperations.get(username)!
+    }
+
+    /**
+     * getCurrentStats() calculates the total stats for this cycle. In this case,
+     * getCurrentStats() returns an Array of UserStats[] objects describing 
+     * the stats for each user
+     * @returns an array of UserStats storing data for each user at the current moment. 
+     */
+    function getCurrentStats(): UserStats[] {
+        let socketPairs: UserStats[] = [];
+        for (let [key, value] of WebSocket.socketMap) {
+            let username = value.split(' ')[0];
+            let connectionTime = new Date(timeMap[username]);
+
+            let connectionTimeString = connectionTime.toLocaleDateString() + " " + connectionTime.toLocaleTimeString();
+
+            if (!key.disconnected) {
+                let lastRecordedOperations = getLastOperationsOrDefault(username);
+                let currentUserOperationCount = getUserOperationsOrDefault(username);
+
+                socketPairs.push({ 
+                    socketId: key.id,
+                    username: username,
+                    time: connectionTimeString.includes("Invalid Date") ? "" : connectionTimeString,
+                    operations : WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0,
+                    rate: lastRecordedOperations.sampleOperations
+                });
+                lastUserOperations.set(username, updateLastOperations(lastRecordedOperations,currentUserOperationCount));
+            }
+        }
+        return socketPairs;
+    }
+
+    /**
+     * convertToCSV() is a helper method that stringifies a CSVStore object
+     * that can be written to the CSV file later. 
+     * @param dataObject the object to stringify
+     * @returns the object as a string. 
+     */
+    function convertToCSV(dataObject: CSVStore): string {
+        return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`;
+    }
+}
diff --git a/src/server/Message.ts b/src/server/Message.ts
index d87ae5027..8f0af08bc 100644
--- a/src/server/Message.ts
+++ b/src/server/Message.ts
@@ -94,4 +94,6 @@ export namespace MessageStore {
     export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query");
     export const DeleteField = new Message<string>("Delete field");
     export const DeleteFields = new Message<string[]>("Delete fields");
+
+    export const UpdateStats = new Message<string>("updatestats");
 }
diff --git a/src/server/index.ts b/src/server/index.ts
index 6562860fe..8b2e18847 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -19,6 +19,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader';
 import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils';
 import { DashSessionAgent } from './DashSession/DashSessionAgent';
 import { AppliedSessionAgent } from './DashSession/Session/agents/applied_session_agent';
+import { DashStats } from './DashStats';
 import { DashUploadUtils } from './DashUploadUtils';
 import { Database } from './database';
 import { Logger } from './ProcessFactory';
@@ -98,6 +99,18 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }:
 
     addSupervisedRoute({
         method: Method.GET,
+        subscription: '/stats',
+        secureHandler: ({ res }) => DashStats.handleStats(res),
+    });
+
+    addSupervisedRoute({
+        method: Method.GET,
+        subscription: '/statsview',
+        secureHandler: ({ res }) => DashStats.handleStatsView(res),
+    });
+
+    addSupervisedRoute({
+        method: Method.GET,
         subscription: '/resolvedPorts',
         secureHandler: ({ res }) => res.send(resolvedPorts),
         publicHandler: ({ res }) => res.send(resolvedPorts),
diff --git a/src/server/websocket.ts b/src/server/websocket.ts
index 68b003496..a11d20cfa 100644
--- a/src/server/websocket.ts
+++ b/src/server/websocket.ts
@@ -12,6 +12,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader';
 import YoutubeApi from './apis/youtube/youtubeApiSample';
 import { initializeGuest } from './authentication/DashUserModel';
 import { Client } from './Client';
+import { DashStats } from './DashStats';
 import { Database } from './database';
 import { DocumentsCollection } from './IDatabase';
 import { Diff, GestureContent, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, Transferable, Types, UpdateMobileInkOverlayPositionContent, YoutubeQueryInput, YoutubeQueryTypes } from './Message';
@@ -20,8 +21,9 @@ import { resolvedPorts } from './server_Initialization';
 
 export namespace WebSocket {
     export let _socket: Socket;
-    const clients: { [key: string]: Client } = {};
+    export const clients: { [key: string]: Client } = {};
     export const socketMap = new Map<SocketIO.Socket, string>();
+    export const userOperations = new Map<string, number>();
     export let disconnect: Function;
 
     export async function initialize(isRelease: boolean, app: express.Express) {
@@ -49,6 +51,8 @@ export namespace WebSocket {
                 next();
             });
 
+            socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle())
+
             // convenience function to log server messages on the client
             function log(message?: any, ...optionalParams: any[]) {
                 socket.emit('log', ['Message from server:', message, ...optionalParams]);
@@ -97,6 +101,17 @@ export namespace WebSocket {
                 console.log('received bye');
             });
 
+            socket.on('disconnect', function () {
+                let currentUser = socketMap.get(socket);
+                if (!(currentUser === undefined)) {
+                    let currentUsername = currentUser.split(' ')[0]
+                    DashStats.logUserLogout(currentUsername, socket);
+                    delete timeMap[currentUsername]
+                }
+            });
+
+
+
             Utils.Emit(socket, MessageStore.Foo, 'handshooken');
 
             Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid));
@@ -130,6 +145,12 @@ export namespace WebSocket {
                 socket.disconnect(true);
             };
         });
+
+        setInterval(function() {
+            // Utils.Emit(socket, MessageStore.UpdateStats, DashStats.getUpdatedStatsBundle());
+
+            io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle())
+        }, DashStats.SAMPLING_INTERVAL);
     }
 
     function processGesturePoints(socket: Socket, content: GestureContent) {
@@ -172,11 +193,16 @@ export namespace WebSocket {
     }
 
     function barReceived(socket: SocketIO.Socket, userEmail: string) {
-        clients[userEmail] = new Client(userEmail.toString());
+         clients[userEmail] = new Client(userEmail.toString());
         const currentdate = new Date();
         const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds();
         console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`));
+        printActiveUsers();
+
+        timeMap[userEmail] = Date.now();
         socketMap.set(socket, userEmail + ' at ' + datetime);
+        userOperations.set(userEmail, 0);
+        DashStats.logUserLogin(userEmail, socket);
     }
 
     function getField([id, callback]: [string, (result?: Transferable) => void]) {
@@ -300,7 +326,8 @@ export namespace WebSocket {
         const remListItems = diff.diff.$set[updatefield].fields;
         const curList = (curListItems as any)?.fields?.[updatefield.replace('fields.', '')]?.fields.filter((f: any) => f !== null) || [];
         diff.diff.$set[updatefield].fields = curList?.filter(
-            (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : remItem.heading ? remItem.heading === curItem.heading : remItem === curItem))
+            (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId :
+                 remItem.heading ? remItem.heading === curItem.heading : remItem === curItem))
         );
         const sendBack = diff.diff.length !== diff.diff.$set[updatefield].fields.length;
         delete diff.diff.length;
@@ -346,6 +373,9 @@ export namespace WebSocket {
     var CurUser: string | undefined = undefined;
 
     function UpdateField(socket: Socket, diff: Diff) {
+        let currentUsername = socketMap.get(socket)!.split(' ')[0];
+        userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0);
+        
         if (CurUser !== socketMap.get(socket)) {
             CurUser = socketMap.get(socket);
             console.log('Switch User: ' + CurUser);