From 0f3f5fa7f63dddbfbf095f65b05f89cf27cbc6cf Mon Sep 17 00:00:00 2001 From: jameshu111 Date: Mon, 6 Mar 2023 16:14:57 -0500 Subject: /stats handler --- src/server/ApiManagers/UserManager.ts | 1 + src/server/DashStats.ts | 70 +++++++++++++++++++++++++++++++++++ src/server/index.ts | 7 ++++ src/server/stats/userLoginStats.csv | 2 + src/server/websocket.ts | 17 ++++++++- 5 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 src/server/DashStats.ts create mode 100644 src/server/stats/userLoginStats.csv (limited to 'src/server') diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 53e55c1c3..c3dadd821 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -5,6 +5,7 @@ import { msToTime } from '../ActionUtilities'; import * as bcrypt from 'bcrypt-nodejs'; import { Opt } from '../../fields/Doc'; import { WebSocket } from '../websocket'; +import { DashStats } from '../DashStats'; export const timeMap: { [id: string]: number } = {}; interface ActivityUnit { diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts new file mode 100644 index 000000000..13ee61b3a --- /dev/null +++ b/src/server/DashStats.ts @@ -0,0 +1,70 @@ +import { magenta } from 'colors'; +import { Request, Response } from 'express'; +import SocketIO from 'socket.io'; +import { Client } from './Client'; +import { WebSocket } from './websocket'; +const fs = require('fs'); +const csv = require('csv-parser'); +import { stringify } from 'csv-stringify/sync'; + +export namespace DashStats { + const statsCSVFilename = './src/server/stats/userLoginStats.csv'; + const columns = ['USERNAME', 'ACTION', 'TIME']; + + interface SocketPair { + socketId: string; + username: string; + } + + interface CSVStore { + USERNAME: string; + ACTION: string; + TIME: string; + } + + export function handleStatsView(res: Response) { + const results: CSVStore[] = []; + fs.createReadStream(statsCSVFilename) + .pipe(csv(columns)) + .on('data', (data: any) => results.push(data)) + .on('end', () => { + console.log(results); + }); + + // let newRow = stringify([{ USERNAME: 'hi', ACTION: 'hi', TIME: 'hi' }], { header: true, columns: columns }); + // console.log(newRow); + + let current = getCurrentConnections(); + res.json({ + message: 'welcome to stats', + currentConnections: current.length, + socketMap: current, + }); + } + + export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { + if (!(username === undefined)) { + let currentDate = new Date(); + console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + console.log('stringify -> '); + } + } + + export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { + if (!(username === undefined)) { + let currentDate = new Date(); + console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); + } + } + + function getCurrentConnections(): SocketPair[] { + let socketPairs: SocketPair[] = []; + for (let [key, value] of WebSocket.socketMap) { + if (!key.disconnected) { + socketPairs.push({ socketId: key.id, username: value.split(' ')[0] }); + } + } + console.log(socketPairs); + return socketPairs; + } +} diff --git a/src/server/index.ts b/src/server/index.ts index 6e6bde3cb..0848d828e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -18,6 +18,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { DashSessionAgent } from './DashSession/DashSessionAgent'; import { AppliedSessionAgent } from './DashSession/Session/agents/applied_session_agent'; +import { DashStats } from './DashStats'; import { DashUploadUtils } from './DashUploadUtils'; import { Database } from './database'; import { Logger } from './ProcessFactory'; @@ -83,6 +84,12 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ res }) => res.send(true), }); + addSupervisedRoute({ + method: Method.GET, + subscription: '/stats', + secureHandler: ({ res }) => DashStats.handleStatsView(res), + }); + addSupervisedRoute({ method: Method.GET, subscription: '/resolvedPorts', diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv new file mode 100644 index 000000000..ec33b01b2 --- /dev/null +++ b/src/server/stats/userLoginStats.csv @@ -0,0 +1,2 @@ +USERNAME,ACTION,TIME +boo15869,loggedIn,2023-01 \ No newline at end of file diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 9b91a35a6..e556ecc17 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -1,4 +1,4 @@ -import { blue } from 'colors'; +import { blue, magenta } from 'colors'; import * as express from 'express'; import { createServer, Server } from 'https'; import { networkInterfaces } from 'os'; @@ -12,6 +12,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; import YoutubeApi from './apis/youtube/youtubeApiSample'; import { initializeGuest } from './authentication/DashUserModel'; import { Client } from './Client'; +import { DashStats } from './DashStats'; import { Database } from './database'; import { DocumentsCollection } from './IDatabase'; import { Diff, GestureContent, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, Transferable, Types, UpdateMobileInkOverlayPositionContent, YoutubeQueryInput, YoutubeQueryTypes } from './Message'; @@ -20,7 +21,7 @@ import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { export let _socket: Socket; - const clients: { [key: string]: Client } = {}; + export const clients: { [key: string]: Client } = {}; export const socketMap = new Map(); export let disconnect: Function; @@ -97,6 +98,13 @@ export namespace WebSocket { console.log('received bye'); }); + socket.on('disconnect', function () { + let currentUser = socketMap.get(socket); + if (!(currentUser === undefined)) { + DashStats.logUserLogout(socketMap.get(socket), socket); + } + }); + Utils.Emit(socket, MessageStore.Foo, 'handshooken'); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -176,7 +184,12 @@ export namespace WebSocket { const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); + console.log(magenta(`currently connected: ${JSON.stringify(clients)}`)); + // console.log(magenta('socket map below')); + // console.log([...socketMap.entries()]); + printActiveUsers(); socketMap.set(socket, userEmail + ' at ' + datetime); + DashStats.logUserLogin(socketMap.get(socket), socket); } function getField([id, callback]: [string, (result?: Transferable) => void]) { -- cgit v1.2.3-70-g09d2 From d51a57e1d0823fc08a90b73d427ab5e35b72422f Mon Sep 17 00:00:00 2001 From: Mohammad Amoush <51237606+jameshu111@users.noreply.github.com> Date: Sat, 18 Mar 2023 16:56:13 -0400 Subject: Add operations counter --- src/server/Client.ts | 13 +++- src/server/DashStats.ts | 119 ++++++++++++++++++++++++++++++------ src/server/index.ts | 6 ++ src/server/stats/userLoginStats.csv | 77 ++++++++++++++++++++++- src/server/websocket.ts | 24 ++++++-- views/stats.pug | 22 +++++++ views/stylesheets/statsview.css | 56 +++++++++++++++++ 7 files changed, 290 insertions(+), 27 deletions(-) create mode 100644 views/stats.pug create mode 100644 views/stylesheets/statsview.css (limited to 'src/server') diff --git a/src/server/Client.ts b/src/server/Client.ts index e6f953712..be1ffc2ba 100644 --- a/src/server/Client.ts +++ b/src/server/Client.ts @@ -1,11 +1,22 @@ -import { computed } from "mobx"; +import { action, computed } from "mobx"; export class Client { private _guid: string; + // private operations: number; constructor(guid: string) { this._guid = guid; + // this.operations = 0; } + // @computed public get OPERATIONS(): number { + // return this.operations; + // } + + // @action + // public setOperations(newOperations: number): void { + // this.operations = newOperations; + // } + @computed public get GUID(): string { return this._guid; } } \ No newline at end of file diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts index 13ee61b3a..a10b28608 100644 --- a/src/server/DashStats.ts +++ b/src/server/DashStats.ts @@ -1,11 +1,11 @@ -import { magenta } from 'colors'; +import { cyan, magenta } from 'colors'; import { Request, Response } from 'express'; +import { Server } from 'http'; import SocketIO from 'socket.io'; -import { Client } from './Client'; +import { timeMap } from './ApiManagers/UserManager'; import { WebSocket } from './websocket'; const fs = require('fs'); const csv = require('csv-parser'); -import { stringify } from 'csv-stringify/sync'; export namespace DashStats { const statsCSVFilename = './src/server/stats/userLoginStats.csv'; @@ -14,6 +14,8 @@ export namespace DashStats { interface SocketPair { socketId: string; username: string; + time: string; + operations: number; } interface CSVStore { @@ -22,49 +24,128 @@ export namespace DashStats { TIME: string; } - export function handleStatsView(res: Response) { + enum ServerTraffic { + NOT_BUSY, + BUSY, + VERY_BUSY + } + + const BUSY_SERVER_BOUND = 2; + const VERY_BUSY_SERVER_BOUND = 3; + + const serverTrafficMessages = [ + "Not Busy", + "Busy", + "Very Busy" + ] + + export function handleStats(res: Response) { + let current = getCurrentConnections(); const results: CSVStore[] = []; - fs.createReadStream(statsCSVFilename) - .pipe(csv(columns)) - .on('data', (data: any) => results.push(data)) - .on('end', () => { - console.log(results); + res.json({ + message: 'welcome to stats', + currentConnections: current.length, + socketMap: current, }); - // let newRow = stringify([{ USERNAME: 'hi', ACTION: 'hi', TIME: 'hi' }], { header: true, columns: columns }); - // console.log(newRow); + // fs.createReadStream(statsCSVFilename) + // .pipe(csv()) + // .on('data', (data: any) => results.push(data)) + // .on('end', () => { + // console.log(results); + // res.json({ + // message: 'welcome to stats', + // currentConnections: current.length, + // socketMap: current, + // results: results, + // }); + // }); + } + export function handleStatsView(res: Response) { let current = getCurrentConnections(); - res.json({ - message: 'welcome to stats', - currentConnections: current.length, - socketMap: current, + + let connectedUsers = current.map((socketPair) => { + return socketPair.time + " - " + socketPair.username + " Operations: " + socketPair.operations; + }) + + let serverTraffic = ServerTraffic.NOT_BUSY; + if(current.length < BUSY_SERVER_BOUND) { + serverTraffic = ServerTraffic.NOT_BUSY; + } else if(current.length >= BUSY_SERVER_BOUND && current.length < VERY_BUSY_SERVER_BOUND) { + serverTraffic = ServerTraffic.BUSY; + } else { + serverTraffic = ServerTraffic.VERY_BUSY; + } + + res.render("stats.pug", { + title: "Dash Stats", + numConnections: current.length, + serverTraffic: serverTraffic, + serverTrafficMessage : serverTrafficMessages[serverTraffic], + connectedUsers: connectedUsers }); } export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); - console.log('stringify -> '); + // console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + + let toWrite: CSVStore = { + USERNAME : username, + ACTION : "loggedIn", + TIME : currentDate.toISOString() + } + + let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"}); + statsFile.write(convertToCSV(toWrite)); + statsFile.end(); + console.log(cyan(convertToCSV(toWrite))); } } export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); + // console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); + + let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"}); + let toWrite: CSVStore = { + USERNAME : username, + ACTION : "loggedOut", + TIME : currentDate.toISOString() + } + statsFile.write(convertToCSV(toWrite)); + statsFile.end(); } } function getCurrentConnections(): SocketPair[] { + console.log("timeMap: " + timeMap); + console.log("clients:" + WebSocket.clients); let socketPairs: SocketPair[] = []; for (let [key, value] of WebSocket.socketMap) { + let username = value.split(' ')[0]; + let connectionTime = new Date(timeMap[username]); + + let connectionTimeString = connectionTime.toLocaleDateString() + " " + connectionTime.toLocaleTimeString(); + if (!key.disconnected) { - socketPairs.push({ socketId: key.id, username: value.split(' ')[0] }); + socketPairs.push({ + socketId: key.id, + username: username, + time: connectionTimeString.includes("Invalid Date") ? "" : connectionTimeString, + operations : WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0, + }); } } console.log(socketPairs); + // console.log([...WebSocket.clients.entries()]); return socketPairs; } + + function convertToCSV(dataObject: CSVStore): string { + return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`; + } } diff --git a/src/server/index.ts b/src/server/index.ts index 0848d828e..d76f12b95 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -87,6 +87,12 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: addSupervisedRoute({ method: Method.GET, subscription: '/stats', + secureHandler: ({ res }) => DashStats.handleStats(res), + }); + + addSupervisedRoute({ + method: Method.GET, + subscription: '/statsview', secureHandler: ({ res }) => DashStats.handleStatsView(res), }); diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv index ec33b01b2..d5e56502c 100644 --- a/src/server/stats/userLoginStats.csv +++ b/src/server/stats/userLoginStats.csv @@ -1,2 +1,75 @@ -USERNAME,ACTION,TIME -boo15869,loggedIn,2023-01 \ No newline at end of file +USER,ACTION,TIME +aaa@gmail.com,loggedIn,2023-03-18T20:10:29.928Z +guest,loggedIn,2023-03-18T20:10:47.384Z +aaa@gmail.com,loggedOut,2023-03-18T20:10:55.364Z +aaa@gmail.com,loggedIn,2023-03-18T20:11:15.879Z +guest,loggedIn,2023-03-18T20:18:03.724Z +aaa@gmail.com,loggedIn,2023-03-18T20:18:03.860Z +guest,loggedOut,2023-03-18T20:19:00.211Z +guest,loggedIn,2023-03-18T20:19:03.087Z +guest,loggedOut,2023-03-18T20:19:50.668Z +boo15869@gmail.com,loggedIn,2023-03-18T20:20:17.890Z +boo15869@gmail.com,loggedOut,2023-03-18T20:21:03.542Z +boo15869@gmail.com,loggedIn,2023-03-18T20:21:06.149Z +boo15869@gmail.com,loggedOut,2023-03-18T20:21:51.874Z +a@gmail.com,loggedIn,2023-03-18T20:22:02.122Z +aaa@gmail.com,loggedOut,2023-03-18T20:22:42.882Z +aaa@gmail.com,loggedIn,2023-03-18T20:22:45.631Z +aaa@gmail.com,loggedIn,2023-03-18T20:25:34.658Z +a@gmail.com,loggedIn,2023-03-18T20:25:34.681Z +aaa@gmail.com,loggedIn,2023-03-18T20:29:04.297Z +a@gmail.com,loggedIn,2023-03-18T20:29:08.701Z +a@gmail.com,loggedIn,2023-03-18T20:29:18.565Z +aaa@gmail.com,loggedIn,2023-03-18T20:29:21.974Z +aaa@gmail.com,loggedIn,2023-03-18T20:29:51.477Z +a@gmail.com,loggedIn,2023-03-18T20:29:51.489Z +aaa@gmail.com,loggedIn,2023-03-18T20:30:15.011Z +aaa@gmail.com,loggedIn,2023-03-18T20:30:57.818Z +a@gmail.com,loggedIn,2023-03-18T20:30:57.838Z +aaa@gmail.com,loggedIn,2023-03-18T20:31:12.061Z +a@gmail.com,loggedIn,2023-03-18T20:31:12.080Z +a@gmail.com,loggedIn,2023-03-18T20:31:19.447Z +aaa@gmail.com,loggedIn,2023-03-18T20:31:22.738Z +aaa@gmail.com,loggedIn,2023-03-18T20:32:36.919Z +a@gmail.com,loggedIn,2023-03-18T20:32:36.929Z +a@gmail.com,loggedIn,2023-03-18T20:32:56.212Z +aaa@gmail.com,loggedIn,2023-03-18T20:32:59.300Z +aaa@gmail.com,loggedIn,2023-03-18T20:34:27.543Z +a@gmail.com,loggedIn,2023-03-18T20:34:27.570Z +a@gmail.com,loggedIn,2023-03-18T20:34:35.299Z +aaa@gmail.com,loggedIn,2023-03-18T20:34:35.302Z +a@gmail.com,loggedIn,2023-03-18T20:34:51.579Z +aaa@gmail.com,loggedIn,2023-03-18T20:34:52.392Z +a@gmail.com,loggedIn,2023-03-18T20:35:08.509Z +aaa@gmail.com,loggedIn,2023-03-18T20:35:15.202Z +a@gmail.com,loggedIn,2023-03-18T20:36:46.796Z +aaa@gmail.com,loggedIn,2023-03-18T20:36:51.756Z +a@gmail.com,loggedIn,2023-03-18T20:36:55.286Z +a@gmail.com,loggedIn,2023-03-18T20:40:06.226Z +a@gmail.com,loggedIn,2023-03-18T20:40:18.474Z +aaa@gmail.com,loggedIn,2023-03-18T20:40:31.894Z +a@gmail.com,loggedIn,2023-03-18T20:40:31.903Z +a@gmail.com,loggedIn,2023-03-18T20:42:25.301Z +aaa@gmail.com,loggedIn,2023-03-18T20:42:31.182Z +aaa@gmail.com,loggedOut,2023-03-18T20:43:05.741Z +aaa@gmail.com,loggedIn,2023-03-18T20:43:09.203Z +a@gmail.com,loggedOut,2023-03-18T20:45:24.343Z +a@gmail.com,loggedIn,2023-03-18T20:45:27.207Z +aaa@gmail.com,loggedIn,2023-03-18T20:46:15.618Z +a@gmail.com,loggedIn,2023-03-18T20:46:56.163Z +a@gmail.com,loggedOut,2023-03-18T20:48:37.464Z +a@gmail.com,loggedIn,2023-03-18T20:48:40.562Z +a@gmail.com,loggedOut,2023-03-18T20:50:32.478Z +a@gmail.com,loggedIn,2023-03-18T20:50:40.384Z +a@gmail.com,loggedOut,2023-03-18T20:51:34.159Z +a@gmail.com,loggedIn,2023-03-18T20:51:49.206Z +a@gmail.com,loggedOut,2023-03-18T20:52:04.673Z +qw@gmail.com,loggedIn,2023-03-18T20:52:36.270Z +qw@gmail.com,loggedIn,2023-03-18T20:53:58.175Z +aaa@gmail.com,loggedIn,2023-03-18T20:53:58.204Z +aaa@gmail.com,loggedIn,2023-03-18T20:54:20.518Z +qw@gmail.com,loggedIn,2023-03-18T20:54:24.225Z +qw@gmail.com,loggedIn,2023-03-18T20:54:37.007Z +aaa@gmail.com,loggedIn,2023-03-18T20:54:39.959Z +aaa@gmail.com,loggedOut,2023-03-18T20:55:55.199Z +qw@gmail.com,loggedOut,2023-03-18T20:55:59.010Z diff --git a/src/server/websocket.ts b/src/server/websocket.ts index e556ecc17..54944e944 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -1,4 +1,4 @@ -import { blue, magenta } from 'colors'; +import { blue, magenta, yellow } from 'colors'; import * as express from 'express'; import { createServer, Server } from 'https'; import { networkInterfaces } from 'os'; @@ -22,7 +22,9 @@ import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { export let _socket: Socket; export const clients: { [key: string]: Client } = {}; + // export const clients = new Map(); export const socketMap = new Map(); + export const userOperations = new Map(); export let disconnect: Function; export async function initialize(isRelease: boolean, app: express.Express) { @@ -101,7 +103,9 @@ export namespace WebSocket { socket.on('disconnect', function () { let currentUser = socketMap.get(socket); if (!(currentUser === undefined)) { - DashStats.logUserLogout(socketMap.get(socket), socket); + let currentUsername = currentUser.split(' ')[0] + DashStats.logUserLogout(currentUsername, socket); + delete timeMap[currentUsername] } }); @@ -180,16 +184,20 @@ export namespace WebSocket { } function barReceived(socket: SocketIO.Socket, userEmail: string) { - clients[userEmail] = new Client(userEmail.toString()); + clients[userEmail] = new Client(userEmail.toString()); + // clients.set(userEmail, new Client(userEmail.toString())); const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); - console.log(magenta(`currently connected: ${JSON.stringify(clients)}`)); + // console.log(magenta(`currently connected: ${[...clients.entries()]}`)); // console.log(magenta('socket map below')); // console.log([...socketMap.entries()]); printActiveUsers(); + + timeMap[userEmail] = Date.now(); socketMap.set(socket, userEmail + ' at ' + datetime); - DashStats.logUserLogin(socketMap.get(socket), socket); + userOperations.set(userEmail, 0); + DashStats.logUserLogin(userEmail, socket); } function getField([id, callback]: [string, (result?: Transferable) => void]) { @@ -359,6 +367,12 @@ export namespace WebSocket { var CurUser: string | undefined = undefined; function UpdateField(socket: Socket, diff: Diff) { + console.log(magenta(`1 OP ${socketMap.get(socket)}`)); + + let currentUsername = socketMap.get(socket)!.split(' ')[0]; + userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0); + console.log(yellow("Total Operations: " + userOperations.get(currentUsername))); + if (CurUser !== socketMap.get(socket)) { CurUser = socketMap.get(socket); console.log('Switch User: ' + CurUser); diff --git a/views/stats.pug b/views/stats.pug new file mode 100644 index 000000000..54c017e70 --- /dev/null +++ b/views/stats.pug @@ -0,0 +1,22 @@ +extends ./layout + +block content + style + include ./stylesheets/authentication.css + include ./stylesheets/statsview.css + .outermost + .stats-container + h1 Dash Stats + p(class="stats-content") Current Connections: #{numConnections} + div(class="stats-content stats-server-status-container") + p Server Status: + div(class="stats-server-status-item stats-server-status-" + serverTraffic) + p #{serverTrafficMessage} + div(class="stats-content stats-connected-users") + p Connected Users: + ul + each username in connectedUsers + li(class="none")= username + + + \ No newline at end of file diff --git a/views/stylesheets/statsview.css b/views/stylesheets/statsview.css new file mode 100644 index 000000000..c018bedfc --- /dev/null +++ b/views/stylesheets/statsview.css @@ -0,0 +1,56 @@ +.outermost { + background-color: #251f1f; + display: flex; + flex-direction: row; + height: 98vh; + width: 99vw; + justify-content: center; + position: relative; +} + +.stats-container { + background-color: white; + + padding: 1rem; + width: 80vw; + border-radius: 8px; +} + +.stats-content { + font-size: 1.25em; + +} + +.stats-server-status-container { + display: flex; + flex-direction: row; +} + +.stats-server-status-item { + margin-left: 0.25rem; + padding: 0px 5px; + + border-radius: 3px; + width: 8rem; + text-align: center; +} + +.stats-server-status-0 { + /* not busy */ + border: 3px green solid; +} + +.stats-server-status-1 { + /* busy */ + border: 3px #ffcc00 solid; +} + +.stats-server-status-2 { + /* very busy */ + border: 3px red solid; +} + +.stats-connected-users { + max-height: 70vh; + overflow-y: auto; +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 0a38e3f91f4f85f07fdbb7575ceb678032dcdfe9 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush <51237606+jameshu111@users.noreply.github.com> Date: Sat, 8 Apr 2023 16:57:02 -0400 Subject: Clean code and Comments --- src/server/Client.ts | 13 +-- src/server/DashStats.ts | 198 ++++++++++++++++++++++++++++----- src/server/Message.ts | 2 + src/server/stats/userLoginStats.csv | 82 ++------------ src/server/websocket.ts | 23 ++-- views/resources/statsviewcontroller.js | 114 +++++++++++++++++++ views/stats.pug | 18 ++- 7 files changed, 320 insertions(+), 130 deletions(-) create mode 100644 views/resources/statsviewcontroller.js (limited to 'src/server') diff --git a/src/server/Client.ts b/src/server/Client.ts index be1ffc2ba..e6f953712 100644 --- a/src/server/Client.ts +++ b/src/server/Client.ts @@ -1,22 +1,11 @@ -import { action, computed } from "mobx"; +import { computed } from "mobx"; export class Client { private _guid: string; - // private operations: number; constructor(guid: string) { this._guid = guid; - // this.operations = 0; } - // @computed public get OPERATIONS(): number { - // return this.operations; - // } - - // @action - // public setOperations(newOperations: number): void { - // this.operations = newOperations; - // } - @computed public get GUID(): string { return this._guid; } } \ No newline at end of file diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts index a10b28608..8d341db63 100644 --- a/src/server/DashStats.ts +++ b/src/server/DashStats.ts @@ -1,35 +1,72 @@ import { cyan, magenta } from 'colors'; -import { Request, Response } from 'express'; -import { Server } from 'http'; +import { Response } from 'express'; import SocketIO from 'socket.io'; import { timeMap } from './ApiManagers/UserManager'; import { WebSocket } from './websocket'; const fs = require('fs'); -const csv = require('csv-parser'); +/** + * DashStats focuses on tracking user data for each session. + * + * This includes time connected, number of operations, and + * the rate of their operations + */ export namespace DashStats { + export const SAMPLING_INTERVAL = 1000; // in milliseconds (ms) - Time interval to update the frontend. + export const RATE_INTERVAL = 10; // in seconds (s) - Used to calculate rate + const statsCSVFilename = './src/server/stats/userLoginStats.csv'; const columns = ['USERNAME', 'ACTION', 'TIME']; - interface SocketPair { + /** + * UserStats holds the stats associated with a particular user. + */ + interface UserStats { socketId: string; username: string; time: string; operations: number; + rate: number; + } + + /** + * UserLastOperations is the queue object for each user + * storing their past operations. + */ + interface UserLastOperations { + sampleOperations: number; // stores how many operations total are in this rate section (10 sec, for example) + lastSampleOperations: number; // stores how many total operations were recorded at the last sample + previousOperationsQueue: number[]; // stores the operations to calculate rate. } + /** + * StatsDataBundle represents an object that will be sent to the frontend view + * on each websocket update. + */ + interface StatsDataBundle { + connectedUsers: UserStats[]; + } + + /** + * CSVStore represents how objects will be stored in the CSV + */ interface CSVStore { USERNAME: string; ACTION: string; TIME: string; } + /** + * ServerTraffic describes the current traffic going to the backend. + */ enum ServerTraffic { NOT_BUSY, BUSY, VERY_BUSY } + // These values can be changed after further testing how many + // users correspond to each traffic level in Dash. const BUSY_SERVER_BOUND = 2; const VERY_BUSY_SERVER_BOUND = 3; @@ -39,31 +76,47 @@ export namespace DashStats { "Very Busy" ] + // lastUserOperations maps each username to a UserLastOperations + // structure + export const lastUserOperations = new Map(); + + /** + * handleStats is called when the /stats route is called, providing a JSON + * object with relevant stats. In this case, we return the number of + * current connections and + * @param res Response object from Express + */ export function handleStats(res: Response) { - let current = getCurrentConnections(); + let current = getCurrentStats(); const results: CSVStore[] = []; res.json({ - message: 'welcome to stats', currentConnections: current.length, socketMap: current, }); + } + + /** + * getUpdatedStatesBundle() sends an updated copy of the current stats to the + * frontend /statsview route via websockets. + * + * @returns a StatsDataBundle that is sent to the frontend view on each websocket update + */ + export function getUpdatedStatsBundle(): StatsDataBundle { + let current = getCurrentStats(); - // fs.createReadStream(statsCSVFilename) - // .pipe(csv()) - // .on('data', (data: any) => results.push(data)) - // .on('end', () => { - // console.log(results); - // res.json({ - // message: 'welcome to stats', - // currentConnections: current.length, - // socketMap: current, - // results: results, - // }); - // }); + return { + connectedUsers: current, + } } + /** + * handleStatsView() is called when the /statsview route is called. This + * will use pug to render a frontend view of the current stats + * + * @param res + */ export function handleStatsView(res: Response) { - let current = getCurrentConnections(); + let current = getCurrentStats(); let connectedUsers = current.map((socketPair) => { return socketPair.time + " - " + socketPair.username + " Operations: " + socketPair.operations; @@ -80,17 +133,23 @@ export namespace DashStats { res.render("stats.pug", { title: "Dash Stats", - numConnections: current.length, + numConnections: connectedUsers.length, serverTraffic: serverTraffic, serverTrafficMessage : serverTrafficMessages[serverTraffic], connectedUsers: connectedUsers }); } + /** + * logUserLogin() writes a login event to the CSV file. + * + * @param username the username in the format of "username@domain.com logged in" + * @param socket the websocket associated with the current connection + */ export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - // console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); let toWrite: CSVStore = { USERNAME : username, @@ -105,10 +164,15 @@ export namespace DashStats { } } + /** + * logUserLogout() writes a logout event to the CSV file. + * + * @param username the username in the format of "username@domain.com logged in" + * @param socket the websocket associated with the current connection. + */ export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - // console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"}); let toWrite: CSVStore = { @@ -121,10 +185,79 @@ export namespace DashStats { } } - function getCurrentConnections(): SocketPair[] { - console.log("timeMap: " + timeMap); - console.log("clients:" + WebSocket.clients); - let socketPairs: SocketPair[] = []; + /** + * getLastOperationsOrDefault() is a helper method that will attempt + * to query the lastUserOperations map for a specified username. If the + * username is not in the map, an empty UserLastOperations object is returned. + * @param username + * @returns the user's UserLastOperations structure or an empty + * UserLastOperations object (All values set to 0) if the username is not found. + */ + function getLastOperationsOrDefault(username: string): UserLastOperations { + if(lastUserOperations.get(username) === undefined) { + let initializeOperationsQueue = []; + for(let i = 0; i < RATE_INTERVAL; i++) { + initializeOperationsQueue.push(0); + } + return { + sampleOperations: 0, + lastSampleOperations: 0, + previousOperationsQueue: initializeOperationsQueue + } + } + return lastUserOperations.get(username)!; + } + + /** + * updateLastOperations updates a specific user's UserLastOperations information + * for the current sampling cycle. The method removes old/outdated counts for + * operations from the queue and adds new data for the current sampling + * cycle to the queue, updating the total count as it goes. + * @param lastOperationData the old UserLastOperations data that must be updated + * @param currentOperations the total number of operations measured for this sampling cycle. + * @returns the udpated UserLastOperations structure. + */ + function updateLastOperations(lastOperationData: UserLastOperations, currentOperations: number): UserLastOperations { + // create a copy of the UserLastOperations to modify + let newLastOperationData: UserLastOperations = { + sampleOperations: lastOperationData.sampleOperations, + lastSampleOperations: lastOperationData.lastSampleOperations, + previousOperationsQueue: lastOperationData.previousOperationsQueue.slice() + } + + let newSampleOperations = newLastOperationData.sampleOperations; + newSampleOperations -= newLastOperationData.previousOperationsQueue.shift()!; // removes and returns the first element of the queue + let operationsThisCycle = currentOperations - lastOperationData.lastSampleOperations; + newSampleOperations += operationsThisCycle; // add the operations this cycle to find out what our count for the interval should be (e.g operations in the last 10 seconds) + + // update values for the copy object + newLastOperationData.sampleOperations = newSampleOperations; + + newLastOperationData.previousOperationsQueue.push(operationsThisCycle); + newLastOperationData.lastSampleOperations = currentOperations; + + return newLastOperationData; + } + + /** + * getUserOperationsOrDefault() is a helper method to get the user's total + * operations for the CURRENT sampling interval. The method will return 0 + * if the username is not in the userOperations map. + * @param username the username to search the map for + * @returns the total number of operations recorded up to this sampling cycle. + */ + function getUserOperationsOrDefault(username: string): number { + return WebSocket.userOperations.get(username) === undefined ? 0 : WebSocket.userOperations.get(username)! + } + + /** + * getCurrentStats() calculates the total stats for this cycle. In this case, + * getCurrentStats() returns an Array of UserStats[] objects describing + * the stats for each user + * @returns an array of UserStats storing data for each user at the current moment. + */ + function getCurrentStats(): UserStats[] { + let socketPairs: UserStats[] = []; for (let [key, value] of WebSocket.socketMap) { let username = value.split(' ')[0]; let connectionTime = new Date(timeMap[username]); @@ -132,19 +265,28 @@ export namespace DashStats { let connectionTimeString = connectionTime.toLocaleDateString() + " " + connectionTime.toLocaleTimeString(); if (!key.disconnected) { + let lastRecordedOperations = getLastOperationsOrDefault(username); + let currentUserOperationCount = getUserOperationsOrDefault(username); + socketPairs.push({ socketId: key.id, username: username, time: connectionTimeString.includes("Invalid Date") ? "" : connectionTimeString, operations : WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0, + rate: lastRecordedOperations.sampleOperations }); + lastUserOperations.set(username, updateLastOperations(lastRecordedOperations,currentUserOperationCount)); } } - console.log(socketPairs); - // console.log([...WebSocket.clients.entries()]); return socketPairs; } + /** + * convertToCSV() is a helper method that stringifies a CSVStore object + * that can be written to the CSV file later. + * @param dataObject the object to stringify + * @returns the object as a string. + */ function convertToCSV(dataObject: CSVStore): string { return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`; } diff --git a/src/server/Message.ts b/src/server/Message.ts index d87ae5027..8f0af08bc 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -94,4 +94,6 @@ export namespace MessageStore { export const YoutubeApiQuery = new Message("Youtube Api Query"); export const DeleteField = new Message("Delete field"); export const DeleteFields = new Message("Delete fields"); + + export const UpdateStats = new Message("updatestats"); } diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv index d5e56502c..23bcef885 100644 --- a/src/server/stats/userLoginStats.csv +++ b/src/server/stats/userLoginStats.csv @@ -1,75 +1,7 @@ -USER,ACTION,TIME -aaa@gmail.com,loggedIn,2023-03-18T20:10:29.928Z -guest,loggedIn,2023-03-18T20:10:47.384Z -aaa@gmail.com,loggedOut,2023-03-18T20:10:55.364Z -aaa@gmail.com,loggedIn,2023-03-18T20:11:15.879Z -guest,loggedIn,2023-03-18T20:18:03.724Z -aaa@gmail.com,loggedIn,2023-03-18T20:18:03.860Z -guest,loggedOut,2023-03-18T20:19:00.211Z -guest,loggedIn,2023-03-18T20:19:03.087Z -guest,loggedOut,2023-03-18T20:19:50.668Z -boo15869@gmail.com,loggedIn,2023-03-18T20:20:17.890Z -boo15869@gmail.com,loggedOut,2023-03-18T20:21:03.542Z -boo15869@gmail.com,loggedIn,2023-03-18T20:21:06.149Z -boo15869@gmail.com,loggedOut,2023-03-18T20:21:51.874Z -a@gmail.com,loggedIn,2023-03-18T20:22:02.122Z -aaa@gmail.com,loggedOut,2023-03-18T20:22:42.882Z -aaa@gmail.com,loggedIn,2023-03-18T20:22:45.631Z -aaa@gmail.com,loggedIn,2023-03-18T20:25:34.658Z -a@gmail.com,loggedIn,2023-03-18T20:25:34.681Z -aaa@gmail.com,loggedIn,2023-03-18T20:29:04.297Z -a@gmail.com,loggedIn,2023-03-18T20:29:08.701Z -a@gmail.com,loggedIn,2023-03-18T20:29:18.565Z -aaa@gmail.com,loggedIn,2023-03-18T20:29:21.974Z -aaa@gmail.com,loggedIn,2023-03-18T20:29:51.477Z -a@gmail.com,loggedIn,2023-03-18T20:29:51.489Z -aaa@gmail.com,loggedIn,2023-03-18T20:30:15.011Z -aaa@gmail.com,loggedIn,2023-03-18T20:30:57.818Z -a@gmail.com,loggedIn,2023-03-18T20:30:57.838Z -aaa@gmail.com,loggedIn,2023-03-18T20:31:12.061Z -a@gmail.com,loggedIn,2023-03-18T20:31:12.080Z -a@gmail.com,loggedIn,2023-03-18T20:31:19.447Z -aaa@gmail.com,loggedIn,2023-03-18T20:31:22.738Z -aaa@gmail.com,loggedIn,2023-03-18T20:32:36.919Z -a@gmail.com,loggedIn,2023-03-18T20:32:36.929Z -a@gmail.com,loggedIn,2023-03-18T20:32:56.212Z -aaa@gmail.com,loggedIn,2023-03-18T20:32:59.300Z -aaa@gmail.com,loggedIn,2023-03-18T20:34:27.543Z -a@gmail.com,loggedIn,2023-03-18T20:34:27.570Z -a@gmail.com,loggedIn,2023-03-18T20:34:35.299Z -aaa@gmail.com,loggedIn,2023-03-18T20:34:35.302Z -a@gmail.com,loggedIn,2023-03-18T20:34:51.579Z -aaa@gmail.com,loggedIn,2023-03-18T20:34:52.392Z -a@gmail.com,loggedIn,2023-03-18T20:35:08.509Z -aaa@gmail.com,loggedIn,2023-03-18T20:35:15.202Z -a@gmail.com,loggedIn,2023-03-18T20:36:46.796Z -aaa@gmail.com,loggedIn,2023-03-18T20:36:51.756Z -a@gmail.com,loggedIn,2023-03-18T20:36:55.286Z -a@gmail.com,loggedIn,2023-03-18T20:40:06.226Z -a@gmail.com,loggedIn,2023-03-18T20:40:18.474Z -aaa@gmail.com,loggedIn,2023-03-18T20:40:31.894Z -a@gmail.com,loggedIn,2023-03-18T20:40:31.903Z -a@gmail.com,loggedIn,2023-03-18T20:42:25.301Z -aaa@gmail.com,loggedIn,2023-03-18T20:42:31.182Z -aaa@gmail.com,loggedOut,2023-03-18T20:43:05.741Z -aaa@gmail.com,loggedIn,2023-03-18T20:43:09.203Z -a@gmail.com,loggedOut,2023-03-18T20:45:24.343Z -a@gmail.com,loggedIn,2023-03-18T20:45:27.207Z -aaa@gmail.com,loggedIn,2023-03-18T20:46:15.618Z -a@gmail.com,loggedIn,2023-03-18T20:46:56.163Z -a@gmail.com,loggedOut,2023-03-18T20:48:37.464Z -a@gmail.com,loggedIn,2023-03-18T20:48:40.562Z -a@gmail.com,loggedOut,2023-03-18T20:50:32.478Z -a@gmail.com,loggedIn,2023-03-18T20:50:40.384Z -a@gmail.com,loggedOut,2023-03-18T20:51:34.159Z -a@gmail.com,loggedIn,2023-03-18T20:51:49.206Z -a@gmail.com,loggedOut,2023-03-18T20:52:04.673Z -qw@gmail.com,loggedIn,2023-03-18T20:52:36.270Z -qw@gmail.com,loggedIn,2023-03-18T20:53:58.175Z -aaa@gmail.com,loggedIn,2023-03-18T20:53:58.204Z -aaa@gmail.com,loggedIn,2023-03-18T20:54:20.518Z -qw@gmail.com,loggedIn,2023-03-18T20:54:24.225Z -qw@gmail.com,loggedIn,2023-03-18T20:54:37.007Z -aaa@gmail.com,loggedIn,2023-03-18T20:54:39.959Z -aaa@gmail.com,loggedOut,2023-03-18T20:55:55.199Z -qw@gmail.com,loggedOut,2023-03-18T20:55:59.010Z +USER,ACTION,TIMEaaa@gmail.com,loggedIn,2023-04-08T20:08:17.533Z +aaa@gmail.com,loggedIn,2023-04-08T20:14:32.460Z +aaa@gmail.com,loggedIn,2023-04-08T20:14:44.884Z +aaa@gmail.com,loggedIn,2023-04-08T20:14:56.854Z +aaa@gmail.com,loggedIn,2023-04-08T20:16:59.747Z +aaa@gmail.com,loggedIn,2023-04-08T20:56:50.759Z +aaa@gmail.com,loggedIn,2023-04-08T20:56:58.175Z diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 54944e944..5d1390d37 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -1,4 +1,4 @@ -import { blue, magenta, yellow } from 'colors'; +import { blue } from 'colors'; import * as express from 'express'; import { createServer, Server } from 'https'; import { networkInterfaces } from 'os'; @@ -22,7 +22,6 @@ import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { export let _socket: Socket; export const clients: { [key: string]: Client } = {}; - // export const clients = new Map(); export const socketMap = new Map(); export const userOperations = new Map(); export let disconnect: Function; @@ -52,6 +51,8 @@ export namespace WebSocket { next(); }); + socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + // convenience function to log server messages on the client function log(message?: any, ...optionalParams: any[]) { socket.emit('log', ['Message from server:', message, ...optionalParams]); @@ -109,6 +110,8 @@ export namespace WebSocket { } }); + + Utils.Emit(socket, MessageStore.Foo, 'handshooken'); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -142,6 +145,12 @@ export namespace WebSocket { socket.disconnect(true); }; }); + + setInterval(function() { + // Utils.Emit(socket, MessageStore.UpdateStats, DashStats.getUpdatedStatsBundle()); + + io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + }, DashStats.SAMPLING_INTERVAL); } function processGesturePoints(socket: Socket, content: GestureContent) { @@ -185,13 +194,9 @@ export namespace WebSocket { function barReceived(socket: SocketIO.Socket, userEmail: string) { clients[userEmail] = new Client(userEmail.toString()); - // clients.set(userEmail, new Client(userEmail.toString())); const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); - // console.log(magenta(`currently connected: ${[...clients.entries()]}`)); - // console.log(magenta('socket map below')); - // console.log([...socketMap.entries()]); printActiveUsers(); timeMap[userEmail] = Date.now(); @@ -321,7 +326,8 @@ export namespace WebSocket { const remListItems = diff.diff.$set[updatefield].fields; const curList = (curListItems as any)?.fields?.[updatefield.replace('fields.', '')]?.fields.filter((f: any) => f !== null) || []; diff.diff.$set[updatefield].fields = curList?.filter( - (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) + (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : + remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) ); const sendBack = diff.diff.length !== diff.diff.$set[updatefield].fields.length; delete diff.diff.length; @@ -367,11 +373,8 @@ export namespace WebSocket { var CurUser: string | undefined = undefined; function UpdateField(socket: Socket, diff: Diff) { - console.log(magenta(`1 OP ${socketMap.get(socket)}`)); - let currentUsername = socketMap.get(socket)!.split(' ')[0]; userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0); - console.log(yellow("Total Operations: " + userOperations.get(currentUsername))); if (CurUser !== socketMap.get(socket)) { CurUser = socketMap.get(socket); diff --git a/views/resources/statsviewcontroller.js b/views/resources/statsviewcontroller.js new file mode 100644 index 000000000..090e112e7 --- /dev/null +++ b/views/resources/statsviewcontroller.js @@ -0,0 +1,114 @@ +/** + * statsviewcontroller.js stores the JavaScript functions to update the stats page + * when the websocket updates. + */ + +const BUSY_SERVER_BOUND = 2; +const VERY_BUSY_SERVER_BOUND = 3; + +const MEDIUM_USE_BOUND = 100; //operations per 10 seconds +const HIGH_USE_BOUND = 300; + +const serverTrafficMessages = { + 0 : "Not Busy", + 1 : "Busy", + 2: "Very Busy" +}; + +/** + * userDataComparator sorts the users based on the rate + * + * @param {*} user1 the first user to compare + * @param {*} user2 the second user to comapre + * @returns an integer indiciating which user should come first + */ +function userDataComparator(user1, user2) { + if(user1.rate < user2.rate) { + return 1; + } else if(user1.rate > user2.rate) { + return -1; + } else { + return 0; + } +} + +/** + * calculateServerTraffic() returns an integer corresponding + * to the current traffic that can be used to get the message + * from "serverTrafficMessages" + * + * @param {*} data the incoming data from the backend + * @returns an integer where 0 is not busy, 1 is busy, and 2 is very busy. + */ +function calculateServerTraffic(data) { + let currentTraffic = data.connectedUsers.length; + + let serverTraffic = 0; + if(currentTraffic < BUSY_SERVER_BOUND) { + serverTraffic = 0; + } else if(currentTraffic >= BUSY_SERVER_BOUND && currentTraffic < VERY_BUSY_SERVER_BOUND) { + serverTraffic = 1; + } else { + serverTraffic = 2; + } + + return serverTraffic; +} + +/** + * getUserRateColor determines what color the user's rate should + * be on the front end + * @param {*} rate the operations per time interval for a specific user + * @returns a string representing the color to make the user rate + */ +function getUserRateColor(rate) { + if(rate < MEDIUM_USE_BOUND) { + return "black"; + } else if(rate >= MEDIUM_USE_BOUND && rate < HIGH_USE_BOUND) { + return "orange"; + } else if(rate >= HIGH_USE_BOUND){ + return "red"; + } else { + return "black"; + } +} + +/** + * handleStatsUpdats() is called when new data is received from the backend + * from a websocket event. The method updates the HTML site to reflect the + * updated data + * + * @param {*} data the data coming from the backend. + */ +function handleStatsUpdate(data) { + let userListInnerHTML = ""; + data.connectedUsers.sort(userDataComparator); + data.connectedUsers.map((userData, index) => { + let userRateColor = getUserRateColor(userData.rate); + let userEntry = `

${userData.time}

+

${userData.username}

+

Operations: ${userData.operations}

+

Rate: ${userData.rate} operations per last 10 seconds

+ `; // user data comes as last 10 seconds but it can be adjusted in DastStats.ts and websocket.ts + userListInnerHTML += "
  • " + userEntry + "
    • "; + }) + + document.getElementById("connection-count").innerHTML = `Current Connections: ${data.connectedUsers.length}` + document.getElementById("connected-user-list").innerHTML = userListInnerHTML; + + let serverTraffic = calculateServerTraffic(data); + let serverTrafficMessage = "Not Busy"; + switch(serverTraffic) { + case 0: + serverTrafficMessage = "Not Busy"; + break; + case 1: + serverTrafficMessage = "Busy"; + break; + case 2: + serverTrafficMessage = "Very Busy"; + break; + } + document.getElementById("stats-traffic-message").className="stats-server-status-item stats-server-status-" + serverTraffic; + document.getElementById("stats-traffic-message").innerHTML = `

    ${serverTrafficMessage}

    `; +} \ No newline at end of file diff --git a/views/stats.pug b/views/stats.pug index 54c017e70..16c28087e 100644 --- a/views/stats.pug +++ b/views/stats.pug @@ -1,22 +1,30 @@ extends ./layout +//- stats.pug is the frontend for the stats page block content style include ./stylesheets/authentication.css include ./stylesheets/statsview.css + script(src=`http://localhost:4321/socket.io/socket.io.js`) + script + include ./resources/statsviewcontroller.js + script. + var socket = io.connect("http://localhost:4321"); + socket.on("connect", () => console.log("connected to socket")); + + socket.on("a2cf757f-abd7-537b-953e-ef2f4f798f7e", (data) => handleStatsUpdate(data)); .outermost .stats-container h1 Dash Stats - p(class="stats-content") Current Connections: #{numConnections} + + p(class="stats-content" id="connection-count") Current Connections: #{numConnections} div(class="stats-content stats-server-status-container") p Server Status: - div(class="stats-server-status-item stats-server-status-" + serverTraffic) + div(id="stats-traffic-message" class="stats-server-status-item stats-server-status-" + serverTraffic) p #{serverTrafficMessage} div(class="stats-content stats-connected-users") p Connected Users: - ul - each username in connectedUsers - li(class="none")= username + ul(id="connected-user-list") \ No newline at end of file -- cgit v1.2.3-70-g09d2 From c2ee641e1f6a8003e961d03550aaffeb668f2545 Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 17 Apr 2023 11:14:07 -0400 Subject: more gitignore --- .gitignore | 4 ---- src/server/stats/userLoginStats.csv | 7 ------- 2 files changed, 11 deletions(-) delete mode 100644 src/server/stats/userLoginStats.csv (limited to 'src/server') diff --git a/.gitignore b/.gitignore index f60c94fa4..f841c1a86 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,6 @@ ClientUtils.ts solr-8.3.1/server/logs/ solr-8.3.1/server/solr/dash/data/tlog/* solr-8.3.1/server/solr/dash/data/index/* -src/scraping/buxton/final/source/ -src/scraping/buxton/final/json/ -src/scraping/buxton/source/ src/server/public/files/ src/server/stats/userLoginStats.csv src/scraping/acm/package-lock.json @@ -21,4 +18,3 @@ debug.log .vscodeignore Dockerfile .vscode/launch.json -src/server/stats/userLoginStats.csv diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv deleted file mode 100644 index 23bcef885..000000000 --- a/src/server/stats/userLoginStats.csv +++ /dev/null @@ -1,7 +0,0 @@ -USER,ACTION,TIMEaaa@gmail.com,loggedIn,2023-04-08T20:08:17.533Z -aaa@gmail.com,loggedIn,2023-04-08T20:14:32.460Z -aaa@gmail.com,loggedIn,2023-04-08T20:14:44.884Z -aaa@gmail.com,loggedIn,2023-04-08T20:14:56.854Z -aaa@gmail.com,loggedIn,2023-04-08T20:16:59.747Z -aaa@gmail.com,loggedIn,2023-04-08T20:56:50.759Z -aaa@gmail.com,loggedIn,2023-04-08T20:56:58.175Z -- cgit v1.2.3-70-g09d2 From f83e5d34794ef675d4627ecef2ed7042b17b1b06 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:05:01 -0400 Subject: cleaning up zip/unzip of files --- src/client/views/collections/CollectionView.tsx | 13 ++- src/client/views/nodes/DocumentView.tsx | 2 +- .../views/nodes/formattedText/DashFieldView.tsx | 6 +- src/fields/Doc.ts | 128 +++++++++------------ src/server/ApiManagers/UploadManager.ts | 9 +- 5 files changed, 67 insertions(+), 91 deletions(-) (limited to 'src/server') diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index cfb9310b6..790aa765d 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -156,12 +156,13 @@ export class CollectionView extends ViewBoxAnnotatableComponent { - const newRendition = Doc.MakeAlias(this.rootDoc); - newRendition._viewType = vtype; - this.props.addDocTab(newRendition, OpenWhere.addRight); - return newRendition; - }); + !Doc.noviceMode && + this.setupViewTypes('UI Controls...', vtype => { + const newRendition = Doc.MakeAlias(this.rootDoc); + newRendition._viewType = vtype; + this.props.addDocTab(newRendition, OpenWhere.addRight); + return newRendition; + }); const options = cm.findByDescription('Options...'); const optionItems = options && 'subitems' in options ? options.subitems : []; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 7107707d1..6686f142f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -693,7 +693,7 @@ export class DocumentViewInternal extends DocComponent this.props.addDocTab(templateDoc, OpenWhere.addRight), icon: 'eye' }); !appearance && appearanceItems.length && cm.addItem({ description: 'UI Controls...', subitems: appearanceItems, icon: 'compass' }); - if (!Doc.IsSystem(this.rootDoc) && this.rootDoc._viewType !== CollectionViewType.Docking && this.props.docViewPath().lastElement()?.rootDoc?._viewType !== CollectionViewType.Tree) { + if (!Doc.IsSystem(this.rootDoc) && this.rootDoc.type !== DocumentType.PRES && ![CollectionViewType.Docking, CollectionViewType.Tree].includes(this.rootDoc._viewType as any)) { const existingOnClick = cm.findByDescription('OnClick...'); const onClicks: ContextMenuProps[] = existingOnClick && 'subitems' in existingOnClick ? existingOnClick.subitems : []; diff --git a/src/client/views/nodes/formattedText/DashFieldView.tsx b/src/client/views/nodes/formattedText/DashFieldView.tsx index c43206629..72e8aedac 100644 --- a/src/client/views/nodes/formattedText/DashFieldView.tsx +++ b/src/client/views/nodes/formattedText/DashFieldView.tsx @@ -101,11 +101,7 @@ export class DashFieldViewInternal extends React.Component { - dashDoc instanceof Doc && (this._dashDoc = dashDoc); - }) - ); + DocServer.GetRefField(this.props.docId).then(action(dashDoc => dashDoc instanceof Doc && (this._dashDoc = dashDoc))); } else { this._dashDoc = this.props.tbox.rootDoc; } diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 6c808c145..c5af45262 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -705,7 +705,7 @@ export namespace Doc { if (cloneMap.get(doc[Id])) return cloneMap.get(doc[Id])!; const copy = new Doc(undefined, true); cloneMap.set(doc[Id], copy); - const filter = [...exclusions, ...Cast(doc.cloneFieldFilter, listSpec('string'), [])]; + const filter = [...exclusions, ...StrListCast(doc.cloneFieldFilter)]; await Promise.all( Object.keys(doc).map(async key => { if (filter.includes(key)) return; @@ -725,15 +725,13 @@ export namespace Doc { const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => '(' + exp + ')').join('|') + ')":"([a-z-A-Z0-9_]*)"', 'g'); const rawdocids = field.Data.match(docidsearch); const docids = rawdocids?.map((str: string) => - DocsInTextFieldIds.reduce((output, exp) => { - return output.replace(new RegExp(`${exp}":`, 'g'), ''); - }, str) + DocsInTextFieldIds.reduce((output, exp) => output.replace(new RegExp(`${exp}":`, 'g'), ''), str) .replace(/"/g, '') .trim() ); const results = docids && (await DocServer.GetRefFields(docids)); const docs = results && Array.from(Object.keys(results)).map(key => DocCast(results[key])); - docs && docs.map(doc => Doc.makeClone(doc, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); + docs?.map(doc => doc && Doc.makeClone(doc, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); rtfs.push({ copy, key, field }); } } @@ -741,7 +739,7 @@ export namespace Doc { }; const docAtKey = doc[key]; if (docAtKey instanceof Doc) { - if (!Doc.IsSystem(docAtKey) && (key.startsWith('layout') || key === 'annotationOn' || key === 'proto' || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + if (!Doc.IsSystem(docAtKey) && (key.startsWith('layout') || ['context', 'annotationOn', 'proto'].includes(key) || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); } else { assignKey(docAtKey); @@ -749,8 +747,7 @@ export namespace Doc { } else if (field instanceof RefField) { assignKey(field); } else if (cfield instanceof ComputedField) { - assignKey(cfield[Copy]()); - // ComputedField.MakeFunction(cfield.script.originalScript)); + assignKey(cfield[Copy]()); // ComputedField.MakeFunction(cfield.script.originalScript)); } else if (field instanceof ObjectField) { await copyObjectField(field); } else if (field instanceof Promise) { @@ -804,8 +801,6 @@ export namespace Doc { const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf'], cloneLinks); const repaired = new Set(); const linkedDocs = Array.from(linkMap.values()); - const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; - clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true)); rtfMap.map(({ copy, key, field }) => { const replacer = (match: any, attr: string, id: string, offset: any, string: any) => { @@ -816,20 +811,16 @@ export namespace Doc { const mapped = cloneMap.get(id); return href + (mapped ? mapped[Id] : id); }; - const regex = `(${Doc.localServerPath()})([^"]*)`; - const re = new RegExp(regex, 'g'); + const re = new RegExp(`(${Doc.localServerPath()})([^"]*)`, 'g'); const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => `"${exp}":`).join('|') + ')"([^"]+)"', 'g'); copy[key] = new RichTextField(field.Data.replace(docidsearch, replacer).replace(re, replacer2), field.Text); }); + const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; + clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); return { clone: copy, map: cloneMap, linkMap }; } - export async function Zip(doc: Doc) { - // const a = document.createElement("a"); - // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); - // a.href = url; - // a.download = `DocExport-${this.props.Document[Id]}.zip`; - // a.click(); + export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') { const { clone, map, linkMap } = await Doc.MakeClone(doc); clone.LINKS = new List(Array.from(linkMap.values())); const proms = [] as string[]; @@ -837,76 +828,61 @@ export namespace Doc { if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; else if (value instanceof Doc) { if (key !== 'field' && Number.isNaN(Number(key))) { - const __fields = value[FieldsSym](); - return { id: value[Id], __type: 'Doc', fields: __fields }; - } else { - return { fieldId: value[Id], __type: 'proxy' }; + return { id: value[Id], __type: 'Doc', fields: value[FieldsSym]() }; } - } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; - else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; - else if (value instanceof ImageField) { + return { fieldId: value[Id], __type: 'proxy' }; + } else if (value instanceof ImageField) { const extension = value.url.href.replace(/.*\./, ''); proms.push(value.url.href.replace('.' + extension, '_o.' + extension)); return { url: value.url.href, __type: 'image' }; } else if (value instanceof PdfField) { proms.push(value.url.href); return { url: value.url.href, __type: 'pdf' }; - } else if (value instanceof AudioField) return { url: value.url.href, __type: 'audio' }; - else if (value instanceof VideoField) return { url: value.url.href, __type: 'video' }; + } else if (value instanceof AudioField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'audio' }; + } else if (value instanceof VideoField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'video' }; + } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; + else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; else if (value instanceof WebField) return { url: value.url.href, __type: 'web' }; else if (value instanceof MapField) return { url: value.url.href, __type: 'map' }; else if (value instanceof DateField) return { date: value.toString(), __type: 'date' }; else if (value instanceof ProxyField) return { fieldId: value.fieldId, __type: 'proxy' }; else if (value instanceof Array && key !== 'fields') return { fields: value, __type: 'list' }; else if (value instanceof ComputedField) return { script: value.script, __type: 'computed' }; - else return value; + return value; } const docs: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const docString = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); - - let generateZIP = (proms: string[]) => { - var zip = new JSZip(); - var count = 0; - var zipFilename = 'dashExport.zip'; - - proms - .filter(url => url.startsWith(window.location.origin)) - .forEach((url, i) => { - var filename = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); - // loading a file and add it in a zip file - JSZipUtils.getBinaryContent(url, function (err: any, data: any) { - if (err) { - throw err; // or handle the error - } - zip.file(filename, data, { binary: true }); - count++; - if (count == proms.length) { - zip.file('doc.json', docString); - zip.generateAsync({ type: 'blob' }).then(function (content) { - saveAs(content, zipFilename); - }); - } - }); - }); - }; - generateZIP(proms); - const zip = new JSZip(); - - zip.file('doc.json', docString); - - // // Generate a directory within the Zip file structure - // var img = zip.folder("images"); - - // // Add a file to the directory, in this case an image with data URI as contents - // img.file("smile.gif", imgData, {base64: true}); + const jsonDocs = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); - // Generate the zip file asynchronously - zip.generateAsync({ type: 'blob' }).then((content: any) => { - // Force down of the Zip file - saveAs(content, doc.title + '.zip'); // glr: Possibly change the name of the document to match the title? - }); + const zip = new JSZip(); + var count = 0; + proms + .filter(url => url.startsWith(window.location.origin)) + .forEach((url, i) => { + // loading a file and add it in a zip file + JSZipUtils.getBinaryContent(url, (err: any, data: any) => { + if (err) throw err; // or handle the error + // // Generate a directory within the Zip file structure + // const assets = zip.folder("assets"); + // assets.file(filename, data, {binary: true}); + const assetPathOnServer = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); + zip.file(assetPathOnServer, data, { binary: true }); + if (++count == proms.length) { + zip.file('doc.json', jsonDocs); + zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename)); + // const a = document.createElement("a"); + // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); + // a.href = url; + // a.download = `DocExport-${this.props.Document[Id]}.zip`; + // a.click(); + } + }); + }); } const _pendingMap: Map = new Map(); @@ -963,7 +939,7 @@ export namespace Doc { // otherwise, it just returns the childDoc export function GetLayoutDataDocPair(containerDoc: Doc, containerDataDoc: Opt, childDoc: Doc) { if (!childDoc || childDoc instanceof Promise || !Doc.GetProto(childDoc)) { - console.log('No, no, no!'); + console.log('Warning: GetLayoutDataDocPair childDoc not defined'); return { layout: childDoc, data: childDoc }; } const resolvedDataDoc = Doc.AreProtosEqual(containerDataDoc, containerDoc) || (!childDoc.isTemplateDoc && !childDoc.isTemplateForField) ? undefined : containerDataDoc; @@ -1553,15 +1529,19 @@ export namespace Doc { } } - export async function importDocument(file: File) { + /// + // imports a previously exported zip file which contains a set of documents and their assets (eg, images, videos) + // the 'remap' parameter determines whether the ids of the documents loaded should be kept as they were, or remapped to new ids + // If they are not remapped, loading the file will overwrite any existing documents with those ids + // + export async function importDocument(file: File, remap = false) { const upload = Utils.prepend('/uploadDoc'); const formData = new FormData(); if (file) { formData.append('file', file); - formData.append('remap', 'true'); + formData.append('remap', remap.toString()); const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); - console.log(json); if (json !== 'error') { await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 6e28268a9..2cc2ac46c 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -184,9 +184,8 @@ export default class UploadManager extends ApiManager { if (id.endsWith('Proto')) return id; if (id in ids) { return ids[id]; - } else { - return (ids[id] = v4()); } + return (ids[id] = v4()); }; const mapFn = (doc: any) => { if (doc.id) { @@ -266,7 +265,7 @@ export default class UploadManager extends ApiManager { await Promise.all( docs.map( (doc: any) => - new Promise(res => { + new Promise(res => Database.Instance.replace( doc.id, doc, @@ -275,8 +274,8 @@ export default class UploadManager extends ApiManager { res(); }, true - ); - }) + ) + ) ) ); } catch (e) { -- cgit v1.2.3-70-g09d2 From 237e39029201aba23249de1ab5c405197acffe5a Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:50:28 -0400 Subject: more cleanup of import/export --- src/fields/Doc.ts | 15 +++++++++------ src/server/ApiManagers/UploadManager.ts | 34 +++++++++++++++------------------ 2 files changed, 24 insertions(+), 25 deletions(-) (limited to 'src/server') diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index c5af45262..35f9a1032 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -822,7 +822,6 @@ export namespace Doc { export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') { const { clone, map, linkMap } = await Doc.MakeClone(doc); - clone.LINKS = new List(Array.from(linkMap.values())); const proms = [] as string[]; function replacer(key: any, value: any) { if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; @@ -856,8 +855,10 @@ export namespace Doc { } const docs: { [id: string]: any } = {}; + const links: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const jsonDocs = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); + Array.from(linkMap.entries()).forEach(l => (links[l[0]] = l[1])); + const jsonDocs = JSON.stringify({ id: clone[Id], docs, links }, decycle(replacer)); const zip = new JSZip(); var count = 0; @@ -873,7 +874,7 @@ export namespace Doc { const assetPathOnServer = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); zip.file(assetPathOnServer, data, { binary: true }); if (++count == proms.length) { - zip.file('doc.json', jsonDocs); + zip.file('docs.json', jsonDocs); zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename)); // const a = document.createElement("a"); // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); @@ -1543,10 +1544,12 @@ export namespace Doc { const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); if (json !== 'error') { - await DocServer.GetRefFields(json.docids as string[]); + const docs = await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); - (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); - doc.LINKS = undefined; + const links = await DocServer.GetRefFields(json.linkids as string[]); + Array.from(Object.keys(links)) + .map(key => links[key]) + .forEach(link => link instanceof Doc && LinkManager.Instance.addLink(link)); return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2cc2ac46c..1f72d72dc 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -228,6 +228,7 @@ export default class UploadManager extends ApiManager { remap = fields.remap !== 'false'; let id: string = ''; let docids: string[] = []; + let linkids: string[] = []; try { for (const name in files) { const f = files[name]; @@ -254,28 +255,23 @@ export default class UploadManager extends ApiManager { console.log(e); } }); - const json = zip.getEntry('doc.json'); + const json = zip.getEntry('docs.json'); try { const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); - const datadocs = data.docs; + const { docs, links } = data; id = getId(data.id); - const docs = Object.keys(datadocs).map(key => datadocs[key]); - docs.forEach(mapFn); - docids = docs.map(doc => doc.id); + const rdocs = Object.keys(docs).map(key => docs[key]); + const ldocs = Object.keys(links).map(key => links[key]); + [...rdocs, ...ldocs].forEach(mapFn); + docids = rdocs.map(doc => doc.id); + linkids = ldocs.map(link => link.id); await Promise.all( - docs.map( - (doc: any) => - new Promise(res => - Database.Instance.replace( - doc.id, - doc, - (err, r) => { - err && console.log(err); - res(); - }, - true - ) - ) + [...rdocs, ...ldocs].map( + doc => + new Promise(res => { + // overwrite mongo doc with json doc contents + Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true); + }) ) ); } catch (e) { @@ -284,7 +280,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify({ id, docids } || 'error')); + res.send(JSON.stringify({ id, docids, linkids } || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From 16a33f53c2a031f791ed8c98d539409e1a7c6bb4 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:52:43 -0400 Subject: from last --- src/server/ApiManagers/UploadManager.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 1f72d72dc..74c06b4a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -180,11 +180,8 @@ export default class UploadManager extends ApiManager { const ids: { [id: string]: string } = {}; let remap = true; const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith('Proto')) return id; - if (id in ids) { - return ids[id]; - } + if (!remap || id.endsWith('Proto')) return id; + if (id in ids) return ids[id]; return (ids[id] = v4()); }; const mapFn = (doc: any) => { -- cgit v1.2.3-70-g09d2