From e6d1dd82f5d472bddaf66f9210142249d6fa09aa Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Wed, 6 Jul 2022 20:21:04 -0400 Subject: added endpoint routes for csv and integrated chartjs --- src/server/ApiManagers/DataVizManager.ts | 26 ++++++++++++++++++++++++++ src/server/DataVizUtils.ts | 6 ++++++ src/server/index.ts | 2 ++ 3 files changed, 34 insertions(+) create mode 100644 src/server/ApiManagers/DataVizManager.ts (limited to 'src/server') diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts new file mode 100644 index 000000000..0d43130d1 --- /dev/null +++ b/src/server/ApiManagers/DataVizManager.ts @@ -0,0 +1,26 @@ +import { csvParser, csvToString } from "../DataVizUtils"; +import { Method, _success } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Directory, serverPathToFile } from "./UploadManager"; +import * as path from 'path'; + +export default class DataVizManager extends ApiManager { + protected initialize(register: Registration): void { + register({ + method: Method.GET, + subscription: "/csvData", + secureHandler: async ({ req, res }) => { + const uri = req.query.uri as string; + + return new Promise(resolve => { + const name = path.basename(uri); + const sPath = serverPathToFile(Directory.csv, name); + const parsedCsv = csvParser(csvToString(sPath)); + _success(res, parsedCsv); + resolve(); + }); + } + }); + } + +} \ No newline at end of file diff --git a/src/server/DataVizUtils.ts b/src/server/DataVizUtils.ts index 4fd0ca6ff..2528fb1ab 100644 --- a/src/server/DataVizUtils.ts +++ b/src/server/DataVizUtils.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "fs"; + export function csvParser(csv: string) { const lines = csv.split("\n"); const headers = lines[0].split(","); @@ -10,4 +12,8 @@ export function csvParser(csv: string) { return obj; }); return data; +} + +export function csvToString(path: string) { + return readFileSync(path, 'utf8'); } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index f8c32103b..0acb7f20f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -4,6 +4,7 @@ import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import * as qs from 'query-string'; import { log_execution } from "./ActionUtilities"; +import DataVizManager from "./ApiManagers/DataVizManager"; import DeleteManager from "./ApiManagers/DeleteManager"; import DownloadManager from './ApiManagers/DownloadManager'; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; @@ -73,6 +74,7 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: new UtilManager(), new GeneralGoogleManager(), new GooglePhotosManager(), + new DataVizManager(), ]; // initialize API Managers -- cgit v1.2.3-70-g09d2 From 0f3f5fa7f63dddbfbf095f65b05f89cf27cbc6cf Mon Sep 17 00:00:00 2001 From: jameshu111 Date: Mon, 6 Mar 2023 16:14:57 -0500 Subject: /stats handler --- package-lock.json | 169 ++++++++++++++++++---------------- package.json | 2 + src/server/ApiManagers/UserManager.ts | 1 + src/server/DashStats.ts | 70 ++++++++++++++ src/server/index.ts | 7 ++ src/server/stats/userLoginStats.csv | 2 + src/server/websocket.ts | 17 +++- 7 files changed, 188 insertions(+), 80 deletions(-) create mode 100644 src/server/DashStats.ts create mode 100644 src/server/stats/userLoginStats.csv (limited to 'src/server') diff --git a/package-lock.json b/package-lock.json index 4695adf40..a5f1324a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5318,6 +5318,19 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.6.21.tgz", "integrity": "sha512-Z1PhmomIfypOpoMjRQB70jfvy/wxT50qW08YXO5lMIJkrdq4yOTR+AW7FqutScmB9NkLwxo+jU+kZLbofZZq/w==" }, + "csv-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz", + "integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==", + "requires": { + "minimist": "^1.2.0" + } + }, + "csv-stringify": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.3.0.tgz", + "integrity": "sha512-kTnnBkkLmAR1G409aUdShppWUClNbBQZXhrKrXzKYBGw4yfROspiFvVmjbKonCrdGfwnqwMXKLQG7ej7K/jwjg==" + }, "currently-unhandled": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", @@ -13650,7 +13663,7 @@ "dependencies": { "@iarna/cli": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@iarna/cli/-/cli-2.1.0.tgz", + "resolved": false, "integrity": "sha512-rvVVqDa2g860niRbqs3D5RhL4la3dc1vwk+NlpKPZxKaMSHtE2se6C2x8NeveN+rcjp3/686X+u+09CZ+7lmAQ==", "requires": { "glob": "^7.1.2", @@ -13753,7 +13766,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -13768,7 +13781,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -13777,12 +13790,12 @@ }, "asap": { "version": "2.0.6", - "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "resolved": false, "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "asn1": { "version": "0.2.6", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "resolved": false, "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", "requires": { "safer-buffer": "~2.1.0" @@ -13790,7 +13803,7 @@ }, "assert-plus": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "resolved": false, "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "asynckit": { @@ -13800,22 +13813,22 @@ }, "aws-sign2": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "resolved": false, "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" }, "aws4": { "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", + "resolved": false, "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "balanced-match": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "resolved": false, "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "bcrypt-pbkdf": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "resolved": false, "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", "requires": { "tweetnacl": "^0.14.3" @@ -13836,7 +13849,7 @@ }, "bluebird": { "version": "3.7.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "resolved": false, "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "boxen": { @@ -13884,7 +13897,7 @@ }, "cacache": { "version": "12.0.4", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", + "resolved": false, "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", "requires": { "bluebird": "^3.5.5", @@ -13921,7 +13934,7 @@ }, "caseless": { "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "resolved": false, "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, "chalk": { @@ -14065,7 +14078,7 @@ }, "combined-stream": { "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "resolved": false, "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "requires": { "delayed-stream": "~1.0.0" @@ -14073,7 +14086,7 @@ }, "concat-map": { "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "resolved": false, "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "concat-stream": { @@ -14103,7 +14116,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14118,7 +14131,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14127,7 +14140,7 @@ }, "config-chain": { "version": "1.1.13", - "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "resolved": false, "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", "requires": { "ini": "^1.3.4", @@ -14228,7 +14241,7 @@ }, "dashdash": { "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "resolved": false, "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", "requires": { "assert-plus": "^1.0.0" @@ -14261,7 +14274,7 @@ }, "decode-uri-component": { "version": "0.2.2", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "resolved": false, "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==" }, "deep-extend": { @@ -14307,7 +14320,7 @@ }, "dezalgo": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "resolved": false, "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", "requires": { "asap": "^2.0.0", @@ -14359,7 +14372,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14374,7 +14387,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14383,7 +14396,7 @@ }, "ecc-jsbn": { "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "resolved": false, "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", "requires": { "jsbn": "~0.1.0", @@ -14418,7 +14431,7 @@ }, "env-paths": { "version": "2.2.1", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "resolved": false, "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==" }, "err-code": { @@ -14502,7 +14515,7 @@ }, "extsprintf": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "resolved": false, "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" }, "fast-json-stable-stringify": { @@ -14512,12 +14525,12 @@ }, "figgy-pudding": { "version": "3.5.2", - "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz", + "resolved": false, "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==" }, "filter-obj": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "resolved": false, "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==" }, "find-npm-prefix": { @@ -14550,7 +14563,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14565,7 +14578,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14574,12 +14587,12 @@ }, "forever-agent": { "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "resolved": false, "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" }, "form-data": { "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "resolved": false, "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", "requires": { "asynckit": "^0.4.0", @@ -14612,7 +14625,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14627,7 +14640,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14695,7 +14708,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14710,7 +14723,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14719,7 +14732,7 @@ }, "fs.realpath": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "resolved": false, "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "function-bind": { @@ -14809,7 +14822,7 @@ }, "getpass": { "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "resolved": false, "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", "requires": { "assert-plus": "^1.0.0" @@ -14817,7 +14830,7 @@ }, "glob": { "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "resolved": false, "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "requires": { "fs.realpath": "^1.0.0", @@ -14830,7 +14843,7 @@ "dependencies": { "minimatch": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "resolved": false, "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" @@ -14873,12 +14886,12 @@ }, "graceful-fs": { "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "resolved": false, "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "har-schema": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "resolved": false, "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==" }, "har-validator": { @@ -14957,7 +14970,7 @@ }, "http-signature": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "resolved": false, "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", "requires": { "assert-plus": "^1.0.0", @@ -15084,7 +15097,7 @@ }, "is-cidr": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/is-cidr/-/is-cidr-3.1.1.tgz", + "resolved": false, "integrity": "sha512-Gx+oErgq1j2jAKCR2Kbq0b3wbH0vQKqZ0wOlHxm0o56nq51Cs/DZA8oz9dMDhbHyHEGgJ86eTeVudtgMMOx3Mw==", "requires": { "cidr-regex": "^2.0.10" @@ -15163,7 +15176,7 @@ }, "is-typedarray": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "resolved": false, "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" }, "isarray": { @@ -15178,12 +15191,12 @@ }, "isstream": { "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "resolved": false, "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "jsbn": { "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "resolved": false, "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" }, "json-parse-better-errors": { @@ -15193,7 +15206,7 @@ }, "json-parse-even-better-errors": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "resolved": false, "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" }, "json-schema": { @@ -15203,7 +15216,7 @@ }, "json-stringify-safe": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "resolved": false, "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, "jsonparse": { @@ -15421,7 +15434,7 @@ }, "lock-verify": { "version": "2.2.2", - "resolved": "https://registry.npmjs.org/lock-verify/-/lock-verify-2.2.2.tgz", + "resolved": false, "integrity": "sha512-2CUNtr1ZSVKJHcYP8uEzafmmuyauCB5zZimj8TvQd/Lflt9kXVZs+8S+EbAzZLaVUDn8CYGmeC3DFGdYfnCzeQ==", "requires": { "@iarna/cli": "^2.1.0", @@ -15550,7 +15563,7 @@ }, "meant": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/meant/-/meant-1.0.3.tgz", + "resolved": false, "integrity": "sha512-88ZRGcNxAq4EH38cQ4D85PM57pikCwS8Z99EWHODxN7KBY+UuPiqzRTtZzS8KTXO/ywSWbdjjJST2Hly/EQxLw==" }, "mime-db": { @@ -15568,7 +15581,7 @@ }, "minimatch": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "resolved": false, "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" @@ -15617,7 +15630,7 @@ }, "mkdirp": { "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "resolved": false, "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "requires": { "minimist": "^1.2.6" @@ -15665,7 +15678,7 @@ }, "node-gyp": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-5.1.1.tgz", + "resolved": false, "integrity": "sha512-WH0WKGi+a4i4DUt2mHnvocex/xPLp9pYt5R6M2JdFB7pJ7Z34hveZ4nDTGTiLXCkitA9T8HFZjhinBCiVHYcWw==", "requires": { "env-paths": "^2.2.0", @@ -16003,7 +16016,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16018,7 +16031,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16032,7 +16045,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": false, "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, "path-is-inside": { @@ -16052,7 +16065,7 @@ }, "performance-now": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "resolved": false, "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "pify": { @@ -16101,7 +16114,7 @@ }, "proto-list": { "version": "1.2.4", - "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "resolved": false, "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==" }, "protoduck": { @@ -16124,7 +16137,7 @@ }, "psl": { "version": "1.9.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "resolved": false, "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, "pump": { @@ -16164,12 +16177,12 @@ }, "qs": { "version": "6.5.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", + "resolved": false, "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==" }, "query-string": { "version": "6.14.1", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.1.tgz", + "resolved": false, "integrity": "sha512-XDxAeVmpfu1/6IjyT/gXHOl+S0vQ9owggJ30hhWKdHAsNPOcasn5o9BW0eejZqL2e4vMjhAxoW3jVHcD6mbcYw==", "requires": { "decode-uri-component": "^0.2.0", @@ -16180,7 +16193,7 @@ }, "qw": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/qw/-/qw-1.0.2.tgz", + "resolved": false, "integrity": "sha512-1PhZ/iLKwlVNq45dnerTMKFjMof49uqli7/0QsvPNbX5OJ3IZ8msa9lUpvPheVdP+IYYPrf6cOaVil7S35joVA==" }, "rc": { @@ -16226,7 +16239,7 @@ }, "read-package-json": { "version": "2.1.2", - "resolved": "https://registry.npmjs.org/read-package-json/-/read-package-json-2.1.2.tgz", + "resolved": false, "integrity": "sha512-D1KmuLQr6ZSJS0tW8hf3WGpRlwszJOXZ3E8Yd/DNRaM5d+1wVRZdHlpGBLAuovjr28LbWvjpWkBHMxpRGGjzNA==", "requires": { "glob": "^7.1.1", @@ -16285,7 +16298,7 @@ }, "request": { "version": "2.88.2", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "resolved": false, "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "requires": { "aws-sign2": "~0.7.0", @@ -16355,7 +16368,7 @@ }, "safe-buffer": { "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "resolved": false, "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safer-buffer": { @@ -16526,7 +16539,7 @@ }, "sshpk": { "version": "1.17.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", + "resolved": false, "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", "requires": { "asn1": "~0.2.3", @@ -16582,7 +16595,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16597,7 +16610,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16611,7 +16624,7 @@ }, "strict-uri-encode": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "resolved": false, "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==" }, "string-width": { @@ -16767,7 +16780,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16782,7 +16795,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16801,7 +16814,7 @@ }, "tough-cookie": { "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "resolved": false, "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "requires": { "psl": "^1.1.28", @@ -16810,7 +16823,7 @@ "dependencies": { "punycode": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "resolved": false, "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" } } @@ -16825,7 +16838,7 @@ }, "tweetnacl": { "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "resolved": false, "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "typedarray": { @@ -16896,7 +16909,7 @@ }, "uri-js": { "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "resolved": false, "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "requires": { "punycode": "^2.1.0" @@ -16937,7 +16950,7 @@ }, "uuid": { "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "resolved": false, "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" }, "validate-npm-package-license": { @@ -16959,7 +16972,7 @@ }, "verror": { "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "resolved": false, "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", "requires": { "assert-plus": "^1.0.0", diff --git a/package.json b/package.json index 00ce356f9..7fa90e431 100644 --- a/package.json +++ b/package.json @@ -184,6 +184,8 @@ "cookie-session": "^2.0.0", "core-js": "^3.28.0", "cors": "^2.8.5", + "csv-parser": "^3.0.0", + "csv-stringify": "^6.3.0", "depcheck": "^0.9.2", "equation-editor-react": "github:bobzel/equation-editor-react#useLocally", "exif": "^0.6.0", diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 53e55c1c3..c3dadd821 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -5,6 +5,7 @@ import { msToTime } from '../ActionUtilities'; import * as bcrypt from 'bcrypt-nodejs'; import { Opt } from '../../fields/Doc'; import { WebSocket } from '../websocket'; +import { DashStats } from '../DashStats'; export const timeMap: { [id: string]: number } = {}; interface ActivityUnit { diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts new file mode 100644 index 000000000..13ee61b3a --- /dev/null +++ b/src/server/DashStats.ts @@ -0,0 +1,70 @@ +import { magenta } from 'colors'; +import { Request, Response } from 'express'; +import SocketIO from 'socket.io'; +import { Client } from './Client'; +import { WebSocket } from './websocket'; +const fs = require('fs'); +const csv = require('csv-parser'); +import { stringify } from 'csv-stringify/sync'; + +export namespace DashStats { + const statsCSVFilename = './src/server/stats/userLoginStats.csv'; + const columns = ['USERNAME', 'ACTION', 'TIME']; + + interface SocketPair { + socketId: string; + username: string; + } + + interface CSVStore { + USERNAME: string; + ACTION: string; + TIME: string; + } + + export function handleStatsView(res: Response) { + const results: CSVStore[] = []; + fs.createReadStream(statsCSVFilename) + .pipe(csv(columns)) + .on('data', (data: any) => results.push(data)) + .on('end', () => { + console.log(results); + }); + + // let newRow = stringify([{ USERNAME: 'hi', ACTION: 'hi', TIME: 'hi' }], { header: true, columns: columns }); + // console.log(newRow); + + let current = getCurrentConnections(); + res.json({ + message: 'welcome to stats', + currentConnections: current.length, + socketMap: current, + }); + } + + export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { + if (!(username === undefined)) { + let currentDate = new Date(); + console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + console.log('stringify -> '); + } + } + + export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { + if (!(username === undefined)) { + let currentDate = new Date(); + console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); + } + } + + function getCurrentConnections(): SocketPair[] { + let socketPairs: SocketPair[] = []; + for (let [key, value] of WebSocket.socketMap) { + if (!key.disconnected) { + socketPairs.push({ socketId: key.id, username: value.split(' ')[0] }); + } + } + console.log(socketPairs); + return socketPairs; + } +} diff --git a/src/server/index.ts b/src/server/index.ts index 6e6bde3cb..0848d828e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -18,6 +18,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { DashSessionAgent } from './DashSession/DashSessionAgent'; import { AppliedSessionAgent } from './DashSession/Session/agents/applied_session_agent'; +import { DashStats } from './DashStats'; import { DashUploadUtils } from './DashUploadUtils'; import { Database } from './database'; import { Logger } from './ProcessFactory'; @@ -83,6 +84,12 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ res }) => res.send(true), }); + addSupervisedRoute({ + method: Method.GET, + subscription: '/stats', + secureHandler: ({ res }) => DashStats.handleStatsView(res), + }); + addSupervisedRoute({ method: Method.GET, subscription: '/resolvedPorts', diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv new file mode 100644 index 000000000..ec33b01b2 --- /dev/null +++ b/src/server/stats/userLoginStats.csv @@ -0,0 +1,2 @@ +USERNAME,ACTION,TIME +boo15869,loggedIn,2023-01 \ No newline at end of file diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 9b91a35a6..e556ecc17 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -1,4 +1,4 @@ -import { blue } from 'colors'; +import { blue, magenta } from 'colors'; import * as express from 'express'; import { createServer, Server } from 'https'; import { networkInterfaces } from 'os'; @@ -12,6 +12,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; import YoutubeApi from './apis/youtube/youtubeApiSample'; import { initializeGuest } from './authentication/DashUserModel'; import { Client } from './Client'; +import { DashStats } from './DashStats'; import { Database } from './database'; import { DocumentsCollection } from './IDatabase'; import { Diff, GestureContent, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, Transferable, Types, UpdateMobileInkOverlayPositionContent, YoutubeQueryInput, YoutubeQueryTypes } from './Message'; @@ -20,7 +21,7 @@ import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { export let _socket: Socket; - const clients: { [key: string]: Client } = {}; + export const clients: { [key: string]: Client } = {}; export const socketMap = new Map(); export let disconnect: Function; @@ -97,6 +98,13 @@ export namespace WebSocket { console.log('received bye'); }); + socket.on('disconnect', function () { + let currentUser = socketMap.get(socket); + if (!(currentUser === undefined)) { + DashStats.logUserLogout(socketMap.get(socket), socket); + } + }); + Utils.Emit(socket, MessageStore.Foo, 'handshooken'); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -176,7 +184,12 @@ export namespace WebSocket { const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); + console.log(magenta(`currently connected: ${JSON.stringify(clients)}`)); + // console.log(magenta('socket map below')); + // console.log([...socketMap.entries()]); + printActiveUsers(); socketMap.set(socket, userEmail + ' at ' + datetime); + DashStats.logUserLogin(socketMap.get(socket), socket); } function getField([id, callback]: [string, (result?: Transferable) => void]) { -- cgit v1.2.3-70-g09d2 From 0e55893d0f7f2a0aa5098df73d0ece5a7f1a4ddf Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 15 Mar 2023 22:33:22 -0400 Subject: fixed up Clone() and export/import collection to work with links, presentations, and contexts better. --- .eslintrc.json | 4 ++ package-lock.json | 78 +++++++++++++--------- package.json | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 6 +- src/client/views/nodes/DocumentView.tsx | 1 + src/client/views/nodes/trails/PresBox.tsx | 7 +- src/decycler/decycler.d.ts | 2 + src/decycler/decycler.js | 51 ++++++++++++++ src/fields/Doc.ts | 75 ++++++++++++--------- src/server/ApiManagers/UploadManager.ts | 24 +++---- tsconfig.json | 24 ++----- 11 files changed, 170 insertions(+), 104 deletions(-) create mode 100644 src/decycler/decycler.d.ts create mode 100644 src/decycler/decycler.js (limited to 'src/server') diff --git a/.eslintrc.json b/.eslintrc.json index b9f8e1b7a..43bb53566 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -10,5 +10,9 @@ "object-shorthand": "off", "class-methods-use-this": "off", "single-quote": "off" + }, + "parserOptions": { + "ecmaVersion": 11, + "sourceType": "module" } } diff --git a/package-lock.json b/package-lock.json index 4695adf40..cc51ad9e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -590,15 +590,30 @@ "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz", "integrity": "sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA==" }, + "@eslint-community/eslint-utils": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.2.0.tgz", + "integrity": "sha512-gB8T4H4DEfX2IV9zGDJPOBgP1e/DbfCPDTtEqUMckpvzS1OYtva8JdFYBqMwYk7xAQ429WGF/UPqn8uQ//h2vQ==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^3.3.0" + } + }, + "@eslint-community/regexpp": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.4.0.tgz", + "integrity": "sha512-A9983Q0LnDGdLPjxyXQ00sbV+K+O+ko2Dr+CZigbHWtX9pNfxlaBkMR8X1CztI73zuEyEBXTVjx7CE+/VSwDiQ==", + "dev": true + }, "@eslint/eslintrc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz", - "integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.1.tgz", + "integrity": "sha512-eFRmABvW2E5Ho6f5fHLqgena46rOj7r7OKHYfLElqcBfGFHHpjBhivyi5+jOEQuSpdc/1phIZJlbC2te+tZNIw==", "dev": true, "requires": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.4.0", + "espree": "^9.5.0", "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", @@ -660,6 +675,12 @@ } } }, + "@eslint/js": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.36.0.tgz", + "integrity": "sha512-lxJ9R5ygVm8ZWgYdUweoq5ownDlJ4upvoWmO4eLxBYHdMo+vZ/Rx0EN6MbKWDJOSUGrqJy2Gt+Dyv/VKml0fjg==", + "dev": true + }, "@ffmpeg/core": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz", @@ -6683,12 +6704,15 @@ } }, "eslint": { - "version": "8.34.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.34.0.tgz", - "integrity": "sha512-1Z8iFsucw+7kSqXNZVslXS8Ioa4u2KM7GPwuKtkTFAqZ/cHMcEaR+1+Br0wLlot49cNxIiZk5wp8EAbPcYZxTg==", + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.36.0.tgz", + "integrity": "sha512-Y956lmS7vDqomxlaaQAHVmeb4tNMp2FWIvU/RnU5BD3IKMD/MJPr76xdyr68P8tV1iNMvN2mRK0yy3c+UjL+bw==", "dev": true, "requires": { - "@eslint/eslintrc": "^1.4.1", + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.4.0", + "@eslint/eslintrc": "^2.0.1", + "@eslint/js": "8.36.0", "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", @@ -6699,10 +6723,9 @@ "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", "eslint-scope": "^7.1.1", - "eslint-utils": "^3.0.0", "eslint-visitor-keys": "^3.3.0", - "espree": "^9.4.0", - "esquery": "^1.4.0", + "espree": "^9.5.0", + "esquery": "^1.4.2", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", @@ -6723,7 +6746,6 @@ "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", - "regexpp": "^3.2.0", "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", "text-table": "^0.2.0" @@ -6811,6 +6833,15 @@ "estraverse": "^5.2.0" } }, + "esquery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "dev": true, + "requires": { + "estraverse": "^5.1.0" + } + }, "estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", @@ -7852,23 +7883,6 @@ "estraverse": "^4.1.1" } }, - "eslint-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", - "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^2.0.0" - }, - "dependencies": { - "eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true - } - } - }, "eslint-visitor-keys": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", @@ -7876,9 +7890,9 @@ "dev": true }, "espree": { - "version": "9.4.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", - "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.0.tgz", + "integrity": "sha512-JPbJGhKc47++oo4JkEoTe2wjy4fmMwvFpgJT9cQzmfXKp22Dr6Hf1tdCteLz1h0P3t+mGvWZ+4Uankvh8+c6zw==", "dev": true, "requires": { "acorn": "^8.8.0", diff --git a/package.json b/package.json index 00ce356f9..2c4c41917 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,7 @@ "cross-env": "^5.2.1", "css-loader": "^2.1.1", "dotenv": "^8.6.0", - "eslint": "^8.18.0", + "eslint": "^8.36.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-node": "^4.1.0", "eslint-config-prettier": "^8.5.0", diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 15d8144fc..f0c140ef1 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1796,11 +1796,7 @@ export class CollectionFreeFormView extends CollectionSubView Doc.Zip(this.props.Document) }); - moreItems.push({ description: 'Import exported collection', icon: 'upload', event: ({ x, y }) => this.importDocument(e.clientX, e.clientY) }); - } + moreItems.push({ description: 'Import exported collection', icon: 'upload', event: ({ x, y }) => this.importDocument(e.clientX, e.clientY) }); !mores && ContextMenu.Instance.addItem({ description: 'More...', subitems: moreItems, icon: 'eye' }); }; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 02af30d0c..b7a760c1e 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -1002,6 +1002,7 @@ export class DocumentViewInternal extends DocComponent Utils.CopyText(Doc.globalServerPath(this.props.Document)), icon: 'fingerprint' }); } + moreItems.push({ description: 'Export collection', icon: 'download', event: async () => Doc.Zip(this.props.Document) }); } if (this.props.removeDocument && !Doc.IsSystem(this.rootDoc) && Doc.ActiveDashboard !== this.props.Document) { diff --git a/src/client/views/nodes/trails/PresBox.tsx b/src/client/views/nodes/trails/PresBox.tsx index be40b3592..e79e7472a 100644 --- a/src/client/views/nodes/trails/PresBox.tsx +++ b/src/client/views/nodes/trails/PresBox.tsx @@ -570,9 +570,9 @@ export class PresBox extends ViewBoxBaseComponent() { } } else { if (bestTarget._panX !== activeItem.presPanX || bestTarget._panY !== activeItem.presPanY || bestTarget._viewScale !== activeItem.presViewScale) { - bestTarget._panX = activeItem.presPanX; - bestTarget._panY = activeItem.presPanY; - bestTarget._viewScale = activeItem.presViewScale; + bestTarget._panX = activeItem.presPanX ?? bestTarget._panX; + bestTarget._panY = activeItem.presPanY ?? bestTarget._panY; + bestTarget._viewScale = activeItem.presViewScale ?? bestTarget._viewScale; changed = true; } } @@ -717,6 +717,7 @@ export class PresBox extends ViewBoxBaseComponent() { zoomTime: activeItem.presMovement === PresMovement.Jump ? 0 : Math.min(Math.max(effect ? 750 : 500, (effect ? 0.2 : 1) * presTime), presTime), effect: activeItem, noSelect: true, + openLocation: OpenWhere.addLeft, anchorDoc: activeItem, easeFunc: StrCast(activeItem.presEaseFunc, 'ease') as any, zoomTextSelections: BoolCast(activeItem.presZoomText), diff --git a/src/decycler/decycler.d.ts b/src/decycler/decycler.d.ts new file mode 100644 index 000000000..84620f79c --- /dev/null +++ b/src/decycler/decycler.d.ts @@ -0,0 +1,2 @@ +export declare const decycle: Function; +export declare const retrocycle: Function; diff --git a/src/decycler/decycler.js b/src/decycler/decycler.js new file mode 100644 index 000000000..7fb8a45c7 --- /dev/null +++ b/src/decycler/decycler.js @@ -0,0 +1,51 @@ +/// +/// this is a modified copy of the npm project: https://www.npmjs.com/package/json-decycle +/// the original code is used as replacer when stringifying JSON objects that have cycles. +/// However, we want an additional replacer that stringifies Dash Fields and Docs in a custom way. +/// So this modified code allows for a custom replacer to be added to this object that will run before this replacer +/// + +const g = e => typeof e === 'object' && e != null && !(e instanceof Boolean) && !(e instanceof Date) && !(e instanceof Number) && !(e instanceof RegExp) && !(e instanceof String); +const b = e => String('#') + e.map(t => String(t).replace(/~/g, '~0').replace(/\//g, '~1')).join('/'); +// eslint-disable-next-line node/no-unsupported-features/es-syntax +export function decycle(replacer) { + const e = new WeakMap(); + return function (n, rr) { + const r = replacer(n, rr); + if (n !== '$ref' && g(r)) { + if (e.has(r)) return { $ref: b(e.get(r)) }; + e.set(r, [...(e.get(this) === undefined ? [] : e.get(this)), n]); + } + return r; + }; +} +// eslint-disable-next-line node/no-unsupported-features/es-syntax +export function retrocycle() { + const e = new WeakMap(); + const t = new WeakMap(); + const n = new Set(); + function r(o) { + const c = o.$ref.slice(1).split('/'); + let s; + let a = this; + // eslint-disable-next-line no-plusplus + for (let p = 0; p < c.length; p++) { + s = c[p].replace(/~1/g, '/').replace(/~0/g, '~'); + a = a[s]; + } + const f = e.get(o); + f[t.get(o)] = a; + } + return function (c, s) { + if (c === '$ref') n.add(this); + else if (g(s)) { + const f = c === '' && Object.keys(this).length === 1; + if (f) n.forEach(r, this); + else { + e.set(s, this); + t.set(s, c); + } + } + return s; + }; +} diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index de94ed5db..168e29dd5 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -10,6 +10,7 @@ import { scriptingGlobal, ScriptingGlobals } from '../client/util/ScriptingGloba import { SelectionManager } from '../client/util/SelectionManager'; import { afterDocDeserialize, autoObject, Deserializable, SerializationHelper } from '../client/util/SerializationHelper'; import { UndoManager } from '../client/util/UndoManager'; +import { decycle } from '../decycler/decycler'; import { DashColor, incrementTitleCopy, intersectRect, Utils } from '../Utils'; import { DateField } from './DateField'; import { Copy, HandleUpdate, Id, OnUpdate, Parent, Self, SelfProxy, ToScriptString, ToString, Update } from './FieldSymbols'; @@ -25,7 +26,6 @@ import { Cast, DocCast, FieldValue, NumCast, StrCast, ToConstructor } from './Ty import { AudioField, ImageField, MapField, PdfField, VideoField, WebField } from './URLField'; import { deleteProperty, GetEffectiveAcl, getField, getter, makeEditable, makeReadOnly, normalizeEmail, setter, SharingPermissions, updateFunction } from './util'; import JSZip = require('jszip'); - export namespace Field { export function toKeyValueString(doc: Doc, key: string): string { const onDelegate = Object.keys(doc).includes(key); @@ -701,22 +701,12 @@ export namespace Doc { return bestAlias ?? Doc.MakeAlias(doc); } - export async function makeClone( - doc: Doc, - cloneMap: Map, - linkMap: Map, - rtfs: { copy: Doc; key: string; field: RichTextField }[], - exclusions: string[], - topLevelExclusions: string[], - dontCreate: boolean, - asBranch: boolean - ): Promise { + export async function makeClone(doc: Doc, cloneMap: Map, linkMap: Map, rtfs: { copy: Doc; key: string; field: RichTextField }[], exclusions: string[], dontCreate: boolean, asBranch: boolean): Promise { if (Doc.IsBaseProto(doc)) return doc; if (cloneMap.get(doc[Id])) return cloneMap.get(doc[Id])!; - const copy = dontCreate ? (asBranch ? Cast(doc.branchMaster, Doc, null) || doc : doc) : new Doc(undefined, true); + const copy = dontCreate ? (asBranch ? Cast(doc.branchMaster, Doc, null) ?? doc : doc) : new Doc(undefined, true); cloneMap.set(doc[Id], copy); - const fieldExclusions = doc.type === DocumentType.MARKER ? exclusions.filter(ex => ex !== 'annotationOn') : exclusions; - const filter = [...fieldExclusions, ...topLevelExclusions, ...Cast(doc.cloneFieldFilter, listSpec('string'), [])]; + const filter = [...exclusions, ...Cast(doc.cloneFieldFilter, listSpec('string'), [])]; await Promise.all( Object.keys(doc).map(async key => { if (filter.includes(key)) return; @@ -727,10 +717,10 @@ export namespace Doc { const list = await Cast(doc[key], listSpec(Doc)); const docs = list && (await DocListCastAsync(list))?.filter(d => d instanceof Doc); if (docs !== undefined && docs.length) { - const clones = await Promise.all(docs.map(async d => Doc.makeClone(d, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch))); + const clones = await Promise.all(docs.map(async d => Doc.makeClone(d, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch))); !dontCreate && assignKey(new List(clones)); } else if (doc[key] instanceof Doc) { - assignKey(key.includes('layout[') ? undefined : key.startsWith('layout') ? (doc[key] as Doc) : await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch)); // reference documents except copy documents that are expanded template fields + assignKey(key.includes('layout[') ? undefined : key.startsWith('layout') ? (doc[key] as Doc) : await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); // reference documents except copy documents that are expanded template fields } else { !dontCreate && assignKey(ObjectField.MakeCopy(field)); if (field instanceof RichTextField) { @@ -740,13 +730,12 @@ export namespace Doc { } } }; - if (key === 'proto') { - if (doc[key] instanceof Doc) { - assignKey(await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch)); - } - } else if (key === 'anchor1' || key === 'anchor2') { - if (doc[key] instanceof Doc) { - assignKey(await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, [], true, asBranch)); + const docAtKey = doc[key]; + if (docAtKey instanceof Doc) { + if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || (key === 'proto' && cloneMap.has(doc[Id])) || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); + } else { + assignKey(docAtKey); } } else { if (field instanceof RefField) { @@ -765,8 +754,8 @@ export namespace Doc { }) ); for (const link of Array.from(doc[DirectLinksSym])) { - const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch); - linkMap.set(link, linkClone); + const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch); + linkMap.set(link[Id], linkClone); } if (!dontCreate) { Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); @@ -779,11 +768,29 @@ export namespace Doc { Doc.AddFileOrphan(copy); return copy; } + export function repairClone(doc: Doc, cloned: Doc[], visited: Set) { + if (visited.has(doc)) return; + visited.add(doc); + Object.keys(doc).map(key => { + const docAtKey = DocCast(doc[key]); + if (docAtKey && !Doc.IsSystem(docAtKey)) { + if (!cloned.includes(docAtKey)) { + doc[key] = undefined; + } else { + repairClone(docAtKey, cloned, visited); + } + } + }); + } export async function MakeClone(doc: Doc, dontCreate: boolean = false, asBranch = false, cloneMap: Map = new Map()) { - const linkMap = new Map(); + const linkMap = new Map(); const rtfMap: { copy: Doc; key: string; field: RichTextField }[] = []; - const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf', 'branches', 'branchOf'], ['context'], dontCreate, asBranch); - Array.from(linkMap.entries()).map((links: Doc[]) => LinkManager.Instance.addLink(links[1], true)); + const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf', 'branches', 'branchOf'], dontCreate, asBranch); + const repaired = new Set(); + const linkedDocs = Array.from(linkMap.values()); + const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; + clonedDocs.map(clone => Doc.repairClone(clone, Array.from(cloneMap.values()), repaired)); + linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true)); rtfMap.map(({ copy, key, field }) => { const replacer = (match: any, attr: string, id: string, offset: any, string: any) => { const mapped = cloneMap.get(id); @@ -797,7 +804,7 @@ export namespace Doc { const re = new RegExp(regex, 'g'); copy[key] = new RichTextField(field.Data.replace(/("textId":|"audioId":|"anchorId":)"([^"]+)"/g, replacer).replace(re, replacer2), field.Text); }); - return { clone: copy, map: cloneMap }; + return { clone: copy, map: cloneMap, linkMap }; } export async function Zip(doc: Doc) { @@ -806,9 +813,10 @@ export namespace Doc { // a.href = url; // a.download = `DocExport-${this.props.Document[Id]}.zip`; // a.click(); - const { clone, map } = await Doc.MakeClone(doc, true); + const { clone, map, linkMap } = await Doc.MakeClone(doc, true); + clone.LINKS = new List(Array.from(linkMap.values())); function replacer(key: any, value: any) { - if (['branchOf', 'cloneOf', 'context', 'cursors'].includes(key)) return undefined; + if (['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; else if (value instanceof Doc) { if (key !== 'field' && Number.isNaN(Number(key))) { const __fields = value[FieldsSym](); @@ -833,7 +841,7 @@ export namespace Doc { const docs: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const docString = JSON.stringify({ id: doc[Id], docs }, replacer); + const docString = JSON.stringify({ id: doc[Id], docs }, decycle(replacer)); const zip = new JSZip(); @@ -1520,7 +1528,8 @@ export namespace Doc { const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); if (json !== 'error') { - const doc = await DocServer.GetRefField(json); + const doc = DocCast(await DocServer.GetRefField(json)); + (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index fe4c475c9..9bacbd5c8 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -1,19 +1,19 @@ -import ApiManager, { Registration } from './ApiManager'; -import { Method, _success } from '../RouteManager'; import * as formidable from 'formidable'; -import v4 = require('uuid/v4'); -const AdmZip = require('adm-zip'); -import { extname, basename, dirname } from 'path'; import { createReadStream, createWriteStream, unlink, writeFile } from 'fs'; -import { publicDirectory, filesDirectory } from '..'; -import { Database } from '../database'; -import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; +import { basename, dirname, extname, normalize } from 'path'; import * as sharp from 'sharp'; -import { AcceptableMedia, Upload } from '../SharedMediaTypes'; -import { normalize } from 'path'; +import { filesDirectory, publicDirectory } from '..'; +import { retrocycle } from '../../decycler/decycler'; +import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; +import { Database } from '../database'; +import { Method, _success } from '../RouteManager'; import RouteSubscriber from '../RouteSubscriber'; -const imageDataUri = require('image-data-uri'); +import { AcceptableMedia, Upload } from '../SharedMediaTypes'; +import ApiManager, { Registration } from './ApiManager'; import { SolrManager } from './SearchManager'; +import v4 = require('uuid/v4'); +const AdmZip = require('adm-zip'); +const imageDataUri = require('image-data-uri'); const fs = require('fs'); export enum Directory { @@ -252,7 +252,7 @@ export default class UploadManager extends ApiManager { }); const json = zip.getEntry('doc.json'); try { - const data = JSON.parse(json.getData().toString('utf8')); + const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); const datadocs = data.docs; id = getId(data.id); const docs = Object.keys(datadocs).map(key => datadocs[key]); diff --git a/tsconfig.json b/tsconfig.json index 993ab13b9..bff9255db 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,34 +2,22 @@ "compilerOptions": { "target": "es5", "downlevelIteration": true, - // "module": "system", "removeComments": true, "experimentalDecorators": true, "allowSyntheticDefaultImports": true, + "moduleDetection": "auto", "strict": true, "jsx": "react", "allowJs": true, "sourceMap": true, "outDir": "dist", - "lib": [ - "dom", - "es2015" - ], - "typeRoots": [ - "node_modules/@types", - "./src/typings" - ], - "types": [ - "youtube", - "node" - ] + "lib": ["dom", "es2015"], + "typeRoots": ["node_modules/@types", "./src/typings"], + "types": ["youtube", "node"] }, // "exclude": [ // "node_modules", // "static" // ], - "typeRoots": [ - "./node_modules/@types", - "./src/typings" - ] -} \ No newline at end of file + "typeRoots": ["./node_modules/@types", "./src/typings"] +} -- cgit v1.2.3-70-g09d2 From d2bca182a311e95515bbff8fb378b29918fe99d7 Mon Sep 17 00:00:00 2001 From: geireann Date: Thu, 16 Mar 2023 12:00:59 -0400 Subject: fixed export/import collectoin --- src/fields/Doc.ts | 34 +++++++++++++++++++-------------- src/server/ApiManagers/UploadManager.ts | 4 +++- 2 files changed, 23 insertions(+), 15 deletions(-) (limited to 'src/server') diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 168e29dd5..6543679ad 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -732,7 +732,7 @@ export namespace Doc { }; const docAtKey = doc[key]; if (docAtKey instanceof Doc) { - if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || (key === 'proto' && cloneMap.has(doc[Id])) || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || key === 'proto'|| ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); } else { assignKey(docAtKey); @@ -757,8 +757,8 @@ export namespace Doc { const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch); linkMap.set(link[Id], linkClone); } - if (!dontCreate) { Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); + if (!dontCreate) { asBranch ? (copy.branchOf = doc) : (copy.cloneOf = doc); if (!Doc.IsPrototype(copy)) { Doc.AddDocToList(doc, 'branches', Doc.GetProto(copy)); @@ -768,16 +768,19 @@ export namespace Doc { Doc.AddFileOrphan(copy); return copy; } - export function repairClone(doc: Doc, cloned: Doc[], visited: Set) { - if (visited.has(doc)) return; - visited.add(doc); - Object.keys(doc).map(key => { - const docAtKey = DocCast(doc[key]); + export function repairClone(clone: Doc, cloneMap: Map, visited: Set) { + if (visited.has(clone)) return; + visited.add(clone); + Object.keys(clone).filter(key => key !== "cloneOf").map(key => { + const docAtKey = DocCast(clone[key]); if (docAtKey && !Doc.IsSystem(docAtKey)) { - if (!cloned.includes(docAtKey)) { - doc[key] = undefined; + if (!Array.from(cloneMap.values()).includes(docAtKey)) { + if (cloneMap.has(docAtKey[Id])) { + clone[key] = cloneMap.get(docAtKey[Id]); + } + else clone[key] = undefined; } else { - repairClone(docAtKey, cloned, visited); + repairClone(docAtKey, cloneMap, visited); } } }); @@ -789,7 +792,7 @@ export namespace Doc { const repaired = new Set(); const linkedDocs = Array.from(linkMap.values()); const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; - clonedDocs.map(clone => Doc.repairClone(clone, Array.from(cloneMap.values()), repaired)); + clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true)); rtfMap.map(({ copy, key, field }) => { const replacer = (match: any, attr: string, id: string, offset: any, string: any) => { @@ -813,7 +816,7 @@ export namespace Doc { // a.href = url; // a.download = `DocExport-${this.props.Document[Id]}.zip`; // a.click(); - const { clone, map, linkMap } = await Doc.MakeClone(doc, true); + const { clone, map, linkMap } = await Doc.MakeClone(doc, false); clone.LINKS = new List(Array.from(linkMap.values())); function replacer(key: any, value: any) { if (['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; @@ -841,7 +844,7 @@ export namespace Doc { const docs: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const docString = JSON.stringify({ id: doc[Id], docs }, decycle(replacer)); + const docString = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); const zip = new JSZip(); @@ -1527,8 +1530,11 @@ export namespace Doc { formData.append('remap', 'true'); const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); + console.log(json) if (json !== 'error') { - const doc = DocCast(await DocServer.GetRefField(json)); + await DocServer.GetRefFields(json.docids as string[]); + const doc = DocCast(await DocServer.GetRefField(json.id)); + console.log("Doc = ", doc, doc?.title); (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); return doc; } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 9bacbd5c8..5da3dfd3f 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -228,6 +228,7 @@ export default class UploadManager extends ApiManager { form.parse(req, async (_err, fields, files) => { remap = fields.remap !== 'false'; let id: string = ''; + let docids: string[] = []; try { for (const name in files) { const f = files[name]; @@ -257,6 +258,7 @@ export default class UploadManager extends ApiManager { id = getId(data.id); const docs = Object.keys(datadocs).map(key => datadocs[key]); docs.forEach(mapFn); + docids = docs.map(doc => doc.id) await Promise.all( docs.map( (doc: any) => @@ -279,7 +281,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify(id || 'error')); + res.send(JSON.stringify({id, docids} || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From de0df48cba8e89256a3208fbadfd5afaaa9e22d3 Mon Sep 17 00:00:00 2001 From: bobzel Date: Thu, 16 Mar 2023 21:10:23 -0400 Subject: added importing and exporting of assets (images, pdfs, etc) to collection importer/exporter --- src/JSZipUtils.js | 142 ++++++++++++++++++++++++++++++++ src/fields/Doc.ts | 82 ++++++++++++------ src/server/ApiManagers/UploadManager.ts | 28 ++++--- 3 files changed, 216 insertions(+), 36 deletions(-) create mode 100644 src/JSZipUtils.js (limited to 'src/server') diff --git a/src/JSZipUtils.js b/src/JSZipUtils.js new file mode 100644 index 000000000..5ce1bd471 --- /dev/null +++ b/src/JSZipUtils.js @@ -0,0 +1,142 @@ +var JSZipUtils = {}; +// just use the responseText with xhr1, response with xhr2. +// The transformation doesn't throw away high-order byte (with responseText) +// because JSZip handles that case. If not used with JSZip, you may need to +// do it, see https://developer.mozilla.org/En/Using_XMLHttpRequest#Handling_binary_data +JSZipUtils._getBinaryFromXHR = function (xhr) { + // for xhr.responseText, the 0xFF mask is applied by JSZip + return xhr.response || xhr.responseText; +}; + +// taken from jQuery +function createStandardXHR() { + try { + return new window.XMLHttpRequest(); + } catch (e) {} +} + +function createActiveXHR() { + try { + return new window.ActiveXObject('Microsoft.XMLHTTP'); + } catch (e) {} +} + +// Create the request object +var createXHR = + typeof window !== 'undefined' && window.ActiveXObject + ? /* Microsoft failed to properly + * implement the XMLHttpRequest in IE7 (can't request local files), + * so we use the ActiveXObject when it is available + * Additionally XMLHttpRequest can be disabled in IE7/IE8 so + * we need a fallback. + */ + function () { + return createStandardXHR() || createActiveXHR(); + } + : // For all other browsers, use the standard XMLHttpRequest object + createStandardXHR; + +/** + * @param {string} path The path to the resource to GET. + * @param {function|{callback: function, progress: function}} options + * @return {Promise|undefined} If no callback is passed then a promise is returned + */ +JSZipUtils.getBinaryContent = function (path, options) { + var promise, resolve, reject; + var callback; + + if (!options) { + options = {}; + } + + // backward compatible callback + if (typeof options === 'function') { + callback = options; + options = {}; + } else if (typeof options.callback === 'function') { + // callback inside options object + callback = options.callback; + } + + if (!callback && typeof Promise !== 'undefined') { + promise = new Promise(function (_resolve, _reject) { + resolve = _resolve; + reject = _reject; + }); + } else { + resolve = function (data) { + callback(null, data); + }; + reject = function (err) { + callback(err, null); + }; + } + + /* + * Here is the tricky part : getting the data. + * In firefox/chrome/opera/... setting the mimeType to 'text/plain; charset=x-user-defined' + * is enough, the result is in the standard xhr.responseText. + * cf https://developer.mozilla.org/En/XMLHttpRequest/Using_XMLHttpRequest#Receiving_binary_data_in_older_browsers + * In IE <= 9, we must use (the IE only) attribute responseBody + * (for binary data, its content is different from responseText). + * In IE 10, the 'charset=x-user-defined' trick doesn't work, only the + * responseType will work : + * http://msdn.microsoft.com/en-us/library/ie/hh673569%28v=vs.85%29.aspx#Binary_Object_upload_and_download + * + * I'd like to use jQuery to avoid this XHR madness, but it doesn't support + * the responseType attribute : http://bugs.jquery.com/ticket/11461 + */ + try { + var xhr = createXHR(); + + xhr.open('GET', path, true); + + // recent browsers + if ('responseType' in xhr) { + xhr.responseType = 'arraybuffer'; + } + + // older browser + if (xhr.overrideMimeType) { + xhr.overrideMimeType('text/plain; charset=x-user-defined'); + } + + xhr.onreadystatechange = function (event) { + // use `xhr` and not `this`... thanks IE + if (xhr.readyState === 4) { + if (xhr.status === 200 || xhr.status === 0) { + try { + resolve(JSZipUtils._getBinaryFromXHR(xhr)); + } catch (err) { + reject(new Error(err)); + } + } else { + reject(new Error('Ajax error for ' + path + ' : ' + this.status + ' ' + this.statusText)); + } + } + }; + + if (options.progress) { + xhr.onprogress = function (e) { + options.progress({ + path: path, + originalEvent: e, + percent: (e.loaded / e.total) * 100, + loaded: e.loaded, + total: e.total, + }); + }; + } + + xhr.send(); + } catch (e) { + reject(new Error(e), null); + } + + // returns a promise or undefined depending on whether a callback was + // provided + return promise; +}; + +// export +module.exports = JSZipUtils; diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 6543679ad..deda8aa1f 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -26,6 +26,7 @@ import { Cast, DocCast, FieldValue, NumCast, StrCast, ToConstructor } from './Ty import { AudioField, ImageField, MapField, PdfField, VideoField, WebField } from './URLField'; import { deleteProperty, GetEffectiveAcl, getField, getter, makeEditable, makeReadOnly, normalizeEmail, setter, SharingPermissions, updateFunction } from './util'; import JSZip = require('jszip'); +import * as JSZipUtils from '../JSZipUtils'; export namespace Field { export function toKeyValueString(doc: Doc, key: string): string { const onDelegate = Object.keys(doc).includes(key); @@ -732,7 +733,7 @@ export namespace Doc { }; const docAtKey = doc[key]; if (docAtKey instanceof Doc) { - if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || key === 'proto'|| ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || key === 'proto' || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); } else { assignKey(docAtKey); @@ -757,7 +758,7 @@ export namespace Doc { const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch); linkMap.set(link[Id], linkClone); } - Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); + Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); if (!dontCreate) { asBranch ? (copy.branchOf = doc) : (copy.cloneOf = doc); if (!Doc.IsPrototype(copy)) { @@ -768,22 +769,23 @@ export namespace Doc { Doc.AddFileOrphan(copy); return copy; } - export function repairClone(clone: Doc, cloneMap: Map, visited: Set) { + export function repairClone(clone: Doc, cloneMap: Map, visited: Set) { if (visited.has(clone)) return; visited.add(clone); - Object.keys(clone).filter(key => key !== "cloneOf").map(key => { - const docAtKey = DocCast(clone[key]); - if (docAtKey && !Doc.IsSystem(docAtKey)) { - if (!Array.from(cloneMap.values()).includes(docAtKey)) { - if (cloneMap.has(docAtKey[Id])) { - clone[key] = cloneMap.get(docAtKey[Id]); + Object.keys(clone) + .filter(key => key !== 'cloneOf') + .map(key => { + const docAtKey = DocCast(clone[key]); + if (docAtKey && !Doc.IsSystem(docAtKey)) { + if (!Array.from(cloneMap.values()).includes(docAtKey)) { + if (cloneMap.has(docAtKey[Id])) { + clone[key] = cloneMap.get(docAtKey[Id]); + } else clone[key] = undefined; + } else { + repairClone(docAtKey, cloneMap, visited); } - else clone[key] = undefined; - } else { - repairClone(docAtKey, cloneMap, visited); } - } - }); + }); } export async function MakeClone(doc: Doc, dontCreate: boolean = false, asBranch = false, cloneMap: Map = new Map()) { const linkMap = new Map(); @@ -818,8 +820,9 @@ export namespace Doc { // a.click(); const { clone, map, linkMap } = await Doc.MakeClone(doc, false); clone.LINKS = new List(Array.from(linkMap.values())); + const proms = [] as string[]; function replacer(key: any, value: any) { - if (['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; + if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; else if (value instanceof Doc) { if (key !== 'field' && Number.isNaN(Number(key))) { const __fields = value[FieldsSym](); @@ -829,9 +832,14 @@ export namespace Doc { } } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; - else if (value instanceof ImageField) return { url: value.url.href, __type: 'image' }; - else if (value instanceof PdfField) return { url: value.url.href, __type: 'pdf' }; - else if (value instanceof AudioField) return { url: value.url.href, __type: 'audio' }; + else if (value instanceof ImageField) { + const extension = value.url.href.replace(/.*\./, ''); + proms.push(value.url.href.replace('.' + extension, '_o.' + extension)); + return { url: value.url.href, __type: 'image' }; + } else if (value instanceof PdfField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'pdf' }; + } else if (value instanceof AudioField) return { url: value.url.href, __type: 'audio' }; else if (value instanceof VideoField) return { url: value.url.href, __type: 'video' }; else if (value instanceof WebField) return { url: value.url.href, __type: 'web' }; else if (value instanceof MapField) return { url: value.url.href, __type: 'map' }; @@ -846,6 +854,32 @@ export namespace Doc { Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); const docString = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); + let generateZIP = (proms: string[]) => { + var zip = new JSZip(); + var count = 0; + var zipFilename = 'dashExport.zip'; + + proms + .filter(url => url.startsWith(window.location.origin)) + .forEach((url, i) => { + var filename = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); + // loading a file and add it in a zip file + JSZipUtils.getBinaryContent(url, function (err: any, data: any) { + if (err) { + throw err; // or handle the error + } + zip.file(filename, data, { binary: true }); + count++; + if (count == proms.length) { + zip.file('doc.json', docString); + zip.generateAsync({ type: 'blob' }).then(function (content) { + saveAs(content, zipFilename); + }); + } + }); + }); + }; + generateZIP(proms); const zip = new JSZip(); zip.file('doc.json', docString); @@ -857,10 +891,10 @@ export namespace Doc { // img.file("smile.gif", imgData, {base64: true}); // Generate the zip file asynchronously - zip.generateAsync({ type: 'blob' }).then((content: any) => { - // Force down of the Zip file - saveAs(content, doc.title + '.zip'); // glr: Possibly change the name of the document to match the title? - }); + // zip.generateAsync({ type: 'blob' }).then((content: any) => { + // // Force down of the Zip file + // saveAs(content, doc.title + '.zip'); // glr: Possibly change the name of the document to match the title? + // }); } // // Determines whether the layout needs to be expanded (as a template). @@ -1530,12 +1564,12 @@ export namespace Doc { formData.append('remap', 'true'); const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); - console.log(json) + console.log(json); if (json !== 'error') { await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); - console.log("Doc = ", doc, doc?.title); (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); + doc.LINKS = undefined; return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 5da3dfd3f..6e28268a9 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -235,18 +235,22 @@ export default class UploadManager extends ApiManager { const path_2 = Array.isArray(f) ? '' : f.path; const zip = new AdmZip(path_2); zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith('files/')) return; - let directory = dirname(entry.entryName) + '/'; - const extension = extname(entry.entryName); - const base = basename(entry.entryName).split('.')[0]; + let entryName = entry.entryName.replace(/%%%/g, '/'); + if (!entryName.startsWith('files/')) { + return; + } + const extension = extname(entryName); + const pathname = publicDirectory + '/' + entry.entryName; + const targetname = publicDirectory + '/' + entryName; try { zip.extractEntryTo(entry.entryName, publicDirectory, true, false); - directory = '/' + directory; - - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_o' + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_s' + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_m' + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_l' + extension)); + createReadStream(pathname).pipe(createWriteStream(targetname)); + if (extension !== '.pdf') { + createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_s' + extension))); + createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_m' + extension))); + createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_l' + extension))); + } + unlink(pathname, () => {}); } catch (e) { console.log(e); } @@ -258,7 +262,7 @@ export default class UploadManager extends ApiManager { id = getId(data.id); const docs = Object.keys(datadocs).map(key => datadocs[key]); docs.forEach(mapFn); - docids = docs.map(doc => doc.id) + docids = docs.map(doc => doc.id); await Promise.all( docs.map( (doc: any) => @@ -281,7 +285,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify({id, docids} || 'error')); + res.send(JSON.stringify({ id, docids } || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From d51a57e1d0823fc08a90b73d427ab5e35b72422f Mon Sep 17 00:00:00 2001 From: Mohammad Amoush <51237606+jameshu111@users.noreply.github.com> Date: Sat, 18 Mar 2023 16:56:13 -0400 Subject: Add operations counter --- src/server/Client.ts | 13 +++- src/server/DashStats.ts | 119 ++++++++++++++++++++++++++++++------ src/server/index.ts | 6 ++ src/server/stats/userLoginStats.csv | 77 ++++++++++++++++++++++- src/server/websocket.ts | 24 ++++++-- views/stats.pug | 22 +++++++ views/stylesheets/statsview.css | 56 +++++++++++++++++ 7 files changed, 290 insertions(+), 27 deletions(-) create mode 100644 views/stats.pug create mode 100644 views/stylesheets/statsview.css (limited to 'src/server') diff --git a/src/server/Client.ts b/src/server/Client.ts index e6f953712..be1ffc2ba 100644 --- a/src/server/Client.ts +++ b/src/server/Client.ts @@ -1,11 +1,22 @@ -import { computed } from "mobx"; +import { action, computed } from "mobx"; export class Client { private _guid: string; + // private operations: number; constructor(guid: string) { this._guid = guid; + // this.operations = 0; } + // @computed public get OPERATIONS(): number { + // return this.operations; + // } + + // @action + // public setOperations(newOperations: number): void { + // this.operations = newOperations; + // } + @computed public get GUID(): string { return this._guid; } } \ No newline at end of file diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts index 13ee61b3a..a10b28608 100644 --- a/src/server/DashStats.ts +++ b/src/server/DashStats.ts @@ -1,11 +1,11 @@ -import { magenta } from 'colors'; +import { cyan, magenta } from 'colors'; import { Request, Response } from 'express'; +import { Server } from 'http'; import SocketIO from 'socket.io'; -import { Client } from './Client'; +import { timeMap } from './ApiManagers/UserManager'; import { WebSocket } from './websocket'; const fs = require('fs'); const csv = require('csv-parser'); -import { stringify } from 'csv-stringify/sync'; export namespace DashStats { const statsCSVFilename = './src/server/stats/userLoginStats.csv'; @@ -14,6 +14,8 @@ export namespace DashStats { interface SocketPair { socketId: string; username: string; + time: string; + operations: number; } interface CSVStore { @@ -22,49 +24,128 @@ export namespace DashStats { TIME: string; } - export function handleStatsView(res: Response) { + enum ServerTraffic { + NOT_BUSY, + BUSY, + VERY_BUSY + } + + const BUSY_SERVER_BOUND = 2; + const VERY_BUSY_SERVER_BOUND = 3; + + const serverTrafficMessages = [ + "Not Busy", + "Busy", + "Very Busy" + ] + + export function handleStats(res: Response) { + let current = getCurrentConnections(); const results: CSVStore[] = []; - fs.createReadStream(statsCSVFilename) - .pipe(csv(columns)) - .on('data', (data: any) => results.push(data)) - .on('end', () => { - console.log(results); + res.json({ + message: 'welcome to stats', + currentConnections: current.length, + socketMap: current, }); - // let newRow = stringify([{ USERNAME: 'hi', ACTION: 'hi', TIME: 'hi' }], { header: true, columns: columns }); - // console.log(newRow); + // fs.createReadStream(statsCSVFilename) + // .pipe(csv()) + // .on('data', (data: any) => results.push(data)) + // .on('end', () => { + // console.log(results); + // res.json({ + // message: 'welcome to stats', + // currentConnections: current.length, + // socketMap: current, + // results: results, + // }); + // }); + } + export function handleStatsView(res: Response) { let current = getCurrentConnections(); - res.json({ - message: 'welcome to stats', - currentConnections: current.length, - socketMap: current, + + let connectedUsers = current.map((socketPair) => { + return socketPair.time + " - " + socketPair.username + " Operations: " + socketPair.operations; + }) + + let serverTraffic = ServerTraffic.NOT_BUSY; + if(current.length < BUSY_SERVER_BOUND) { + serverTraffic = ServerTraffic.NOT_BUSY; + } else if(current.length >= BUSY_SERVER_BOUND && current.length < VERY_BUSY_SERVER_BOUND) { + serverTraffic = ServerTraffic.BUSY; + } else { + serverTraffic = ServerTraffic.VERY_BUSY; + } + + res.render("stats.pug", { + title: "Dash Stats", + numConnections: current.length, + serverTraffic: serverTraffic, + serverTrafficMessage : serverTrafficMessages[serverTraffic], + connectedUsers: connectedUsers }); } export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); - console.log('stringify -> '); + // console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + + let toWrite: CSVStore = { + USERNAME : username, + ACTION : "loggedIn", + TIME : currentDate.toISOString() + } + + let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"}); + statsFile.write(convertToCSV(toWrite)); + statsFile.end(); + console.log(cyan(convertToCSV(toWrite))); } } export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); + // console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); + + let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"}); + let toWrite: CSVStore = { + USERNAME : username, + ACTION : "loggedOut", + TIME : currentDate.toISOString() + } + statsFile.write(convertToCSV(toWrite)); + statsFile.end(); } } function getCurrentConnections(): SocketPair[] { + console.log("timeMap: " + timeMap); + console.log("clients:" + WebSocket.clients); let socketPairs: SocketPair[] = []; for (let [key, value] of WebSocket.socketMap) { + let username = value.split(' ')[0]; + let connectionTime = new Date(timeMap[username]); + + let connectionTimeString = connectionTime.toLocaleDateString() + " " + connectionTime.toLocaleTimeString(); + if (!key.disconnected) { - socketPairs.push({ socketId: key.id, username: value.split(' ')[0] }); + socketPairs.push({ + socketId: key.id, + username: username, + time: connectionTimeString.includes("Invalid Date") ? "" : connectionTimeString, + operations : WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0, + }); } } console.log(socketPairs); + // console.log([...WebSocket.clients.entries()]); return socketPairs; } + + function convertToCSV(dataObject: CSVStore): string { + return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`; + } } diff --git a/src/server/index.ts b/src/server/index.ts index 0848d828e..d76f12b95 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -87,6 +87,12 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: addSupervisedRoute({ method: Method.GET, subscription: '/stats', + secureHandler: ({ res }) => DashStats.handleStats(res), + }); + + addSupervisedRoute({ + method: Method.GET, + subscription: '/statsview', secureHandler: ({ res }) => DashStats.handleStatsView(res), }); diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv index ec33b01b2..d5e56502c 100644 --- a/src/server/stats/userLoginStats.csv +++ b/src/server/stats/userLoginStats.csv @@ -1,2 +1,75 @@ -USERNAME,ACTION,TIME -boo15869,loggedIn,2023-01 \ No newline at end of file +USER,ACTION,TIME +aaa@gmail.com,loggedIn,2023-03-18T20:10:29.928Z +guest,loggedIn,2023-03-18T20:10:47.384Z +aaa@gmail.com,loggedOut,2023-03-18T20:10:55.364Z +aaa@gmail.com,loggedIn,2023-03-18T20:11:15.879Z +guest,loggedIn,2023-03-18T20:18:03.724Z +aaa@gmail.com,loggedIn,2023-03-18T20:18:03.860Z +guest,loggedOut,2023-03-18T20:19:00.211Z +guest,loggedIn,2023-03-18T20:19:03.087Z +guest,loggedOut,2023-03-18T20:19:50.668Z +boo15869@gmail.com,loggedIn,2023-03-18T20:20:17.890Z +boo15869@gmail.com,loggedOut,2023-03-18T20:21:03.542Z +boo15869@gmail.com,loggedIn,2023-03-18T20:21:06.149Z +boo15869@gmail.com,loggedOut,2023-03-18T20:21:51.874Z +a@gmail.com,loggedIn,2023-03-18T20:22:02.122Z +aaa@gmail.com,loggedOut,2023-03-18T20:22:42.882Z +aaa@gmail.com,loggedIn,2023-03-18T20:22:45.631Z +aaa@gmail.com,loggedIn,2023-03-18T20:25:34.658Z +a@gmail.com,loggedIn,2023-03-18T20:25:34.681Z +aaa@gmail.com,loggedIn,2023-03-18T20:29:04.297Z +a@gmail.com,loggedIn,2023-03-18T20:29:08.701Z +a@gmail.com,loggedIn,2023-03-18T20:29:18.565Z +aaa@gmail.com,loggedIn,2023-03-18T20:29:21.974Z +aaa@gmail.com,loggedIn,2023-03-18T20:29:51.477Z +a@gmail.com,loggedIn,2023-03-18T20:29:51.489Z +aaa@gmail.com,loggedIn,2023-03-18T20:30:15.011Z +aaa@gmail.com,loggedIn,2023-03-18T20:30:57.818Z +a@gmail.com,loggedIn,2023-03-18T20:30:57.838Z +aaa@gmail.com,loggedIn,2023-03-18T20:31:12.061Z +a@gmail.com,loggedIn,2023-03-18T20:31:12.080Z +a@gmail.com,loggedIn,2023-03-18T20:31:19.447Z +aaa@gmail.com,loggedIn,2023-03-18T20:31:22.738Z +aaa@gmail.com,loggedIn,2023-03-18T20:32:36.919Z +a@gmail.com,loggedIn,2023-03-18T20:32:36.929Z +a@gmail.com,loggedIn,2023-03-18T20:32:56.212Z +aaa@gmail.com,loggedIn,2023-03-18T20:32:59.300Z +aaa@gmail.com,loggedIn,2023-03-18T20:34:27.543Z +a@gmail.com,loggedIn,2023-03-18T20:34:27.570Z +a@gmail.com,loggedIn,2023-03-18T20:34:35.299Z +aaa@gmail.com,loggedIn,2023-03-18T20:34:35.302Z +a@gmail.com,loggedIn,2023-03-18T20:34:51.579Z +aaa@gmail.com,loggedIn,2023-03-18T20:34:52.392Z +a@gmail.com,loggedIn,2023-03-18T20:35:08.509Z +aaa@gmail.com,loggedIn,2023-03-18T20:35:15.202Z +a@gmail.com,loggedIn,2023-03-18T20:36:46.796Z +aaa@gmail.com,loggedIn,2023-03-18T20:36:51.756Z +a@gmail.com,loggedIn,2023-03-18T20:36:55.286Z +a@gmail.com,loggedIn,2023-03-18T20:40:06.226Z +a@gmail.com,loggedIn,2023-03-18T20:40:18.474Z +aaa@gmail.com,loggedIn,2023-03-18T20:40:31.894Z +a@gmail.com,loggedIn,2023-03-18T20:40:31.903Z +a@gmail.com,loggedIn,2023-03-18T20:42:25.301Z +aaa@gmail.com,loggedIn,2023-03-18T20:42:31.182Z +aaa@gmail.com,loggedOut,2023-03-18T20:43:05.741Z +aaa@gmail.com,loggedIn,2023-03-18T20:43:09.203Z +a@gmail.com,loggedOut,2023-03-18T20:45:24.343Z +a@gmail.com,loggedIn,2023-03-18T20:45:27.207Z +aaa@gmail.com,loggedIn,2023-03-18T20:46:15.618Z +a@gmail.com,loggedIn,2023-03-18T20:46:56.163Z +a@gmail.com,loggedOut,2023-03-18T20:48:37.464Z +a@gmail.com,loggedIn,2023-03-18T20:48:40.562Z +a@gmail.com,loggedOut,2023-03-18T20:50:32.478Z +a@gmail.com,loggedIn,2023-03-18T20:50:40.384Z +a@gmail.com,loggedOut,2023-03-18T20:51:34.159Z +a@gmail.com,loggedIn,2023-03-18T20:51:49.206Z +a@gmail.com,loggedOut,2023-03-18T20:52:04.673Z +qw@gmail.com,loggedIn,2023-03-18T20:52:36.270Z +qw@gmail.com,loggedIn,2023-03-18T20:53:58.175Z +aaa@gmail.com,loggedIn,2023-03-18T20:53:58.204Z +aaa@gmail.com,loggedIn,2023-03-18T20:54:20.518Z +qw@gmail.com,loggedIn,2023-03-18T20:54:24.225Z +qw@gmail.com,loggedIn,2023-03-18T20:54:37.007Z +aaa@gmail.com,loggedIn,2023-03-18T20:54:39.959Z +aaa@gmail.com,loggedOut,2023-03-18T20:55:55.199Z +qw@gmail.com,loggedOut,2023-03-18T20:55:59.010Z diff --git a/src/server/websocket.ts b/src/server/websocket.ts index e556ecc17..54944e944 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -1,4 +1,4 @@ -import { blue, magenta } from 'colors'; +import { blue, magenta, yellow } from 'colors'; import * as express from 'express'; import { createServer, Server } from 'https'; import { networkInterfaces } from 'os'; @@ -22,7 +22,9 @@ import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { export let _socket: Socket; export const clients: { [key: string]: Client } = {}; + // export const clients = new Map(); export const socketMap = new Map(); + export const userOperations = new Map(); export let disconnect: Function; export async function initialize(isRelease: boolean, app: express.Express) { @@ -101,7 +103,9 @@ export namespace WebSocket { socket.on('disconnect', function () { let currentUser = socketMap.get(socket); if (!(currentUser === undefined)) { - DashStats.logUserLogout(socketMap.get(socket), socket); + let currentUsername = currentUser.split(' ')[0] + DashStats.logUserLogout(currentUsername, socket); + delete timeMap[currentUsername] } }); @@ -180,16 +184,20 @@ export namespace WebSocket { } function barReceived(socket: SocketIO.Socket, userEmail: string) { - clients[userEmail] = new Client(userEmail.toString()); + clients[userEmail] = new Client(userEmail.toString()); + // clients.set(userEmail, new Client(userEmail.toString())); const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); - console.log(magenta(`currently connected: ${JSON.stringify(clients)}`)); + // console.log(magenta(`currently connected: ${[...clients.entries()]}`)); // console.log(magenta('socket map below')); // console.log([...socketMap.entries()]); printActiveUsers(); + + timeMap[userEmail] = Date.now(); socketMap.set(socket, userEmail + ' at ' + datetime); - DashStats.logUserLogin(socketMap.get(socket), socket); + userOperations.set(userEmail, 0); + DashStats.logUserLogin(userEmail, socket); } function getField([id, callback]: [string, (result?: Transferable) => void]) { @@ -359,6 +367,12 @@ export namespace WebSocket { var CurUser: string | undefined = undefined; function UpdateField(socket: Socket, diff: Diff) { + console.log(magenta(`1 OP ${socketMap.get(socket)}`)); + + let currentUsername = socketMap.get(socket)!.split(' ')[0]; + userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0); + console.log(yellow("Total Operations: " + userOperations.get(currentUsername))); + if (CurUser !== socketMap.get(socket)) { CurUser = socketMap.get(socket); console.log('Switch User: ' + CurUser); diff --git a/views/stats.pug b/views/stats.pug new file mode 100644 index 000000000..54c017e70 --- /dev/null +++ b/views/stats.pug @@ -0,0 +1,22 @@ +extends ./layout + +block content + style + include ./stylesheets/authentication.css + include ./stylesheets/statsview.css + .outermost + .stats-container + h1 Dash Stats + p(class="stats-content") Current Connections: #{numConnections} + div(class="stats-content stats-server-status-container") + p Server Status: + div(class="stats-server-status-item stats-server-status-" + serverTraffic) + p #{serverTrafficMessage} + div(class="stats-content stats-connected-users") + p Connected Users: + ul + each username in connectedUsers + li(class="none")= username + + + \ No newline at end of file diff --git a/views/stylesheets/statsview.css b/views/stylesheets/statsview.css new file mode 100644 index 000000000..c018bedfc --- /dev/null +++ b/views/stylesheets/statsview.css @@ -0,0 +1,56 @@ +.outermost { + background-color: #251f1f; + display: flex; + flex-direction: row; + height: 98vh; + width: 99vw; + justify-content: center; + position: relative; +} + +.stats-container { + background-color: white; + + padding: 1rem; + width: 80vw; + border-radius: 8px; +} + +.stats-content { + font-size: 1.25em; + +} + +.stats-server-status-container { + display: flex; + flex-direction: row; +} + +.stats-server-status-item { + margin-left: 0.25rem; + padding: 0px 5px; + + border-radius: 3px; + width: 8rem; + text-align: center; +} + +.stats-server-status-0 { + /* not busy */ + border: 3px green solid; +} + +.stats-server-status-1 { + /* busy */ + border: 3px #ffcc00 solid; +} + +.stats-server-status-2 { + /* very busy */ + border: 3px red solid; +} + +.stats-connected-users { + max-height: 70vh; + overflow-y: auto; +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 99fba6d6e99da0154d40d2e3e87e120d8e6f2810 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 5 Apr 2023 22:44:38 -0400 Subject: fixed up dataviz to work again. fixed point selection, tooltips, making and following links --- src/client/documents/Documents.ts | 4 +- src/client/util/CurrentUserUtils.ts | 2 +- src/client/views/nodes/DataVizBox/ChartBox.tsx | 62 ++++----------------- .../views/nodes/DataVizBox/ChartInterface.ts | 6 +-- src/client/views/nodes/DataVizBox/DataVizBox.scss | 4 ++ src/client/views/nodes/DataVizBox/DataVizBox.tsx | 63 ++++++++++------------ .../views/nodes/DataVizBox/components/Chart.scss | 9 +++- .../nodes/DataVizBox/components/LineChart.tsx | 41 ++++++++------ src/client/views/nodes/DocumentView.tsx | 1 + src/client/views/nodes/trails/PresBox.tsx | 9 +++- src/server/DataVizUtils.ts | 22 ++++---- 11 files changed, 98 insertions(+), 125 deletions(-) (limited to 'src/server') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index c4014a752..1f09cdd3d 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -1148,8 +1148,8 @@ export namespace Docs { return Prototypes.get(DocumentType.PRESELEMENT); } - export function DataVizDocument(url: string, options?: DocumentOptions) { - return InstanceFromProto(Prototypes.get(DocumentType.DATAVIZ), new CsvField(url), { title: 'Data Viz', ...options }); + export function DataVizDocument(url: string, options?: DocumentOptions, overwriteDoc?: Doc) { + return InstanceFromProto(Prototypes.get(DocumentType.DATAVIZ), new CsvField(url), { title: 'Data Viz', ...options }, undefined, undefined, undefined, overwriteDoc); } export function DockDocument(documents: Array, config: string, options: DocumentOptions, id?: string) { diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 9aceed366..623aaf357 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -273,7 +273,7 @@ export class CurrentUserUtils { {key: "WebCam", creator: opts => Docs.Create.WebCamDocument("", opts), opts: { _width: 400, _height: 200, recording:true, system: true, cloneFieldFilter: new List(["system"]) }}, {key: "Button", creator: Docs.Create.ButtonDocument, opts: { _width: 150, _height: 50, _xPadding: 10, _yPadding: 10, _isLinkButton: true }}, {key: "Script", creator: opts => Docs.Create.ScriptingDocument(null, opts), opts: { _width: 200, _height: 250, }}, - // {key: "DataViz", creator: opts => Docs.Create.DataVizDocument(opts), opts: { _width: 300, _height: 300 }}, + {key: "DataViz", creator: opts => Docs.Create.DataVizDocument("/users/rz/Downloads/addresses.csv", opts), opts: { _width: 300, _height: 300 }}, {key: "Header", creator: headerTemplate, opts: { _width: 300, _height: 70, _headerPointerEvents: "all", _headerHeight: 12, _headerFontSize: 9, _autoHeight: true, treeViewHideUnrendered: true}}, {key: "Trail", creator: Docs.Create.PresDocument, opts: { _width: 400, _height: 30, _viewType: CollectionViewType.Stacking, targetDropAction: "alias" as any, treeViewHideTitle: true, _fitWidth:true, _chromeHidden: true, boxShadow: "0 0" }}, {key: "Tab", creator: opts => Docs.Create.FreeformDocument([], opts), opts: { _width: 500, _height: 800, _fitWidth: true, _backgroundGridShow: true, }}, diff --git a/src/client/views/nodes/DataVizBox/ChartBox.tsx b/src/client/views/nodes/DataVizBox/ChartBox.tsx index c229e5471..ee577b9fd 100644 --- a/src/client/views/nodes/DataVizBox/ChartBox.tsx +++ b/src/client/views/nodes/DataVizBox/ChartBox.tsx @@ -5,10 +5,12 @@ import { action, computed, observable } from 'mobx'; import { NumCast, StrCast } from '../../../../fields/Types'; import { LineChart } from './components/LineChart'; import { Chart, ChartData } from './ChartInterface'; +import { PinProps } from '../trails'; export interface ChartBoxProps { rootDoc: Doc; dataDoc: Doc; + height: number; pairs: { x: number; y: number }[]; setChartBox: (chartBox: ChartBox) => void; getAnchor: () => Doc; @@ -71,11 +73,7 @@ export class ChartBox extends React.Component { componentDidMount = () => { this.generateChartData(); - // setting timeout to allow rerender cycle of the actual chart tof inish - // setTimeout(() => { this.props.setChartBox(this); - // }); - // this.props.setChartBox(this); }; @action @@ -86,59 +84,17 @@ export class ChartBox extends React.Component { this.props.rootDoc._currChartView = e.currentTarget.value.toLowerCase(); }; - scrollFocus(doc: Doc, smooth: boolean): number | undefined { - // if x and y exist on this - // then highlight/focus the x and y position (datapoint) - // highlight for a few seconds and then remove (set a timer to unhighlight after a period of time, - // to be consistent, use 5 seconds and highlight color is orange) - // if x and y don't exist => link to whole doc => dash will take care of focusing on the entire doc - // return value is how long this action takes - // undefined => immediately, 0 => introduces a timeout - // number in ms => won't do any of the focus or call the automated highlighting thing until this timeout expires - // in this case probably number in ms doesn't matter + getAnchor = (pinProps?: PinProps) => this.currChart?._getAnchor(pinProps); - // for now could just update the currSelected in linechart to be what doc.x and doc.y - - if (this.currChart && doc.x && doc.y) { - // update curr selected - this.currChart.setCurrSelected(Number(doc.x), Number(doc.y)); - } - return 0; - } - - _getAnchor() { - return this.currChart?._getAnchor(); - } + restoreView = (data: Doc) => this.currChart?.restoreView(data); render() { if (this.props.pairs && this._chartData) { - let width = NumCast(this.props.rootDoc._width); - width = width * 0.7; - let height = NumCast(this.props.rootDoc._height); - height = height * 0.7; - console.log(width, height); - const margin = { top: 50, right: 50, bottom: 50, left: 50 }; - return ( -
-
- {/*{this.props.rootDoc._currChartView == 'line' ? ( - this.onClick(e)} /> - ) : ( - this.onClick(e)} /> - )} */} - {/* {this.reLineChart} */} - -
- - -
- ); - } else { - return
; + const width = NumCast(this.props.rootDoc._width) * 0.9; + const height = this.props.height * 0.9; + const margin = { top: 10, right: 50, bottom: 50, left: 50 }; + return ; } + return
; } } diff --git a/src/client/views/nodes/DataVizBox/ChartInterface.ts b/src/client/views/nodes/DataVizBox/ChartInterface.ts index 494242ac5..8ebcc2a5f 100644 --- a/src/client/views/nodes/DataVizBox/ChartInterface.ts +++ b/src/client/views/nodes/DataVizBox/ChartInterface.ts @@ -1,15 +1,15 @@ import { Doc } from '../../../../fields/Doc'; +import { PinProps } from '../trails'; import { DataPoint } from './ChartBox'; -import { LineChart } from './components/LineChart'; export interface Chart { tooltipContent: (data: DataPoint) => string; drawChart: () => void; + restoreView: (data: any) => boolean; height: number; width: number; // TODO: nda - probably want to get rid of this to keep charts more generic - _getAnchor: () => Doc; - setCurrSelected: (x: number, y: number) => void; + _getAnchor: (pinProps?: PinProps) => Doc; } export interface ChartProps { diff --git a/src/client/views/nodes/DataVizBox/DataVizBox.scss b/src/client/views/nodes/DataVizBox/DataVizBox.scss index e69de29bb..cd500e9ae 100644 --- a/src/client/views/nodes/DataVizBox/DataVizBox.scss +++ b/src/client/views/nodes/DataVizBox/DataVizBox.scss @@ -0,0 +1,4 @@ +.dataviz { + overflow: auto; + height: 100%; +} diff --git a/src/client/views/nodes/DataVizBox/DataVizBox.tsx b/src/client/views/nodes/DataVizBox/DataVizBox.tsx index db677ff61..68766e8dc 100644 --- a/src/client/views/nodes/DataVizBox/DataVizBox.tsx +++ b/src/client/views/nodes/DataVizBox/DataVizBox.tsx @@ -2,7 +2,7 @@ import { action, computed, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { Doc } from '../../../../fields/Doc'; -import { Cast, StrCast } from '../../../../fields/Types'; +import { BoolCast, Cast, StrCast } from '../../../../fields/Types'; import { CsvField } from '../../../../fields/URLField'; import { Docs } from '../../../documents/Documents'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; @@ -10,6 +10,7 @@ import { FieldViewProps, FieldView } from '../FieldView'; import { ChartBox } from './ChartBox'; import './DataVizBox.scss'; import { TableBox } from './components/TableBox'; +import { PinProps } from '../trails'; enum DataVizView { TABLE = 'table', @@ -54,40 +55,23 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { }; @computed get currView() { - if (this.rootDoc._dataVizView) { - return StrCast(this.rootDoc._dataVizView); - } else { - return 'table'; - } + return StrCast(this.rootDoc._dataVizView, 'table'); } - scrollFocus = (doc: Doc, smooth: boolean) => { - // reconstruct the view based on the anchor -> - // keep track of the datavizbox view and the type of chart we're rendering - - // use dataview to restore part of it and then pass on the rest to the chartbox - - // pseudocode - return this._chartBox?.scrollFocus(doc, smooth); + @action + restoreView = (data: Doc) => { + const changed = this.currView !== data.dataView && (this.rootDoc._dataVizView = data.dataView); + const func = () => this._chartBox?.restoreView(data); + if (changed) { + setTimeout(func); + return changed ? true : false; + } + return func() ?? false; }; - getAnchor = () => { - // TODO: nda - look into DocumentButtonBar and DocumentLinksButton - - /* - if nothing selected: - return this.rootDoc - this part does not specify view type (doesn't change type when following the link) - - // this slightly diff than the stuff below: - below part returns an anchor that specifies the viewtype for the whole doc and nothing else - - */ - - // store whatever information would allow me to reselect the same thing (store parameters I would pass to get the exact same element) + getAnchor = (addAsAnnotation?: boolean, pinProps?: PinProps) => { const anchor = - // this._COMPONENTNAME._getAnchor() ?? - this._chartBox?._getAnchor() ?? + this._chartBox?.getAnchor(pinProps) ?? Docs.Create.TextanchorDocument({ // when we clear selection -> we should have it so chartBox getAnchor returns undefined // this is for when we want the whole doc (so when the chartBox getAnchor returns without a marker) @@ -98,7 +82,6 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { this.addDocument(anchor); return anchor; - // have some other function in code that }; constructor(props: any) { super(props); @@ -117,7 +100,7 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { case 'table': return ; case 'histogram': - return ; + return ; // case "histogram": // return () } @@ -152,7 +135,7 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { @action changeViewHandler(e: React.MouseEvent) { e.preventDefault(); e.stopPropagation(); - this.rootDoc._dataVizView = this.currView == 'table' ? 'histogram' : 'table'; + this.rootDoc._dataVizView = this.currView === 'table' ? 'histogram' : 'table'; } render() { @@ -161,7 +144,19 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { } return ( -
+
e.stopPropagation()} + ref={r => + r?.addEventListener( + 'wheel', // if scrollTop is 0, then don't let wheel trigger scroll on any container (which it would since onScroll won't be triggered on this) + (e: WheelEvent) => { + if (!r.scrollTop && e.deltaY <= 0) e.preventDefault(); + e.stopPropagation(); + }, + { passive: false } + ) + }> {this.selectView}
diff --git a/src/client/views/nodes/DataVizBox/components/Chart.scss b/src/client/views/nodes/DataVizBox/components/Chart.scss index 7792a2758..2d6c5f0f2 100644 --- a/src/client/views/nodes/DataVizBox/components/Chart.scss +++ b/src/client/views/nodes/DataVizBox/components/Chart.scss @@ -1,10 +1,15 @@ .tooltip { // make the height width bigger - width: 50px; - height: 50px; + width: fit-content; + height: fit-content; } .highlight { // change the color of the circle element to be red fill: red; } +.chart-container { + display: flex; + flex-direction: column; + align-items: center; +} diff --git a/src/client/views/nodes/DataVizBox/components/LineChart.tsx b/src/client/views/nodes/DataVizBox/components/LineChart.tsx index d893b3e12..e5f7dc4f4 100644 --- a/src/client/views/nodes/DataVizBox/components/LineChart.tsx +++ b/src/client/views/nodes/DataVizBox/components/LineChart.tsx @@ -9,6 +9,10 @@ import { Docs } from '../../../../documents/Documents'; import { Doc, DocListCast } from '../../../../../fields/Doc'; import { Chart, ChartProps } from '../ChartInterface'; import './Chart.scss'; +import { List } from '../../../../../fields/List'; +import { PinProps, PresBox } from '../../trails'; +import { listSpec } from '../../../../../fields/Schema'; +import { Cast } from '../../../../../fields/Types'; type minMaxRange = { xMin: number | undefined; @@ -103,21 +107,25 @@ export class LineChart extends React.Component implements Chart { // loop through and remove any annotations that no longer exist } - _getAnchor() { + restoreView = (data: Doc) => { + const coords = Cast(data.presDataViz, listSpec('number'), null); + if ((coords && this._currSelected?.x !== coords[0]) || this._currSelected?.y !== coords[1]) { + this.setCurrSelected(coords[0], coords[1]); + return true; + } + return false; + }; + _getAnchor = (pinProps?: PinProps) => { // store whatever information would allow me to reselect the same thing (store parameters I would pass to get the exact same element) - // TODO: nda - look at pdfviewer get anchor for args - const doc = Docs.Create.TextanchorDocument({ + const anchor = Docs.Create.TextanchorDocument({ /*put in some options*/ title: 'line doc selection' + this._currSelected?.x, }); - // access curr selected from the charts - doc.x = this._currSelected?.x; - doc.y = this._currSelected?.y; - doc.chartType = 'line'; - return doc; + PresBox.pinDocView(anchor, { pinDocLayout: pinProps?.pinDocLayout, pinData: { ...(pinProps?.pinData ?? {}), dataviz: this._currSelected ? [this._currSelected.x, this._currSelected.y] : undefined } }, this.props.dataDoc); + return anchor; // have some other function in code that - } + }; componentWillUnmount() { if (this._dataReactionDisposer) { @@ -132,7 +140,7 @@ export class LineChart extends React.Component implements Chart { } tooltipContent(data: DataPoint) { - return `x: ${data.x} y: ${data.y}`; + return `(${data.x},${data.y})`; } @computed get height(): number { @@ -235,7 +243,7 @@ export class LineChart extends React.Component implements Chart { const mousemove = action((e: any) => { const bisect = d3.bisector((d: DataPoint) => d.x).left; const xPos = d3.pointer(e)[0]; - const x0 = bisect(data, xScale.invert(xPos)); + const x0 = bisect(data, xScale.invert(xPos - 25)); const d0 = data[x0]; this._x = d0.x; this._y = d0.y; @@ -243,23 +251,22 @@ export class LineChart extends React.Component implements Chart { // TODO: nda - implement tooltips tooltip.transition().duration(300).style('opacity', 0.9); // TODO: nda - updating the inner html could be deadly cause injection attacks! - tooltip.html(() => this.tooltipContent(d0)).style('transform', `translate(${xScale(d0.x) - (this.width + margin.left + margin.right) + 30}px,${yScale(d0.y) + 30}px)`); + tooltip + .html(() => this.tooltipContent(d0)) + .style('pointer-events', 'none') + .style('transform', `translate(${xScale(d0.x) - this.width / 2 + 12}px,${yScale(d0.y) + 30}px)`); }); const onPointClick = action((e: any) => { const bisect = d3.bisector((d: DataPoint) => d.x).left; const xPos = d3.pointer(e)[0]; - const x0 = bisect(data, xScale.invert(xPos)); + const x0 = bisect(data, xScale.invert(xPos - 25)); const d0 = data[x0]; this._x = d0.x; this._y = d0.y; // find .circle-d1 with data-x = d0.x and data-y = d0.y const selected = svg.selectAll('.datapoint').filter((d: any) => d['data-x'] === d0.x && d['data-y'] === d0.y); this._currSelected = { x: d0.x, y: d0.y, elem: selected }; - console.log('Getting here'); - // this.drawAnnotations(this._x, this._y); - // this.props.getAnchor(); - console.log(this._currSelected); }); this._chartSvg diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 0e0e22b84..e24bf35f5 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -108,6 +108,7 @@ export type StyleProviderFunc = (doc: Opt, props: Opt, p export interface DocComponentView { updateIcon?: () => void; // updates the icon representation of the document getAnchor?: (addAsAnnotation: boolean, pinData?: PinProps) => Doc; // returns an Anchor Doc that represents the current state of the doc's componentview (e.g., the current playhead location of a an audio/video box) + restoreView?: (viewSpec: Doc) => boolean; scrollPreview?: (docView: DocumentView, doc: Doc, focusSpeed: number, options: DocFocusOptions) => Opt; // returns the duration of the focus brushView?: (view: { width: number; height: number; panX: number; panY: number }) => void; getView?: (doc: Doc) => Promise>; // returns a nested DocumentView for the specified doc or undefined diff --git a/src/client/views/nodes/trails/PresBox.tsx b/src/client/views/nodes/trails/PresBox.tsx index bfaae8069..0e22ea3b1 100644 --- a/src/client/views/nodes/trails/PresBox.tsx +++ b/src/client/views/nodes/trails/PresBox.tsx @@ -3,7 +3,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@material-ui/core'; import { action, computed, IReactionDisposer, observable, ObservableSet, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; -import { AnimationSym, Doc, DocListCast, FieldResult, Opt, StrListCast } from '../../../../fields/Doc'; +import { AnimationSym, Doc, DocListCast, Field, FieldResult, Opt, StrListCast } from '../../../../fields/Doc'; import { Copy, Id } from '../../../../fields/FieldSymbols'; import { InkField, InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; @@ -39,6 +39,7 @@ const { Howl } = require('howler'); export interface pinDataTypes { scrollable?: boolean; + dataviz?: number[]; pannable?: boolean; viewType?: boolean; inkable?: boolean; @@ -491,6 +492,11 @@ export class PresBox extends ViewBoxBaseComponent() { changed = true; } } + if (!pinDataTypes && activeItem.presDataViz !== undefined) { + if (bestTargetView?.ComponentView?.restoreView?.(activeItem)) { + changed = true; + } + } if (pinDataTypes?.scrollable || (!pinDataTypes && activeItem.presViewScroll !== undefined)) { if (bestTarget._scrollTop !== activeItem.presViewScroll) { @@ -648,6 +654,7 @@ export class PresBox extends ViewBoxBaseComponent() { if (pinProps.pinData.viewType) pinDoc.presViewType = targetDoc._viewType; if (pinProps.pinData.filters) pinDoc.presDocFilters = ObjectField.MakeCopy(targetDoc.docFilters as ObjectField); if (pinProps.pinData.pivot) pinDoc.presPivotField = targetDoc._pivotField; + if (pinProps.pinData.dataviz) pinDoc.presDataViz = new List(pinProps.pinData.dataviz); if (pinProps.pinData.pannable) { pinDoc.presPanX = NumCast(targetDoc._panX); pinDoc.presPanY = NumCast(targetDoc._panY); diff --git a/src/server/DataVizUtils.ts b/src/server/DataVizUtils.ts index 2528fb1ab..15f03b319 100644 --- a/src/server/DataVizUtils.ts +++ b/src/server/DataVizUtils.ts @@ -1,19 +1,17 @@ -import { readFileSync } from "fs"; +import { readFileSync } from 'fs'; export function csvParser(csv: string) { - const lines = csv.split("\n"); - const headers = lines[0].split(","); - const data = lines.slice(1).map(line => { - const values = line.split(","); - const obj: any = {}; - for (let i = 0; i < headers.length; i++) { - obj[headers[i]] = values[i]; - } - return obj; - }); + const lines = csv.split('\n'); + const headers = lines[0].split(',').map(header => header.trim()); + const data = lines.slice(1).map(line => + line.split(',').reduce((last, value, i) => { + last[headers[i]] = value.trim(); + return last; + }, {} as any) + ); return data; } export function csvToString(path: string) { return readFileSync(path, 'utf8'); -} \ No newline at end of file +} -- cgit v1.2.3-70-g09d2 From 0a38e3f91f4f85f07fdbb7575ceb678032dcdfe9 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush <51237606+jameshu111@users.noreply.github.com> Date: Sat, 8 Apr 2023 16:57:02 -0400 Subject: Clean code and Comments --- src/server/Client.ts | 13 +-- src/server/DashStats.ts | 198 ++++++++++++++++++++++++++++----- src/server/Message.ts | 2 + src/server/stats/userLoginStats.csv | 82 ++------------ src/server/websocket.ts | 23 ++-- views/resources/statsviewcontroller.js | 114 +++++++++++++++++++ views/stats.pug | 18 ++- 7 files changed, 320 insertions(+), 130 deletions(-) create mode 100644 views/resources/statsviewcontroller.js (limited to 'src/server') diff --git a/src/server/Client.ts b/src/server/Client.ts index be1ffc2ba..e6f953712 100644 --- a/src/server/Client.ts +++ b/src/server/Client.ts @@ -1,22 +1,11 @@ -import { action, computed } from "mobx"; +import { computed } from "mobx"; export class Client { private _guid: string; - // private operations: number; constructor(guid: string) { this._guid = guid; - // this.operations = 0; } - // @computed public get OPERATIONS(): number { - // return this.operations; - // } - - // @action - // public setOperations(newOperations: number): void { - // this.operations = newOperations; - // } - @computed public get GUID(): string { return this._guid; } } \ No newline at end of file diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts index a10b28608..8d341db63 100644 --- a/src/server/DashStats.ts +++ b/src/server/DashStats.ts @@ -1,35 +1,72 @@ import { cyan, magenta } from 'colors'; -import { Request, Response } from 'express'; -import { Server } from 'http'; +import { Response } from 'express'; import SocketIO from 'socket.io'; import { timeMap } from './ApiManagers/UserManager'; import { WebSocket } from './websocket'; const fs = require('fs'); -const csv = require('csv-parser'); +/** + * DashStats focuses on tracking user data for each session. + * + * This includes time connected, number of operations, and + * the rate of their operations + */ export namespace DashStats { + export const SAMPLING_INTERVAL = 1000; // in milliseconds (ms) - Time interval to update the frontend. + export const RATE_INTERVAL = 10; // in seconds (s) - Used to calculate rate + const statsCSVFilename = './src/server/stats/userLoginStats.csv'; const columns = ['USERNAME', 'ACTION', 'TIME']; - interface SocketPair { + /** + * UserStats holds the stats associated with a particular user. + */ + interface UserStats { socketId: string; username: string; time: string; operations: number; + rate: number; + } + + /** + * UserLastOperations is the queue object for each user + * storing their past operations. + */ + interface UserLastOperations { + sampleOperations: number; // stores how many operations total are in this rate section (10 sec, for example) + lastSampleOperations: number; // stores how many total operations were recorded at the last sample + previousOperationsQueue: number[]; // stores the operations to calculate rate. } + /** + * StatsDataBundle represents an object that will be sent to the frontend view + * on each websocket update. + */ + interface StatsDataBundle { + connectedUsers: UserStats[]; + } + + /** + * CSVStore represents how objects will be stored in the CSV + */ interface CSVStore { USERNAME: string; ACTION: string; TIME: string; } + /** + * ServerTraffic describes the current traffic going to the backend. + */ enum ServerTraffic { NOT_BUSY, BUSY, VERY_BUSY } + // These values can be changed after further testing how many + // users correspond to each traffic level in Dash. const BUSY_SERVER_BOUND = 2; const VERY_BUSY_SERVER_BOUND = 3; @@ -39,31 +76,47 @@ export namespace DashStats { "Very Busy" ] + // lastUserOperations maps each username to a UserLastOperations + // structure + export const lastUserOperations = new Map(); + + /** + * handleStats is called when the /stats route is called, providing a JSON + * object with relevant stats. In this case, we return the number of + * current connections and + * @param res Response object from Express + */ export function handleStats(res: Response) { - let current = getCurrentConnections(); + let current = getCurrentStats(); const results: CSVStore[] = []; res.json({ - message: 'welcome to stats', currentConnections: current.length, socketMap: current, }); + } + + /** + * getUpdatedStatesBundle() sends an updated copy of the current stats to the + * frontend /statsview route via websockets. + * + * @returns a StatsDataBundle that is sent to the frontend view on each websocket update + */ + export function getUpdatedStatsBundle(): StatsDataBundle { + let current = getCurrentStats(); - // fs.createReadStream(statsCSVFilename) - // .pipe(csv()) - // .on('data', (data: any) => results.push(data)) - // .on('end', () => { - // console.log(results); - // res.json({ - // message: 'welcome to stats', - // currentConnections: current.length, - // socketMap: current, - // results: results, - // }); - // }); + return { + connectedUsers: current, + } } + /** + * handleStatsView() is called when the /statsview route is called. This + * will use pug to render a frontend view of the current stats + * + * @param res + */ export function handleStatsView(res: Response) { - let current = getCurrentConnections(); + let current = getCurrentStats(); let connectedUsers = current.map((socketPair) => { return socketPair.time + " - " + socketPair.username + " Operations: " + socketPair.operations; @@ -80,17 +133,23 @@ export namespace DashStats { res.render("stats.pug", { title: "Dash Stats", - numConnections: current.length, + numConnections: connectedUsers.length, serverTraffic: serverTraffic, serverTrafficMessage : serverTrafficMessages[serverTraffic], connectedUsers: connectedUsers }); } + /** + * logUserLogin() writes a login event to the CSV file. + * + * @param username the username in the format of "username@domain.com logged in" + * @param socket the websocket associated with the current connection + */ export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - // console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); let toWrite: CSVStore = { USERNAME : username, @@ -105,10 +164,15 @@ export namespace DashStats { } } + /** + * logUserLogout() writes a logout event to the CSV file. + * + * @param username the username in the format of "username@domain.com logged in" + * @param socket the websocket associated with the current connection. + */ export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { if (!(username === undefined)) { let currentDate = new Date(); - // console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"}); let toWrite: CSVStore = { @@ -121,10 +185,79 @@ export namespace DashStats { } } - function getCurrentConnections(): SocketPair[] { - console.log("timeMap: " + timeMap); - console.log("clients:" + WebSocket.clients); - let socketPairs: SocketPair[] = []; + /** + * getLastOperationsOrDefault() is a helper method that will attempt + * to query the lastUserOperations map for a specified username. If the + * username is not in the map, an empty UserLastOperations object is returned. + * @param username + * @returns the user's UserLastOperations structure or an empty + * UserLastOperations object (All values set to 0) if the username is not found. + */ + function getLastOperationsOrDefault(username: string): UserLastOperations { + if(lastUserOperations.get(username) === undefined) { + let initializeOperationsQueue = []; + for(let i = 0; i < RATE_INTERVAL; i++) { + initializeOperationsQueue.push(0); + } + return { + sampleOperations: 0, + lastSampleOperations: 0, + previousOperationsQueue: initializeOperationsQueue + } + } + return lastUserOperations.get(username)!; + } + + /** + * updateLastOperations updates a specific user's UserLastOperations information + * for the current sampling cycle. The method removes old/outdated counts for + * operations from the queue and adds new data for the current sampling + * cycle to the queue, updating the total count as it goes. + * @param lastOperationData the old UserLastOperations data that must be updated + * @param currentOperations the total number of operations measured for this sampling cycle. + * @returns the udpated UserLastOperations structure. + */ + function updateLastOperations(lastOperationData: UserLastOperations, currentOperations: number): UserLastOperations { + // create a copy of the UserLastOperations to modify + let newLastOperationData: UserLastOperations = { + sampleOperations: lastOperationData.sampleOperations, + lastSampleOperations: lastOperationData.lastSampleOperations, + previousOperationsQueue: lastOperationData.previousOperationsQueue.slice() + } + + let newSampleOperations = newLastOperationData.sampleOperations; + newSampleOperations -= newLastOperationData.previousOperationsQueue.shift()!; // removes and returns the first element of the queue + let operationsThisCycle = currentOperations - lastOperationData.lastSampleOperations; + newSampleOperations += operationsThisCycle; // add the operations this cycle to find out what our count for the interval should be (e.g operations in the last 10 seconds) + + // update values for the copy object + newLastOperationData.sampleOperations = newSampleOperations; + + newLastOperationData.previousOperationsQueue.push(operationsThisCycle); + newLastOperationData.lastSampleOperations = currentOperations; + + return newLastOperationData; + } + + /** + * getUserOperationsOrDefault() is a helper method to get the user's total + * operations for the CURRENT sampling interval. The method will return 0 + * if the username is not in the userOperations map. + * @param username the username to search the map for + * @returns the total number of operations recorded up to this sampling cycle. + */ + function getUserOperationsOrDefault(username: string): number { + return WebSocket.userOperations.get(username) === undefined ? 0 : WebSocket.userOperations.get(username)! + } + + /** + * getCurrentStats() calculates the total stats for this cycle. In this case, + * getCurrentStats() returns an Array of UserStats[] objects describing + * the stats for each user + * @returns an array of UserStats storing data for each user at the current moment. + */ + function getCurrentStats(): UserStats[] { + let socketPairs: UserStats[] = []; for (let [key, value] of WebSocket.socketMap) { let username = value.split(' ')[0]; let connectionTime = new Date(timeMap[username]); @@ -132,19 +265,28 @@ export namespace DashStats { let connectionTimeString = connectionTime.toLocaleDateString() + " " + connectionTime.toLocaleTimeString(); if (!key.disconnected) { + let lastRecordedOperations = getLastOperationsOrDefault(username); + let currentUserOperationCount = getUserOperationsOrDefault(username); + socketPairs.push({ socketId: key.id, username: username, time: connectionTimeString.includes("Invalid Date") ? "" : connectionTimeString, operations : WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0, + rate: lastRecordedOperations.sampleOperations }); + lastUserOperations.set(username, updateLastOperations(lastRecordedOperations,currentUserOperationCount)); } } - console.log(socketPairs); - // console.log([...WebSocket.clients.entries()]); return socketPairs; } + /** + * convertToCSV() is a helper method that stringifies a CSVStore object + * that can be written to the CSV file later. + * @param dataObject the object to stringify + * @returns the object as a string. + */ function convertToCSV(dataObject: CSVStore): string { return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`; } diff --git a/src/server/Message.ts b/src/server/Message.ts index d87ae5027..8f0af08bc 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -94,4 +94,6 @@ export namespace MessageStore { export const YoutubeApiQuery = new Message("Youtube Api Query"); export const DeleteField = new Message("Delete field"); export const DeleteFields = new Message("Delete fields"); + + export const UpdateStats = new Message("updatestats"); } diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv index d5e56502c..23bcef885 100644 --- a/src/server/stats/userLoginStats.csv +++ b/src/server/stats/userLoginStats.csv @@ -1,75 +1,7 @@ -USER,ACTION,TIME -aaa@gmail.com,loggedIn,2023-03-18T20:10:29.928Z -guest,loggedIn,2023-03-18T20:10:47.384Z -aaa@gmail.com,loggedOut,2023-03-18T20:10:55.364Z -aaa@gmail.com,loggedIn,2023-03-18T20:11:15.879Z -guest,loggedIn,2023-03-18T20:18:03.724Z -aaa@gmail.com,loggedIn,2023-03-18T20:18:03.860Z -guest,loggedOut,2023-03-18T20:19:00.211Z -guest,loggedIn,2023-03-18T20:19:03.087Z -guest,loggedOut,2023-03-18T20:19:50.668Z -boo15869@gmail.com,loggedIn,2023-03-18T20:20:17.890Z -boo15869@gmail.com,loggedOut,2023-03-18T20:21:03.542Z -boo15869@gmail.com,loggedIn,2023-03-18T20:21:06.149Z -boo15869@gmail.com,loggedOut,2023-03-18T20:21:51.874Z -a@gmail.com,loggedIn,2023-03-18T20:22:02.122Z -aaa@gmail.com,loggedOut,2023-03-18T20:22:42.882Z -aaa@gmail.com,loggedIn,2023-03-18T20:22:45.631Z -aaa@gmail.com,loggedIn,2023-03-18T20:25:34.658Z -a@gmail.com,loggedIn,2023-03-18T20:25:34.681Z -aaa@gmail.com,loggedIn,2023-03-18T20:29:04.297Z -a@gmail.com,loggedIn,2023-03-18T20:29:08.701Z -a@gmail.com,loggedIn,2023-03-18T20:29:18.565Z -aaa@gmail.com,loggedIn,2023-03-18T20:29:21.974Z -aaa@gmail.com,loggedIn,2023-03-18T20:29:51.477Z -a@gmail.com,loggedIn,2023-03-18T20:29:51.489Z -aaa@gmail.com,loggedIn,2023-03-18T20:30:15.011Z -aaa@gmail.com,loggedIn,2023-03-18T20:30:57.818Z -a@gmail.com,loggedIn,2023-03-18T20:30:57.838Z -aaa@gmail.com,loggedIn,2023-03-18T20:31:12.061Z -a@gmail.com,loggedIn,2023-03-18T20:31:12.080Z -a@gmail.com,loggedIn,2023-03-18T20:31:19.447Z -aaa@gmail.com,loggedIn,2023-03-18T20:31:22.738Z -aaa@gmail.com,loggedIn,2023-03-18T20:32:36.919Z -a@gmail.com,loggedIn,2023-03-18T20:32:36.929Z -a@gmail.com,loggedIn,2023-03-18T20:32:56.212Z -aaa@gmail.com,loggedIn,2023-03-18T20:32:59.300Z -aaa@gmail.com,loggedIn,2023-03-18T20:34:27.543Z -a@gmail.com,loggedIn,2023-03-18T20:34:27.570Z -a@gmail.com,loggedIn,2023-03-18T20:34:35.299Z -aaa@gmail.com,loggedIn,2023-03-18T20:34:35.302Z -a@gmail.com,loggedIn,2023-03-18T20:34:51.579Z -aaa@gmail.com,loggedIn,2023-03-18T20:34:52.392Z -a@gmail.com,loggedIn,2023-03-18T20:35:08.509Z -aaa@gmail.com,loggedIn,2023-03-18T20:35:15.202Z -a@gmail.com,loggedIn,2023-03-18T20:36:46.796Z -aaa@gmail.com,loggedIn,2023-03-18T20:36:51.756Z -a@gmail.com,loggedIn,2023-03-18T20:36:55.286Z -a@gmail.com,loggedIn,2023-03-18T20:40:06.226Z -a@gmail.com,loggedIn,2023-03-18T20:40:18.474Z -aaa@gmail.com,loggedIn,2023-03-18T20:40:31.894Z -a@gmail.com,loggedIn,2023-03-18T20:40:31.903Z -a@gmail.com,loggedIn,2023-03-18T20:42:25.301Z -aaa@gmail.com,loggedIn,2023-03-18T20:42:31.182Z -aaa@gmail.com,loggedOut,2023-03-18T20:43:05.741Z -aaa@gmail.com,loggedIn,2023-03-18T20:43:09.203Z -a@gmail.com,loggedOut,2023-03-18T20:45:24.343Z -a@gmail.com,loggedIn,2023-03-18T20:45:27.207Z -aaa@gmail.com,loggedIn,2023-03-18T20:46:15.618Z -a@gmail.com,loggedIn,2023-03-18T20:46:56.163Z -a@gmail.com,loggedOut,2023-03-18T20:48:37.464Z -a@gmail.com,loggedIn,2023-03-18T20:48:40.562Z -a@gmail.com,loggedOut,2023-03-18T20:50:32.478Z -a@gmail.com,loggedIn,2023-03-18T20:50:40.384Z -a@gmail.com,loggedOut,2023-03-18T20:51:34.159Z -a@gmail.com,loggedIn,2023-03-18T20:51:49.206Z -a@gmail.com,loggedOut,2023-03-18T20:52:04.673Z -qw@gmail.com,loggedIn,2023-03-18T20:52:36.270Z -qw@gmail.com,loggedIn,2023-03-18T20:53:58.175Z -aaa@gmail.com,loggedIn,2023-03-18T20:53:58.204Z -aaa@gmail.com,loggedIn,2023-03-18T20:54:20.518Z -qw@gmail.com,loggedIn,2023-03-18T20:54:24.225Z -qw@gmail.com,loggedIn,2023-03-18T20:54:37.007Z -aaa@gmail.com,loggedIn,2023-03-18T20:54:39.959Z -aaa@gmail.com,loggedOut,2023-03-18T20:55:55.199Z -qw@gmail.com,loggedOut,2023-03-18T20:55:59.010Z +USER,ACTION,TIMEaaa@gmail.com,loggedIn,2023-04-08T20:08:17.533Z +aaa@gmail.com,loggedIn,2023-04-08T20:14:32.460Z +aaa@gmail.com,loggedIn,2023-04-08T20:14:44.884Z +aaa@gmail.com,loggedIn,2023-04-08T20:14:56.854Z +aaa@gmail.com,loggedIn,2023-04-08T20:16:59.747Z +aaa@gmail.com,loggedIn,2023-04-08T20:56:50.759Z +aaa@gmail.com,loggedIn,2023-04-08T20:56:58.175Z diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 54944e944..5d1390d37 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -1,4 +1,4 @@ -import { blue, magenta, yellow } from 'colors'; +import { blue } from 'colors'; import * as express from 'express'; import { createServer, Server } from 'https'; import { networkInterfaces } from 'os'; @@ -22,7 +22,6 @@ import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { export let _socket: Socket; export const clients: { [key: string]: Client } = {}; - // export const clients = new Map(); export const socketMap = new Map(); export const userOperations = new Map(); export let disconnect: Function; @@ -52,6 +51,8 @@ export namespace WebSocket { next(); }); + socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + // convenience function to log server messages on the client function log(message?: any, ...optionalParams: any[]) { socket.emit('log', ['Message from server:', message, ...optionalParams]); @@ -109,6 +110,8 @@ export namespace WebSocket { } }); + + Utils.Emit(socket, MessageStore.Foo, 'handshooken'); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -142,6 +145,12 @@ export namespace WebSocket { socket.disconnect(true); }; }); + + setInterval(function() { + // Utils.Emit(socket, MessageStore.UpdateStats, DashStats.getUpdatedStatsBundle()); + + io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + }, DashStats.SAMPLING_INTERVAL); } function processGesturePoints(socket: Socket, content: GestureContent) { @@ -185,13 +194,9 @@ export namespace WebSocket { function barReceived(socket: SocketIO.Socket, userEmail: string) { clients[userEmail] = new Client(userEmail.toString()); - // clients.set(userEmail, new Client(userEmail.toString())); const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); - // console.log(magenta(`currently connected: ${[...clients.entries()]}`)); - // console.log(magenta('socket map below')); - // console.log([...socketMap.entries()]); printActiveUsers(); timeMap[userEmail] = Date.now(); @@ -321,7 +326,8 @@ export namespace WebSocket { const remListItems = diff.diff.$set[updatefield].fields; const curList = (curListItems as any)?.fields?.[updatefield.replace('fields.', '')]?.fields.filter((f: any) => f !== null) || []; diff.diff.$set[updatefield].fields = curList?.filter( - (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) + (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : + remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) ); const sendBack = diff.diff.length !== diff.diff.$set[updatefield].fields.length; delete diff.diff.length; @@ -367,11 +373,8 @@ export namespace WebSocket { var CurUser: string | undefined = undefined; function UpdateField(socket: Socket, diff: Diff) { - console.log(magenta(`1 OP ${socketMap.get(socket)}`)); - let currentUsername = socketMap.get(socket)!.split(' ')[0]; userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0); - console.log(yellow("Total Operations: " + userOperations.get(currentUsername))); if (CurUser !== socketMap.get(socket)) { CurUser = socketMap.get(socket); diff --git a/views/resources/statsviewcontroller.js b/views/resources/statsviewcontroller.js new file mode 100644 index 000000000..090e112e7 --- /dev/null +++ b/views/resources/statsviewcontroller.js @@ -0,0 +1,114 @@ +/** + * statsviewcontroller.js stores the JavaScript functions to update the stats page + * when the websocket updates. + */ + +const BUSY_SERVER_BOUND = 2; +const VERY_BUSY_SERVER_BOUND = 3; + +const MEDIUM_USE_BOUND = 100; //operations per 10 seconds +const HIGH_USE_BOUND = 300; + +const serverTrafficMessages = { + 0 : "Not Busy", + 1 : "Busy", + 2: "Very Busy" +}; + +/** + * userDataComparator sorts the users based on the rate + * + * @param {*} user1 the first user to compare + * @param {*} user2 the second user to comapre + * @returns an integer indiciating which user should come first + */ +function userDataComparator(user1, user2) { + if(user1.rate < user2.rate) { + return 1; + } else if(user1.rate > user2.rate) { + return -1; + } else { + return 0; + } +} + +/** + * calculateServerTraffic() returns an integer corresponding + * to the current traffic that can be used to get the message + * from "serverTrafficMessages" + * + * @param {*} data the incoming data from the backend + * @returns an integer where 0 is not busy, 1 is busy, and 2 is very busy. + */ +function calculateServerTraffic(data) { + let currentTraffic = data.connectedUsers.length; + + let serverTraffic = 0; + if(currentTraffic < BUSY_SERVER_BOUND) { + serverTraffic = 0; + } else if(currentTraffic >= BUSY_SERVER_BOUND && currentTraffic < VERY_BUSY_SERVER_BOUND) { + serverTraffic = 1; + } else { + serverTraffic = 2; + } + + return serverTraffic; +} + +/** + * getUserRateColor determines what color the user's rate should + * be on the front end + * @param {*} rate the operations per time interval for a specific user + * @returns a string representing the color to make the user rate + */ +function getUserRateColor(rate) { + if(rate < MEDIUM_USE_BOUND) { + return "black"; + } else if(rate >= MEDIUM_USE_BOUND && rate < HIGH_USE_BOUND) { + return "orange"; + } else if(rate >= HIGH_USE_BOUND){ + return "red"; + } else { + return "black"; + } +} + +/** + * handleStatsUpdats() is called when new data is received from the backend + * from a websocket event. The method updates the HTML site to reflect the + * updated data + * + * @param {*} data the data coming from the backend. + */ +function handleStatsUpdate(data) { + let userListInnerHTML = ""; + data.connectedUsers.sort(userDataComparator); + data.connectedUsers.map((userData, index) => { + let userRateColor = getUserRateColor(userData.rate); + let userEntry = `

${userData.time}

+

${userData.username}

+

Operations: ${userData.operations}

+

Rate: ${userData.rate} operations per last 10 seconds

+ `; // user data comes as last 10 seconds but it can be adjusted in DastStats.ts and websocket.ts + userListInnerHTML += "
  • " + userEntry + "
    • "; + }) + + document.getElementById("connection-count").innerHTML = `Current Connections: ${data.connectedUsers.length}` + document.getElementById("connected-user-list").innerHTML = userListInnerHTML; + + let serverTraffic = calculateServerTraffic(data); + let serverTrafficMessage = "Not Busy"; + switch(serverTraffic) { + case 0: + serverTrafficMessage = "Not Busy"; + break; + case 1: + serverTrafficMessage = "Busy"; + break; + case 2: + serverTrafficMessage = "Very Busy"; + break; + } + document.getElementById("stats-traffic-message").className="stats-server-status-item stats-server-status-" + serverTraffic; + document.getElementById("stats-traffic-message").innerHTML = `

    ${serverTrafficMessage}

    `; +} \ No newline at end of file diff --git a/views/stats.pug b/views/stats.pug index 54c017e70..16c28087e 100644 --- a/views/stats.pug +++ b/views/stats.pug @@ -1,22 +1,30 @@ extends ./layout +//- stats.pug is the frontend for the stats page block content style include ./stylesheets/authentication.css include ./stylesheets/statsview.css + script(src=`http://localhost:4321/socket.io/socket.io.js`) + script + include ./resources/statsviewcontroller.js + script. + var socket = io.connect("http://localhost:4321"); + socket.on("connect", () => console.log("connected to socket")); + + socket.on("a2cf757f-abd7-537b-953e-ef2f4f798f7e", (data) => handleStatsUpdate(data)); .outermost .stats-container h1 Dash Stats - p(class="stats-content") Current Connections: #{numConnections} + + p(class="stats-content" id="connection-count") Current Connections: #{numConnections} div(class="stats-content stats-server-status-container") p Server Status: - div(class="stats-server-status-item stats-server-status-" + serverTraffic) + div(id="stats-traffic-message" class="stats-server-status-item stats-server-status-" + serverTraffic) p #{serverTrafficMessage} div(class="stats-content stats-connected-users") p Connected Users: - ul - each username in connectedUsers - li(class="none")= username + ul(id="connected-user-list") \ No newline at end of file -- cgit v1.2.3-70-g09d2 From c2ee641e1f6a8003e961d03550aaffeb668f2545 Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 17 Apr 2023 11:14:07 -0400 Subject: more gitignore --- .gitignore | 4 ---- src/server/stats/userLoginStats.csv | 7 ------- 2 files changed, 11 deletions(-) delete mode 100644 src/server/stats/userLoginStats.csv (limited to 'src/server') diff --git a/.gitignore b/.gitignore index f60c94fa4..f841c1a86 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,6 @@ ClientUtils.ts solr-8.3.1/server/logs/ solr-8.3.1/server/solr/dash/data/tlog/* solr-8.3.1/server/solr/dash/data/index/* -src/scraping/buxton/final/source/ -src/scraping/buxton/final/json/ -src/scraping/buxton/source/ src/server/public/files/ src/server/stats/userLoginStats.csv src/scraping/acm/package-lock.json @@ -21,4 +18,3 @@ debug.log .vscodeignore Dockerfile .vscode/launch.json -src/server/stats/userLoginStats.csv diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv deleted file mode 100644 index 23bcef885..000000000 --- a/src/server/stats/userLoginStats.csv +++ /dev/null @@ -1,7 +0,0 @@ -USER,ACTION,TIMEaaa@gmail.com,loggedIn,2023-04-08T20:08:17.533Z -aaa@gmail.com,loggedIn,2023-04-08T20:14:32.460Z -aaa@gmail.com,loggedIn,2023-04-08T20:14:44.884Z -aaa@gmail.com,loggedIn,2023-04-08T20:14:56.854Z -aaa@gmail.com,loggedIn,2023-04-08T20:16:59.747Z -aaa@gmail.com,loggedIn,2023-04-08T20:56:50.759Z -aaa@gmail.com,loggedIn,2023-04-08T20:56:58.175Z -- cgit v1.2.3-70-g09d2 From f83e5d34794ef675d4627ecef2ed7042b17b1b06 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:05:01 -0400 Subject: cleaning up zip/unzip of files --- src/client/views/collections/CollectionView.tsx | 13 ++- src/client/views/nodes/DocumentView.tsx | 2 +- .../views/nodes/formattedText/DashFieldView.tsx | 6 +- src/fields/Doc.ts | 128 +++++++++------------ src/server/ApiManagers/UploadManager.ts | 9 +- 5 files changed, 67 insertions(+), 91 deletions(-) (limited to 'src/server') diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index cfb9310b6..790aa765d 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -156,12 +156,13 @@ export class CollectionView extends ViewBoxAnnotatableComponent { - const newRendition = Doc.MakeAlias(this.rootDoc); - newRendition._viewType = vtype; - this.props.addDocTab(newRendition, OpenWhere.addRight); - return newRendition; - }); + !Doc.noviceMode && + this.setupViewTypes('UI Controls...', vtype => { + const newRendition = Doc.MakeAlias(this.rootDoc); + newRendition._viewType = vtype; + this.props.addDocTab(newRendition, OpenWhere.addRight); + return newRendition; + }); const options = cm.findByDescription('Options...'); const optionItems = options && 'subitems' in options ? options.subitems : []; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 7107707d1..6686f142f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -693,7 +693,7 @@ export class DocumentViewInternal extends DocComponent this.props.addDocTab(templateDoc, OpenWhere.addRight), icon: 'eye' }); !appearance && appearanceItems.length && cm.addItem({ description: 'UI Controls...', subitems: appearanceItems, icon: 'compass' }); - if (!Doc.IsSystem(this.rootDoc) && this.rootDoc._viewType !== CollectionViewType.Docking && this.props.docViewPath().lastElement()?.rootDoc?._viewType !== CollectionViewType.Tree) { + if (!Doc.IsSystem(this.rootDoc) && this.rootDoc.type !== DocumentType.PRES && ![CollectionViewType.Docking, CollectionViewType.Tree].includes(this.rootDoc._viewType as any)) { const existingOnClick = cm.findByDescription('OnClick...'); const onClicks: ContextMenuProps[] = existingOnClick && 'subitems' in existingOnClick ? existingOnClick.subitems : []; diff --git a/src/client/views/nodes/formattedText/DashFieldView.tsx b/src/client/views/nodes/formattedText/DashFieldView.tsx index c43206629..72e8aedac 100644 --- a/src/client/views/nodes/formattedText/DashFieldView.tsx +++ b/src/client/views/nodes/formattedText/DashFieldView.tsx @@ -101,11 +101,7 @@ export class DashFieldViewInternal extends React.Component { - dashDoc instanceof Doc && (this._dashDoc = dashDoc); - }) - ); + DocServer.GetRefField(this.props.docId).then(action(dashDoc => dashDoc instanceof Doc && (this._dashDoc = dashDoc))); } else { this._dashDoc = this.props.tbox.rootDoc; } diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 6c808c145..c5af45262 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -705,7 +705,7 @@ export namespace Doc { if (cloneMap.get(doc[Id])) return cloneMap.get(doc[Id])!; const copy = new Doc(undefined, true); cloneMap.set(doc[Id], copy); - const filter = [...exclusions, ...Cast(doc.cloneFieldFilter, listSpec('string'), [])]; + const filter = [...exclusions, ...StrListCast(doc.cloneFieldFilter)]; await Promise.all( Object.keys(doc).map(async key => { if (filter.includes(key)) return; @@ -725,15 +725,13 @@ export namespace Doc { const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => '(' + exp + ')').join('|') + ')":"([a-z-A-Z0-9_]*)"', 'g'); const rawdocids = field.Data.match(docidsearch); const docids = rawdocids?.map((str: string) => - DocsInTextFieldIds.reduce((output, exp) => { - return output.replace(new RegExp(`${exp}":`, 'g'), ''); - }, str) + DocsInTextFieldIds.reduce((output, exp) => output.replace(new RegExp(`${exp}":`, 'g'), ''), str) .replace(/"/g, '') .trim() ); const results = docids && (await DocServer.GetRefFields(docids)); const docs = results && Array.from(Object.keys(results)).map(key => DocCast(results[key])); - docs && docs.map(doc => Doc.makeClone(doc, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); + docs?.map(doc => doc && Doc.makeClone(doc, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); rtfs.push({ copy, key, field }); } } @@ -741,7 +739,7 @@ export namespace Doc { }; const docAtKey = doc[key]; if (docAtKey instanceof Doc) { - if (!Doc.IsSystem(docAtKey) && (key.startsWith('layout') || key === 'annotationOn' || key === 'proto' || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + if (!Doc.IsSystem(docAtKey) && (key.startsWith('layout') || ['context', 'annotationOn', 'proto'].includes(key) || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); } else { assignKey(docAtKey); @@ -749,8 +747,7 @@ export namespace Doc { } else if (field instanceof RefField) { assignKey(field); } else if (cfield instanceof ComputedField) { - assignKey(cfield[Copy]()); - // ComputedField.MakeFunction(cfield.script.originalScript)); + assignKey(cfield[Copy]()); // ComputedField.MakeFunction(cfield.script.originalScript)); } else if (field instanceof ObjectField) { await copyObjectField(field); } else if (field instanceof Promise) { @@ -804,8 +801,6 @@ export namespace Doc { const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf'], cloneLinks); const repaired = new Set(); const linkedDocs = Array.from(linkMap.values()); - const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; - clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true)); rtfMap.map(({ copy, key, field }) => { const replacer = (match: any, attr: string, id: string, offset: any, string: any) => { @@ -816,20 +811,16 @@ export namespace Doc { const mapped = cloneMap.get(id); return href + (mapped ? mapped[Id] : id); }; - const regex = `(${Doc.localServerPath()})([^"]*)`; - const re = new RegExp(regex, 'g'); + const re = new RegExp(`(${Doc.localServerPath()})([^"]*)`, 'g'); const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => `"${exp}":`).join('|') + ')"([^"]+)"', 'g'); copy[key] = new RichTextField(field.Data.replace(docidsearch, replacer).replace(re, replacer2), field.Text); }); + const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; + clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); return { clone: copy, map: cloneMap, linkMap }; } - export async function Zip(doc: Doc) { - // const a = document.createElement("a"); - // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); - // a.href = url; - // a.download = `DocExport-${this.props.Document[Id]}.zip`; - // a.click(); + export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') { const { clone, map, linkMap } = await Doc.MakeClone(doc); clone.LINKS = new List(Array.from(linkMap.values())); const proms = [] as string[]; @@ -837,76 +828,61 @@ export namespace Doc { if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; else if (value instanceof Doc) { if (key !== 'field' && Number.isNaN(Number(key))) { - const __fields = value[FieldsSym](); - return { id: value[Id], __type: 'Doc', fields: __fields }; - } else { - return { fieldId: value[Id], __type: 'proxy' }; + return { id: value[Id], __type: 'Doc', fields: value[FieldsSym]() }; } - } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; - else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; - else if (value instanceof ImageField) { + return { fieldId: value[Id], __type: 'proxy' }; + } else if (value instanceof ImageField) { const extension = value.url.href.replace(/.*\./, ''); proms.push(value.url.href.replace('.' + extension, '_o.' + extension)); return { url: value.url.href, __type: 'image' }; } else if (value instanceof PdfField) { proms.push(value.url.href); return { url: value.url.href, __type: 'pdf' }; - } else if (value instanceof AudioField) return { url: value.url.href, __type: 'audio' }; - else if (value instanceof VideoField) return { url: value.url.href, __type: 'video' }; + } else if (value instanceof AudioField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'audio' }; + } else if (value instanceof VideoField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'video' }; + } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; + else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; else if (value instanceof WebField) return { url: value.url.href, __type: 'web' }; else if (value instanceof MapField) return { url: value.url.href, __type: 'map' }; else if (value instanceof DateField) return { date: value.toString(), __type: 'date' }; else if (value instanceof ProxyField) return { fieldId: value.fieldId, __type: 'proxy' }; else if (value instanceof Array && key !== 'fields') return { fields: value, __type: 'list' }; else if (value instanceof ComputedField) return { script: value.script, __type: 'computed' }; - else return value; + return value; } const docs: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const docString = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); - - let generateZIP = (proms: string[]) => { - var zip = new JSZip(); - var count = 0; - var zipFilename = 'dashExport.zip'; - - proms - .filter(url => url.startsWith(window.location.origin)) - .forEach((url, i) => { - var filename = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); - // loading a file and add it in a zip file - JSZipUtils.getBinaryContent(url, function (err: any, data: any) { - if (err) { - throw err; // or handle the error - } - zip.file(filename, data, { binary: true }); - count++; - if (count == proms.length) { - zip.file('doc.json', docString); - zip.generateAsync({ type: 'blob' }).then(function (content) { - saveAs(content, zipFilename); - }); - } - }); - }); - }; - generateZIP(proms); - const zip = new JSZip(); - - zip.file('doc.json', docString); - - // // Generate a directory within the Zip file structure - // var img = zip.folder("images"); - - // // Add a file to the directory, in this case an image with data URI as contents - // img.file("smile.gif", imgData, {base64: true}); + const jsonDocs = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); - // Generate the zip file asynchronously - zip.generateAsync({ type: 'blob' }).then((content: any) => { - // Force down of the Zip file - saveAs(content, doc.title + '.zip'); // glr: Possibly change the name of the document to match the title? - }); + const zip = new JSZip(); + var count = 0; + proms + .filter(url => url.startsWith(window.location.origin)) + .forEach((url, i) => { + // loading a file and add it in a zip file + JSZipUtils.getBinaryContent(url, (err: any, data: any) => { + if (err) throw err; // or handle the error + // // Generate a directory within the Zip file structure + // const assets = zip.folder("assets"); + // assets.file(filename, data, {binary: true}); + const assetPathOnServer = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); + zip.file(assetPathOnServer, data, { binary: true }); + if (++count == proms.length) { + zip.file('doc.json', jsonDocs); + zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename)); + // const a = document.createElement("a"); + // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); + // a.href = url; + // a.download = `DocExport-${this.props.Document[Id]}.zip`; + // a.click(); + } + }); + }); } const _pendingMap: Map = new Map(); @@ -963,7 +939,7 @@ export namespace Doc { // otherwise, it just returns the childDoc export function GetLayoutDataDocPair(containerDoc: Doc, containerDataDoc: Opt, childDoc: Doc) { if (!childDoc || childDoc instanceof Promise || !Doc.GetProto(childDoc)) { - console.log('No, no, no!'); + console.log('Warning: GetLayoutDataDocPair childDoc not defined'); return { layout: childDoc, data: childDoc }; } const resolvedDataDoc = Doc.AreProtosEqual(containerDataDoc, containerDoc) || (!childDoc.isTemplateDoc && !childDoc.isTemplateForField) ? undefined : containerDataDoc; @@ -1553,15 +1529,19 @@ export namespace Doc { } } - export async function importDocument(file: File) { + /// + // imports a previously exported zip file which contains a set of documents and their assets (eg, images, videos) + // the 'remap' parameter determines whether the ids of the documents loaded should be kept as they were, or remapped to new ids + // If they are not remapped, loading the file will overwrite any existing documents with those ids + // + export async function importDocument(file: File, remap = false) { const upload = Utils.prepend('/uploadDoc'); const formData = new FormData(); if (file) { formData.append('file', file); - formData.append('remap', 'true'); + formData.append('remap', remap.toString()); const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); - console.log(json); if (json !== 'error') { await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 6e28268a9..2cc2ac46c 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -184,9 +184,8 @@ export default class UploadManager extends ApiManager { if (id.endsWith('Proto')) return id; if (id in ids) { return ids[id]; - } else { - return (ids[id] = v4()); } + return (ids[id] = v4()); }; const mapFn = (doc: any) => { if (doc.id) { @@ -266,7 +265,7 @@ export default class UploadManager extends ApiManager { await Promise.all( docs.map( (doc: any) => - new Promise(res => { + new Promise(res => Database.Instance.replace( doc.id, doc, @@ -275,8 +274,8 @@ export default class UploadManager extends ApiManager { res(); }, true - ); - }) + ) + ) ) ); } catch (e) { -- cgit v1.2.3-70-g09d2 From 237e39029201aba23249de1ab5c405197acffe5a Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:50:28 -0400 Subject: more cleanup of import/export --- src/fields/Doc.ts | 15 +++++++++------ src/server/ApiManagers/UploadManager.ts | 34 +++++++++++++++------------------ 2 files changed, 24 insertions(+), 25 deletions(-) (limited to 'src/server') diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index c5af45262..35f9a1032 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -822,7 +822,6 @@ export namespace Doc { export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') { const { clone, map, linkMap } = await Doc.MakeClone(doc); - clone.LINKS = new List(Array.from(linkMap.values())); const proms = [] as string[]; function replacer(key: any, value: any) { if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; @@ -856,8 +855,10 @@ export namespace Doc { } const docs: { [id: string]: any } = {}; + const links: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const jsonDocs = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); + Array.from(linkMap.entries()).forEach(l => (links[l[0]] = l[1])); + const jsonDocs = JSON.stringify({ id: clone[Id], docs, links }, decycle(replacer)); const zip = new JSZip(); var count = 0; @@ -873,7 +874,7 @@ export namespace Doc { const assetPathOnServer = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); zip.file(assetPathOnServer, data, { binary: true }); if (++count == proms.length) { - zip.file('doc.json', jsonDocs); + zip.file('docs.json', jsonDocs); zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename)); // const a = document.createElement("a"); // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); @@ -1543,10 +1544,12 @@ export namespace Doc { const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); if (json !== 'error') { - await DocServer.GetRefFields(json.docids as string[]); + const docs = await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); - (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); - doc.LINKS = undefined; + const links = await DocServer.GetRefFields(json.linkids as string[]); + Array.from(Object.keys(links)) + .map(key => links[key]) + .forEach(link => link instanceof Doc && LinkManager.Instance.addLink(link)); return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2cc2ac46c..1f72d72dc 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -228,6 +228,7 @@ export default class UploadManager extends ApiManager { remap = fields.remap !== 'false'; let id: string = ''; let docids: string[] = []; + let linkids: string[] = []; try { for (const name in files) { const f = files[name]; @@ -254,28 +255,23 @@ export default class UploadManager extends ApiManager { console.log(e); } }); - const json = zip.getEntry('doc.json'); + const json = zip.getEntry('docs.json'); try { const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); - const datadocs = data.docs; + const { docs, links } = data; id = getId(data.id); - const docs = Object.keys(datadocs).map(key => datadocs[key]); - docs.forEach(mapFn); - docids = docs.map(doc => doc.id); + const rdocs = Object.keys(docs).map(key => docs[key]); + const ldocs = Object.keys(links).map(key => links[key]); + [...rdocs, ...ldocs].forEach(mapFn); + docids = rdocs.map(doc => doc.id); + linkids = ldocs.map(link => link.id); await Promise.all( - docs.map( - (doc: any) => - new Promise(res => - Database.Instance.replace( - doc.id, - doc, - (err, r) => { - err && console.log(err); - res(); - }, - true - ) - ) + [...rdocs, ...ldocs].map( + doc => + new Promise(res => { + // overwrite mongo doc with json doc contents + Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true); + }) ) ); } catch (e) { @@ -284,7 +280,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify({ id, docids } || 'error')); + res.send(JSON.stringify({ id, docids, linkids } || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From 16a33f53c2a031f791ed8c98d539409e1a7c6bb4 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:52:43 -0400 Subject: from last --- src/server/ApiManagers/UploadManager.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 1f72d72dc..74c06b4a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -180,11 +180,8 @@ export default class UploadManager extends ApiManager { const ids: { [id: string]: string } = {}; let remap = true; const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith('Proto')) return id; - if (id in ids) { - return ids[id]; - } + if (!remap || id.endsWith('Proto')) return id; + if (id in ids) return ids[id]; return (ids[id] = v4()); }; const mapFn = (doc: any) => { -- cgit v1.2.3-70-g09d2 From 2a584d4827c9ece87f3bd618201f356237ba7fc7 Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 1 May 2023 12:49:06 -0400 Subject: fixed RecordingBox/View serialization to save docid, not Doc. --- src/client/util/ReplayMovements.ts | 27 +++--- src/client/util/TrackMovements.ts | 2 +- .../views/nodes/RecordingBox/RecordingBox.tsx | 106 +++++++++++---------- .../views/nodes/RecordingBox/RecordingView.tsx | 10 +- src/fields/Doc.ts | 5 +- src/server/DashUploadUtils.ts | 14 ++- src/server/websocket.ts | 23 +++-- 7 files changed, 102 insertions(+), 85 deletions(-) (limited to 'src/server') diff --git a/src/client/util/ReplayMovements.ts b/src/client/util/ReplayMovements.ts index 40261985a..22cca4a2e 100644 --- a/src/client/util/ReplayMovements.ts +++ b/src/client/util/ReplayMovements.ts @@ -1,13 +1,11 @@ +import { IReactionDisposer, observable, reaction } from 'mobx'; +import { Doc, IdToDoc } from '../../fields/Doc'; +import { CollectionDockingView } from '../views/collections/CollectionDockingView'; import { CollectionFreeFormView } from '../views/collections/collectionFreeForm'; -import { IReactionDisposer, observable, observe, reaction } from 'mobx'; -import { Doc } from '../../fields/Doc'; +import { OpenWhereMod } from '../views/nodes/DocumentView'; import { VideoBox } from '../views/nodes/VideoBox'; import { DocumentManager } from './DocumentManager'; -import { CollectionDockingView } from '../views/collections/CollectionDockingView'; -import { DocServer } from '../DocServer'; import { Movement, Presentation } from './TrackMovements'; -import { OpenWhereMod } from '../views/nodes/DocumentView'; -import { returnTransparent } from '../../Utils'; export class ReplayMovements { private timers: NodeJS.Timeout[] | null; @@ -61,7 +59,7 @@ export class ReplayMovements { return; } - const docIdtoDoc = this.loadPresentation(presentation); + this.loadPresentation(presentation); this.videoBoxDisposeFunc = reaction( () => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }), @@ -94,13 +92,14 @@ export class ReplayMovements { throw '[recordingApi.ts] followMovements() failed: no presentation data'; } - // generate a set of all unique docIds - const docs = new Set(); - for (const { doc } of movements) { - if (!docs.has(doc)) docs.add(doc); - } - - return docs; + movements.forEach((movement, i) => { + if (typeof movement.doc === 'string') { + movements[i].doc = IdToDoc(movement.doc); + if (!movements[i].doc) { + console.log('ERROR: tracked doc not found'); + } + } + }); }; // returns undefined if the docView isn't open on the screen diff --git a/src/client/util/TrackMovements.ts b/src/client/util/TrackMovements.ts index 2f16307b3..cb8225643 100644 --- a/src/client/util/TrackMovements.ts +++ b/src/client/util/TrackMovements.ts @@ -234,7 +234,7 @@ export class TrackMovements { const movement: Movement = { time, panX, panY, scale, doc }; // add that movement to the current presentation data's movement array - this.currentPresentation.movements && this.currentPresentation.movements.push(movement); + this.currentPresentation.movements?.push(movement); }; // method that concatenates an array of presentatations into one diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 0ff7c4292..f406ffbea 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -1,58 +1,64 @@ -import { action, observable } from "mobx"; -import { observer } from "mobx-react"; -import * as React from "react"; -import { VideoField } from "../../../../fields/URLField"; -import { Upload } from "../../../../server/SharedMediaTypes"; -import { ViewBoxBaseComponent } from "../../DocComponent"; -import { FieldView } from "../FieldView"; -import { VideoBox } from "../VideoBox"; +import { action, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { VideoField } from '../../../../fields/URLField'; +import { Upload } from '../../../../server/SharedMediaTypes'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { FieldView } from '../FieldView'; +import { VideoBox } from '../VideoBox'; import { RecordingView } from './RecordingView'; -import { DocumentType } from "../../../documents/DocumentTypes"; -import { Presentation } from "../../../util/TrackMovements"; -import { Doc } from "../../../../fields/Doc"; -import { Id } from "../../../../fields/FieldSymbols"; - +import { DocumentType } from '../../../documents/DocumentTypes'; +import { Presentation } from '../../../util/TrackMovements'; +import { Doc } from '../../../../fields/Doc'; +import { Id } from '../../../../fields/FieldSymbols'; @observer export class RecordingBox extends ViewBoxBaseComponent() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(RecordingBox, fieldKey); + } - public static LayoutString(fieldKey: string) { return FieldView.LayoutString(RecordingBox, fieldKey); } - - private _ref: React.RefObject = React.createRef(); + private _ref: React.RefObject = React.createRef(); - constructor(props: any) { + constructor(props: any) { super(props); - } - - componentDidMount() { - Doc.SetNativeWidth(this.dataDoc, 1280); - Doc.SetNativeHeight(this.dataDoc, 720); - } - - @observable result: Upload.AccessPathInfo | undefined = undefined - @observable videoDuration: number | undefined = undefined - - @action - setVideoDuration = (duration: number) => { - this.videoDuration = duration - } - - @action - setResult = (info: Upload.AccessPathInfo, presentation?: Presentation) => { - this.result = info - this.dataDoc.type = DocumentType.VID; - this.dataDoc[this.fieldKey + "-duration"] = this.videoDuration; - - this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); - this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); - this.dataDoc[this.fieldKey + "-recorded"] = true; - // stringify the presentation and store it - presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation)); - } - - render() { - return
    - {!this.result && } -
    ; - } + } + + componentDidMount() { + Doc.SetNativeWidth(this.dataDoc, 1280); + Doc.SetNativeHeight(this.dataDoc, 720); + } + + @observable result: Upload.AccessPathInfo | undefined = undefined; + @observable videoDuration: number | undefined = undefined; + + @action + setVideoDuration = (duration: number) => { + this.videoDuration = duration; + }; + + @action + setResult = (info: Upload.AccessPathInfo, presentation?: Presentation) => { + this.result = info; + this.dataDoc.type = DocumentType.VID; + this.dataDoc[this.fieldKey + '-duration'] = this.videoDuration; + + this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); + this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); + this.dataDoc[this.fieldKey + '-recorded'] = true; + // stringify the presentation and store it + if (presentation?.movements) { + const presCopy = { ...presentation }; + presCopy.movements = presentation.movements.map(movement => ({ ...movement, doc: movement.doc[Id] })) as any; + this.dataDoc[this.fieldKey + '-presentation'] = JSON.stringify(presCopy); + } + }; + + render() { + return ( +
    + {!this.result && } +
    + ); + } } diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index 6efe62e0b..424ebc384 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -1,14 +1,14 @@ import * as React from 'react'; -import './RecordingView.scss'; import { useEffect, useRef, useState } from 'react'; -import { ProgressBar } from './ProgressBar'; -import { MdBackspace } from 'react-icons/md'; -import { FaCheckCircle } from 'react-icons/fa'; import { IconContext } from 'react-icons'; -import { Networking } from '../../../Network'; +import { FaCheckCircle } from 'react-icons/fa'; +import { MdBackspace } from 'react-icons/md'; import { Upload } from '../../../../server/SharedMediaTypes'; import { returnFalse, returnTrue, setupMoveUpEvents } from '../../../../Utils'; +import { Networking } from '../../../Network'; import { Presentation, TrackMovements } from '../../../util/TrackMovements'; +import { ProgressBar } from './ProgressBar'; +import './RecordingView.scss'; export interface MediaSegment { videoChunks: any[]; diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 93c28cf08..0ec881c48 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -1703,8 +1703,11 @@ export namespace Doc { } } +export function IdToDoc(id: string) { + return DocCast(DocServer.GetCachedRefField(id)); +} ScriptingGlobals.add(function idToDoc(id: string): any { - return DocServer.GetCachedRefField(id); + return IdToDoc(id); }); ScriptingGlobals.add(function renameAlias(doc: any) { return StrCast(Doc.GetProto(doc).title).replace(/\([0-9]*\)/, '') + `(${doc.aliasNumber})`; diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index f461cf3fa..070d49ec3 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -70,7 +70,14 @@ export namespace DashUploadUtils { // make a list of paths to create the ordered text file for ffmpeg const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n'); // write the text file to the file system - writeFile(textFilePath, filePathsText, err => console.log(err)); + await new Promise((res, reject) => + writeFile(textFilePath, filePathsText, err => { + if (err) { + reject(); + console.log(err); + } else res(); + }) + ); // make output file name based on timestamp const outputFileName = `output-${Utils.GenerateGuid()}.mp4`; @@ -86,7 +93,10 @@ export namespace DashUploadUtils { .outputOptions('-c copy') //.videoCodec("copy") .save(outputFilePath) - .on('error', reject) + .on('error', (err: any) => { + console.log(err); + reject(); + }) .on('end', resolve); }); diff --git a/src/server/websocket.ts b/src/server/websocket.ts index a11d20cfa..2acdaa5a3 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -51,7 +51,7 @@ export namespace WebSocket { next(); }); - socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); // convenience function to log server messages on the client function log(message?: any, ...optionalParams: any[]) { @@ -104,14 +104,12 @@ export namespace WebSocket { socket.on('disconnect', function () { let currentUser = socketMap.get(socket); if (!(currentUser === undefined)) { - let currentUsername = currentUser.split(' ')[0] + let currentUsername = currentUser.split(' ')[0]; DashStats.logUserLogout(currentUsername, socket); - delete timeMap[currentUsername] + delete timeMap[currentUsername]; } }); - - Utils.Emit(socket, MessageStore.Foo, 'handshooken'); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -146,10 +144,10 @@ export namespace WebSocket { }; }); - setInterval(function() { + setInterval(function () { // Utils.Emit(socket, MessageStore.UpdateStats, DashStats.getUpdatedStatsBundle()); - io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); }, DashStats.SAMPLING_INTERVAL); } @@ -193,7 +191,7 @@ export namespace WebSocket { } function barReceived(socket: SocketIO.Socket, userEmail: string) { - clients[userEmail] = new Client(userEmail.toString()); + clients[userEmail] = new Client(userEmail.toString()); const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); @@ -326,8 +324,7 @@ export namespace WebSocket { const remListItems = diff.diff.$set[updatefield].fields; const curList = (curListItems as any)?.fields?.[updatefield.replace('fields.', '')]?.fields.filter((f: any) => f !== null) || []; diff.diff.$set[updatefield].fields = curList?.filter( - (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : - remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) + (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) ); const sendBack = diff.diff.length !== diff.diff.$set[updatefield].fields.length; delete diff.diff.length; @@ -373,9 +370,11 @@ export namespace WebSocket { var CurUser: string | undefined = undefined; function UpdateField(socket: Socket, diff: Diff) { - let currentUsername = socketMap.get(socket)!.split(' ')[0]; + const curUser = socketMap.get(socket); + if (!curUser) return; + let currentUsername = curUser.split(' ')[0]; userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0); - + if (CurUser !== socketMap.get(socket)) { CurUser = socketMap.get(socket); console.log('Switch User: ' + CurUser); -- cgit v1.2.3-70-g09d2 From 37f282f075f521fe9ece21e151a51ffd47016782 Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 5 May 2023 09:37:13 -0400 Subject: fixed issues with webpages that use brotli encoding --- package-lock.json | 219 +++++++++++++++++-------------- package.json | 2 + src/client/views/nodes/WebBoxRenderer.js | 4 +- src/server/server_Initialization.ts | 29 ++-- 4 files changed, 139 insertions(+), 115 deletions(-) (limited to 'src/server') diff --git a/package-lock.json b/package-lock.json index 7d2ef9509..111d1838a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1274,6 +1274,15 @@ "@types/node": "*" } }, + "@types/brotli": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@types/brotli/-/brotli-1.3.1.tgz", + "integrity": "sha512-mGwX0BBQqmpHoX8+b8Oez0X+ZEYnl2gbDL2n0HxYT4imqhTChhj1AAgAKVWNZSuPvXGZXqVoOtBS0071tN6Tkw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/bson": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@types/bson/-/bson-4.0.5.tgz", @@ -2452,7 +2461,7 @@ "@types/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-xevGOReSYGM7g/kUBZzPqCrR/KYAo+F0yiPc85WFTJa0MSLtyFTVTU6cJu/aV4mid7IffDIWqo69THF2o4JiEQ==", + "integrity": "sha1-FKjsOVbC6B7bdSB5CuzyHCkK69I=", "dev": true }, "@types/strip-json-comments": { @@ -2803,7 +2812,7 @@ "textarea-caret": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/textarea-caret/-/textarea-caret-3.0.2.tgz", - "integrity": "sha1-82DEhpmqGr9xhoCkOjGoUGZcLK8=" + "integrity": "sha512-gRzeti2YS4did7UJnPQ47wrjD+vp+CJIe9zbsu0bJ987d8QVLvLNG9757rqiQTIy4hGIeFauTTJt5Xkn51UkXg==" } } }, @@ -2902,7 +2911,7 @@ "after": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz", - "integrity": "sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8=" + "integrity": "sha512-QbJ0NTQ/I9DI3uSJA4cbexiwQeRAfjPScqIbSjUDd9TOrcg6pTkdgziesOqxBMBzit8vFCTwrP27t13vFOORRA==" }, "agent-base": { "version": "6.0.2", @@ -3799,7 +3808,7 @@ "backo2": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", - "integrity": "sha1-MasayLEpNjRj41s+u2n038+6eUc=" + "integrity": "sha512-zj6Z6M7Eq+PBZ7PQxl5NT665MvJdAkzp0f60nAJ+sLaSCBPMwVak5ZegFbgVCzFcCJTKFoMizvM5Ld7+JrRJHA==" }, "bail": { "version": "2.0.2", @@ -3869,7 +3878,7 @@ "base64-arraybuffer": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz", - "integrity": "sha1-mBjHngWbE1X5fgQooBfIOOkLqBI=" + "integrity": "sha512-a1eIFi4R9ySrbiMuyTGx5e92uRH5tQY6kArNcFaKBUleIoLjdjBg7Zxm3Mqm3Kmkf27HLR/1fnxX9q8GQ7Iavg==" }, "base64-js": { "version": "1.5.1", @@ -4121,6 +4130,14 @@ } } }, + "brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "requires": { + "base64-js": "^1.1.2" + } + }, "browndash-components": { "version": "0.0.28", "resolved": "https://registry.npmjs.org/browndash-components/-/browndash-components-0.0.28.tgz", @@ -4842,7 +4859,7 @@ "component-bind": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/component-bind/-/component-bind-1.0.0.tgz", - "integrity": "sha1-AMYIq33Nk4l8AAllGx06jh5zu9E=" + "integrity": "sha512-WZveuKPeKAG9qY+FkYDeADzdHyTYdIboXS59ixDeRJL5ZhxpqUnxSOwop4FQjMsiYm3/Or8cegVbpAHNA7pHxw==" }, "component-emitter": { "version": "1.3.0", @@ -4852,7 +4869,7 @@ "component-inherit": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/component-inherit/-/component-inherit-0.0.3.tgz", - "integrity": "sha1-ZF/ErfWLcrZJ1crmUTVhnbJv8UM=" + "integrity": "sha512-w+LhYREhatpVqTESyGFg3NlP6Iu0kEKUHETY9GoZP/pQyW4mHFZuFWRUCIqVPZ36ueVLtoOEZaAqbCF2RDndaA==" }, "compress-commons": { "version": "2.1.1", @@ -5572,7 +5589,7 @@ "custom-event": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/custom-event/-/custom-event-1.0.1.tgz", - "integrity": "sha1-XQKkaFCt8bSjF5RqOSj8y1v9BCU=" + "integrity": "sha512-GAj5FOq0Hd+RsCGVJxZuKaIDXDf3h6GQoNEjFgbLLI/trgtavwUbSnZ5pVfg27DVCaWjIohryS0JFwIJyT2cMg==" }, "cyclist": { "version": "1.0.1", @@ -6656,7 +6673,7 @@ "dynamic-dedupe": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz", - "integrity": "sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==", + "integrity": "sha1-BuRMIj9eTpTXjvnbI6ZRXOL5YqE=", "dev": true, "requires": { "xtend": "^4.0.0" @@ -6786,7 +6803,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "ws": { "version": "7.4.6", @@ -9316,7 +9333,7 @@ "fs-extra": { "version": "0.26.7", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.26.7.tgz", - "integrity": "sha512-waKu+1KumRhYv8D8gMRCKJGAMI9pRnPuEb1mvgYD0f7wBscg+h6bW4FDTmEZhB9VKxvoTtxW+Y7bnIlB7zja6Q==", + "integrity": "sha1-muH92UiXeY7at20JGM9C0MMYT6k=", "requires": { "graceful-fs": "^4.1.2", "jsonfile": "^2.1.0", @@ -9976,14 +9993,14 @@ "isarray": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz", - "integrity": "sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4=" + "integrity": "sha512-c2cu3UxbI+b6kR3fy0nRnAhodsvR9dx7U5+znCOzdj6IfP3upFURTr0Xl5BlQZNKZjEtxrmVyfSdeE3O57smoQ==" } } }, "has-cors": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz", - "integrity": "sha1-XkdHk/fqmEPRu5nCPu9J/xJv/zk=" + "integrity": "sha512-g5VNKdkFuUuVCP9gYfDJHjK2nqdQJ7aDLTnycnc2+RvsOQbuLdF5pm7vuE5J76SEBIQjs4kQY/BWq74JUmjbXA==" }, "has-flag": { "version": "3.0.0", @@ -10559,7 +10576,7 @@ "indexof": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz", - "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=" + "integrity": "sha512-i0G7hLJ1z0DE8dsqJa2rycj9dBmNKgXBvotXtZYXakU9oivfB9Uj2ZBC27qqef2U58/ZLwalxa1X/RDCdkHtVg==" }, "inflight": { "version": "1.0.6", @@ -11670,7 +11687,7 @@ "jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", - "integrity": "sha512-PKllAqbgLgxHaj8TElYymKCAgrASebJrWpTnEkOaTowt23VKXXN0sUeriJ+eh7y6ufb/CC5ap11pz71/cM0hUw==", + "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "requires": { "graceful-fs": "^4.1.6" } @@ -12093,7 +12110,7 @@ "lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", - "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" }, "lodash.isplainobject": { "version": "4.0.6", @@ -12354,7 +12371,7 @@ "mathquill": { "version": "0.10.1-a", "resolved": "https://registry.npmjs.org/mathquill/-/mathquill-0.10.1-a.tgz", - "integrity": "sha1-vyylaQEAY6w0vNXVKa3Ag3zVPD8=", + "integrity": "sha512-snSAEwAtwdwBFSor+nVBnWWQtTw67kgAgKMyAIxuz4ZPboy0qkWZmd7BL3lfOXp/INihhRlU1PcfaAtDaRhmzA==", "requires": { "jquery": "^1.12.3" }, @@ -12362,7 +12379,7 @@ "jquery": { "version": "1.12.4", "resolved": "https://registry.npmjs.org/jquery/-/jquery-1.12.4.tgz", - "integrity": "sha1-AeHfuikP5z3rp3zurLD5ui/sngw=" + "integrity": "sha512-UEVp7PPK9xXYSk8xqXCJrkXnKZtlgWkd2GsAQbMRFK6S/ePU2JN5G2Zum8hIVjzR3CpdfSqdqAzId/xd4TJHeg==" } } }, @@ -14048,7 +14065,7 @@ "dependencies": { "@iarna/cli": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@iarna/cli/-/cli-2.1.0.tgz", + "resolved": false, "integrity": "sha512-rvVVqDa2g860niRbqs3D5RhL4la3dc1vwk+NlpKPZxKaMSHtE2se6C2x8NeveN+rcjp3/686X+u+09CZ+7lmAQ==", "requires": { "glob": "^7.1.2", @@ -14151,7 +14168,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14166,7 +14183,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14175,12 +14192,12 @@ }, "asap": { "version": "2.0.6", - "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "resolved": false, "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "asn1": { "version": "0.2.6", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "resolved": false, "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", "requires": { "safer-buffer": "~2.1.0" @@ -14188,7 +14205,7 @@ }, "assert-plus": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "resolved": false, "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "asynckit": { @@ -14198,22 +14215,22 @@ }, "aws-sign2": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "resolved": false, "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" }, "aws4": { "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", + "resolved": false, "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "balanced-match": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "resolved": false, "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "bcrypt-pbkdf": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "resolved": false, "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", "requires": { "tweetnacl": "^0.14.3" @@ -14234,7 +14251,7 @@ }, "bluebird": { "version": "3.7.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "resolved": false, "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "boxen": { @@ -14282,7 +14299,7 @@ }, "cacache": { "version": "12.0.4", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", + "resolved": false, "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", "requires": { "bluebird": "^3.5.5", @@ -14319,7 +14336,7 @@ }, "caseless": { "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "resolved": false, "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, "chalk": { @@ -14463,7 +14480,7 @@ }, "combined-stream": { "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "resolved": false, "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "requires": { "delayed-stream": "~1.0.0" @@ -14471,7 +14488,7 @@ }, "concat-map": { "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "resolved": false, "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "concat-stream": { @@ -14501,7 +14518,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14516,7 +14533,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14525,7 +14542,7 @@ }, "config-chain": { "version": "1.1.13", - "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "resolved": false, "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", "requires": { "ini": "^1.3.4", @@ -14626,7 +14643,7 @@ }, "dashdash": { "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "resolved": false, "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", "requires": { "assert-plus": "^1.0.0" @@ -14659,7 +14676,7 @@ }, "decode-uri-component": { "version": "0.2.2", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "resolved": false, "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==" }, "deep-extend": { @@ -14705,7 +14722,7 @@ }, "dezalgo": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "resolved": false, "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", "requires": { "asap": "^2.0.0", @@ -14757,7 +14774,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14772,7 +14789,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14781,7 +14798,7 @@ }, "ecc-jsbn": { "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "resolved": false, "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", "requires": { "jsbn": "~0.1.0", @@ -14816,7 +14833,7 @@ }, "env-paths": { "version": "2.2.1", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "resolved": false, "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==" }, "err-code": { @@ -14900,7 +14917,7 @@ }, "extsprintf": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "resolved": false, "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" }, "fast-json-stable-stringify": { @@ -14910,12 +14927,12 @@ }, "figgy-pudding": { "version": "3.5.2", - "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz", + "resolved": false, "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==" }, "filter-obj": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "resolved": false, "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==" }, "find-npm-prefix": { @@ -14948,7 +14965,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14963,7 +14980,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14972,12 +14989,12 @@ }, "forever-agent": { "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "resolved": false, "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" }, "form-data": { "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "resolved": false, "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", "requires": { "asynckit": "^0.4.0", @@ -15010,7 +15027,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -15025,7 +15042,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -15093,7 +15110,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -15108,7 +15125,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -15117,7 +15134,7 @@ }, "fs.realpath": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "resolved": false, "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "function-bind": { @@ -15207,7 +15224,7 @@ }, "getpass": { "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "resolved": false, "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", "requires": { "assert-plus": "^1.0.0" @@ -15215,7 +15232,7 @@ }, "glob": { "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "resolved": false, "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "requires": { "fs.realpath": "^1.0.0", @@ -15228,7 +15245,7 @@ "dependencies": { "minimatch": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "resolved": false, "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" @@ -15271,12 +15288,12 @@ }, "graceful-fs": { "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "resolved": false, "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "har-schema": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "resolved": false, "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==" }, "har-validator": { @@ -15355,7 +15372,7 @@ }, "http-signature": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "resolved": false, "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", "requires": { "assert-plus": "^1.0.0", @@ -15482,7 +15499,7 @@ }, "is-cidr": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/is-cidr/-/is-cidr-3.1.1.tgz", + "resolved": false, "integrity": "sha512-Gx+oErgq1j2jAKCR2Kbq0b3wbH0vQKqZ0wOlHxm0o56nq51Cs/DZA8oz9dMDhbHyHEGgJ86eTeVudtgMMOx3Mw==", "requires": { "cidr-regex": "^2.0.10" @@ -15561,7 +15578,7 @@ }, "is-typedarray": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "resolved": false, "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" }, "isarray": { @@ -15576,12 +15593,12 @@ }, "isstream": { "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "resolved": false, "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "jsbn": { "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "resolved": false, "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" }, "json-parse-better-errors": { @@ -15591,7 +15608,7 @@ }, "json-parse-even-better-errors": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "resolved": false, "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" }, "json-schema": { @@ -15601,7 +15618,7 @@ }, "json-stringify-safe": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "resolved": false, "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, "jsonparse": { @@ -15819,7 +15836,7 @@ }, "lock-verify": { "version": "2.2.2", - "resolved": "https://registry.npmjs.org/lock-verify/-/lock-verify-2.2.2.tgz", + "resolved": false, "integrity": "sha512-2CUNtr1ZSVKJHcYP8uEzafmmuyauCB5zZimj8TvQd/Lflt9kXVZs+8S+EbAzZLaVUDn8CYGmeC3DFGdYfnCzeQ==", "requires": { "@iarna/cli": "^2.1.0", @@ -15948,7 +15965,7 @@ }, "meant": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/meant/-/meant-1.0.3.tgz", + "resolved": false, "integrity": "sha512-88ZRGcNxAq4EH38cQ4D85PM57pikCwS8Z99EWHODxN7KBY+UuPiqzRTtZzS8KTXO/ywSWbdjjJST2Hly/EQxLw==" }, "mime-db": { @@ -15966,7 +15983,7 @@ }, "minimatch": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "resolved": false, "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" @@ -16015,7 +16032,7 @@ }, "mkdirp": { "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "resolved": false, "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "requires": { "minimist": "^1.2.6" @@ -16063,7 +16080,7 @@ }, "node-gyp": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-5.1.1.tgz", + "resolved": false, "integrity": "sha512-WH0WKGi+a4i4DUt2mHnvocex/xPLp9pYt5R6M2JdFB7pJ7Z34hveZ4nDTGTiLXCkitA9T8HFZjhinBCiVHYcWw==", "requires": { "env-paths": "^2.2.0", @@ -16401,7 +16418,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16416,7 +16433,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16430,7 +16447,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": false, "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, "path-is-inside": { @@ -16450,7 +16467,7 @@ }, "performance-now": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "resolved": false, "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "pify": { @@ -16499,7 +16516,7 @@ }, "proto-list": { "version": "1.2.4", - "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "resolved": false, "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==" }, "protoduck": { @@ -16522,7 +16539,7 @@ }, "psl": { "version": "1.9.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "resolved": false, "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, "pump": { @@ -16562,12 +16579,12 @@ }, "qs": { "version": "6.5.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", + "resolved": false, "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==" }, "query-string": { "version": "6.14.1", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.1.tgz", + "resolved": false, "integrity": "sha512-XDxAeVmpfu1/6IjyT/gXHOl+S0vQ9owggJ30hhWKdHAsNPOcasn5o9BW0eejZqL2e4vMjhAxoW3jVHcD6mbcYw==", "requires": { "decode-uri-component": "^0.2.0", @@ -16578,7 +16595,7 @@ }, "qw": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/qw/-/qw-1.0.2.tgz", + "resolved": false, "integrity": "sha512-1PhZ/iLKwlVNq45dnerTMKFjMof49uqli7/0QsvPNbX5OJ3IZ8msa9lUpvPheVdP+IYYPrf6cOaVil7S35joVA==" }, "rc": { @@ -16624,7 +16641,7 @@ }, "read-package-json": { "version": "2.1.2", - "resolved": "https://registry.npmjs.org/read-package-json/-/read-package-json-2.1.2.tgz", + "resolved": false, "integrity": "sha512-D1KmuLQr6ZSJS0tW8hf3WGpRlwszJOXZ3E8Yd/DNRaM5d+1wVRZdHlpGBLAuovjr28LbWvjpWkBHMxpRGGjzNA==", "requires": { "glob": "^7.1.1", @@ -16683,7 +16700,7 @@ }, "request": { "version": "2.88.2", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "resolved": false, "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "requires": { "aws-sign2": "~0.7.0", @@ -16753,7 +16770,7 @@ }, "safe-buffer": { "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "resolved": false, "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safer-buffer": { @@ -16924,7 +16941,7 @@ }, "sshpk": { "version": "1.17.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", + "resolved": false, "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", "requires": { "asn1": "~0.2.3", @@ -16980,7 +16997,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16995,7 +17012,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -17009,7 +17026,7 @@ }, "strict-uri-encode": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "resolved": false, "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==" }, "string-width": { @@ -17165,7 +17182,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -17180,7 +17197,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -17199,7 +17216,7 @@ }, "tough-cookie": { "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "resolved": false, "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "requires": { "psl": "^1.1.28", @@ -17208,7 +17225,7 @@ "dependencies": { "punycode": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "resolved": false, "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" } } @@ -17223,7 +17240,7 @@ }, "tweetnacl": { "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "resolved": false, "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "typedarray": { @@ -17294,7 +17311,7 @@ }, "uri-js": { "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "resolved": false, "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "requires": { "punycode": "^2.1.0" @@ -17335,7 +17352,7 @@ }, "uuid": { "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "resolved": false, "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" }, "validate-npm-package-license": { @@ -17357,7 +17374,7 @@ }, "verror": { "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "resolved": false, "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", "requires": { "assert-plus": "^1.0.0", @@ -22168,7 +22185,7 @@ "to-array": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/to-array/-/to-array-0.1.4.tgz", - "integrity": "sha1-F+bBH3PdTz10zaek/zI46a2b+JA=" + "integrity": "sha512-LhVdShQD/4Mk4zXNroIQZJC+Ap3zgLcDuwEdcmLv9CCO73NWockQDwyUnW/m8VX/EElfL6FcYx7EeutN4HJA6A==" }, "to-fast-properties": { "version": "2.0.0", @@ -22540,7 +22557,7 @@ "strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", "dev": true } } @@ -23797,7 +23814,7 @@ "resolve-cwd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz", - "integrity": "sha512-ccu8zQTrzVr954472aUVPLEcB3YpKSYR3cg/3lo1okzobPBM+1INXBbBZlDbnI/hbEocnf8j0QVo43hQKrbchg==", + "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=", "dev": true, "requires": { "resolve-from": "^3.0.0" @@ -23806,7 +23823,7 @@ "resolve-from": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz", - "integrity": "sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==", + "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=", "dev": true }, "semver": { @@ -24358,7 +24375,7 @@ "yeast": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz", - "integrity": "sha1-AI4G2AlDIMNy28L47XagymyKxBk=" + "integrity": "sha512-8HFIh676uyGYP6wP13R/j6OJ/1HwJ46snpvzE7aHAN3Ryqh2yX6Xox2B4CUmTwwOIzlG3Bs7ocsP5dZH/R1Qbg==" }, "yn": { "version": "3.1.1", diff --git a/package.json b/package.json index 70dcc1845..a812829ff 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@types/bcrypt-nodejs": "0.0.30", "@types/bluebird": "^3.5.36", "@types/body-parser": "^1.19.2", + "@types/brotli": "^1.3.1", "@types/chai": "^4.3.0", "@types/color": "^3.0.3", "@types/connect-flash": "0.0.34", @@ -171,6 +172,7 @@ "bluebird": "^3.7.2", "body-parser": "^1.19.2", "bootstrap": "^4.6.1", + "brotli": "^1.3.3", "browndash-components": "^0.0.28", "browser-assert": "^1.2.1", "bson": "^4.6.1", diff --git a/src/client/views/nodes/WebBoxRenderer.js b/src/client/views/nodes/WebBoxRenderer.js index 20554b858..eb8064780 100644 --- a/src/client/views/nodes/WebBoxRenderer.js +++ b/src/client/views/nodes/WebBoxRenderer.js @@ -42,6 +42,8 @@ var ForeignHtmlRenderer = function (styleSheets) { url = CorsProxy(new URL(webUrl).origin + inurl); } else if (!inurl.startsWith('http') && !inurl.startsWith('//')) { url = CorsProxy(webUrl + '/' + inurl); + } else if (inurl.startsWith('https')) { + url = CorsProxy(inurl); } xhr.open('GET', url); xhr.responseType = 'blob'; @@ -124,7 +126,7 @@ var ForeignHtmlRenderer = function (styleSheets) { if (url === null) { break; } - searchStartIndex = url.foundAtIndex + url.value.length; + searchStartIndex = url.foundAtIndex + url.value.length + 1; if (mustEndWithQuote && url.value[url.value.length - 1] !== '"') continue; const unquoted = removeQuotes(url.value); if (!unquoted /* || (!unquoted.startsWith('http')&& !unquoted.startsWith("/") )*/ || unquoted === 'http://' || unquoted === 'https://') { diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index b0db71f9c..805da1d43 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -22,6 +22,7 @@ import { Database } from './database'; import RouteManager from './RouteManager'; import RouteSubscriber from './RouteSubscriber'; import { WebSocket } from './websocket'; +import brotli = require('brotli'); import expressFlash = require('express-flash'); import flash = require('connect-flash'); const MongoStore = require('connect-mongo')(session); @@ -171,31 +172,29 @@ function registerCorsProxy(server: express.Express) { function proxyServe(req: any, requrl: string, response: any) { const htmlBodyMemoryStream = new (require('memorystream'))(); var retrieveHTTPBody: any; + var wasinBrFormat = false; const sendModifiedBody = () => { const header = response.headers['content-encoding']; + const httpsToCors = (match: any, href: string, offset: any, string: any) => `href="${resolvedServerUrl + '/corsProxy/http' + href}"`; if (header?.includes('gzip')) { try { - const replacer = (match: any, href: string, offset: any, string: any) => { - return `href="${resolvedServerUrl + '/corsProxy/http' + href}"`; - }; - const zipToStringDecoder = new (require('string_decoder').StringDecoder)('utf8'); const bodyStream = htmlBodyMemoryStream.read(); if (bodyStream) { - const htmlText = zipToStringDecoder.write( - zlib - .gunzipSync(bodyStream) - .toString('utf8') - .replace('', ' ') - .replace(/href="https?([^"]*)"/g, replacer) - .replace(/target="_blank"/g, '') - ); + const htmlInputText = wasinBrFormat ? Buffer.from(brotli.decompress(bodyStream)) : zlib.gunzipSync(bodyStream); + const htmlText = htmlInputText + .toString('utf8') + .replace('', ' ') + .replace(/href="https?([^"]*)"/g, httpsToCors) + .replace(/data-srcset="[^"]*"/g, '') + .replace(/srcset="[^"]*"/g, '') + .replace(/target="_blank"/g, ''); response.send(zlib.gzipSync(htmlText)); } else { req.pipe(request(requrl)).pipe(response); console.log('EMPTY body:' + req.url); } } catch (e) { - console.log('EROR?: ', e); + console.log('ERROR?: ', e); } } else { req.pipe(htmlBodyMemoryStream).pipe(response); @@ -216,6 +215,10 @@ function proxyServe(req: any, requrl: string, response: any) { } else if (headerCharRegex.test(header || '')) { delete res.headers[headerName]; } else res.headers[headerName] = header; + if (headerName === 'content-encoding') { + wasinBrFormat = res.headers[headerName] === 'br'; + res.headers[headerName] = 'gzip'; + } }); res.headers['x-permitted-cross-domain-policies'] = 'all'; res.headers['x-frame-options'] = ''; -- cgit v1.2.3-70-g09d2