From e6d1dd82f5d472bddaf66f9210142249d6fa09aa Mon Sep 17 00:00:00 2001 From: Naafiyan Ahmed Date: Wed, 6 Jul 2022 20:21:04 -0400 Subject: added endpoint routes for csv and integrated chartjs --- src/server/ApiManagers/DataVizManager.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/server/ApiManagers/DataVizManager.ts (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts new file mode 100644 index 000000000..0d43130d1 --- /dev/null +++ b/src/server/ApiManagers/DataVizManager.ts @@ -0,0 +1,26 @@ +import { csvParser, csvToString } from "../DataVizUtils"; +import { Method, _success } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Directory, serverPathToFile } from "./UploadManager"; +import * as path from 'path'; + +export default class DataVizManager extends ApiManager { + protected initialize(register: Registration): void { + register({ + method: Method.GET, + subscription: "/csvData", + secureHandler: async ({ req, res }) => { + const uri = req.query.uri as string; + + return new Promise(resolve => { + const name = path.basename(uri); + const sPath = serverPathToFile(Directory.csv, name); + const parsedCsv = csvParser(csvToString(sPath)); + _success(res, parsedCsv); + resolve(); + }); + } + }); + } + +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 0f3f5fa7f63dddbfbf095f65b05f89cf27cbc6cf Mon Sep 17 00:00:00 2001 From: jameshu111 Date: Mon, 6 Mar 2023 16:14:57 -0500 Subject: /stats handler --- package-lock.json | 169 ++++++++++++++++++---------------- package.json | 2 + src/server/ApiManagers/UserManager.ts | 1 + src/server/DashStats.ts | 70 ++++++++++++++ src/server/index.ts | 7 ++ src/server/stats/userLoginStats.csv | 2 + src/server/websocket.ts | 17 +++- 7 files changed, 188 insertions(+), 80 deletions(-) create mode 100644 src/server/DashStats.ts create mode 100644 src/server/stats/userLoginStats.csv (limited to 'src/server/ApiManagers') diff --git a/package-lock.json b/package-lock.json index 4695adf40..a5f1324a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5318,6 +5318,19 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.6.21.tgz", "integrity": "sha512-Z1PhmomIfypOpoMjRQB70jfvy/wxT50qW08YXO5lMIJkrdq4yOTR+AW7FqutScmB9NkLwxo+jU+kZLbofZZq/w==" }, + "csv-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz", + "integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==", + "requires": { + "minimist": "^1.2.0" + } + }, + "csv-stringify": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.3.0.tgz", + "integrity": "sha512-kTnnBkkLmAR1G409aUdShppWUClNbBQZXhrKrXzKYBGw4yfROspiFvVmjbKonCrdGfwnqwMXKLQG7ej7K/jwjg==" + }, "currently-unhandled": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", @@ -13650,7 +13663,7 @@ "dependencies": { "@iarna/cli": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@iarna/cli/-/cli-2.1.0.tgz", + "resolved": false, "integrity": "sha512-rvVVqDa2g860niRbqs3D5RhL4la3dc1vwk+NlpKPZxKaMSHtE2se6C2x8NeveN+rcjp3/686X+u+09CZ+7lmAQ==", "requires": { "glob": "^7.1.2", @@ -13753,7 +13766,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -13768,7 +13781,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -13777,12 +13790,12 @@ }, "asap": { "version": "2.0.6", - "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "resolved": false, "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "asn1": { "version": "0.2.6", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "resolved": false, "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", "requires": { "safer-buffer": "~2.1.0" @@ -13790,7 +13803,7 @@ }, "assert-plus": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "resolved": false, "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "asynckit": { @@ -13800,22 +13813,22 @@ }, "aws-sign2": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "resolved": false, "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" }, "aws4": { "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", + "resolved": false, "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "balanced-match": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "resolved": false, "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "bcrypt-pbkdf": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "resolved": false, "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", "requires": { "tweetnacl": "^0.14.3" @@ -13836,7 +13849,7 @@ }, "bluebird": { "version": "3.7.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "resolved": false, "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "boxen": { @@ -13884,7 +13897,7 @@ }, "cacache": { "version": "12.0.4", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", + "resolved": false, "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", "requires": { "bluebird": "^3.5.5", @@ -13921,7 +13934,7 @@ }, "caseless": { "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "resolved": false, "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, "chalk": { @@ -14065,7 +14078,7 @@ }, "combined-stream": { "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "resolved": false, "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "requires": { "delayed-stream": "~1.0.0" @@ -14073,7 +14086,7 @@ }, "concat-map": { "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "resolved": false, "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "concat-stream": { @@ -14103,7 +14116,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14118,7 +14131,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14127,7 +14140,7 @@ }, "config-chain": { "version": "1.1.13", - "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "resolved": false, "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", "requires": { "ini": "^1.3.4", @@ -14228,7 +14241,7 @@ }, "dashdash": { "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "resolved": false, "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", "requires": { "assert-plus": "^1.0.0" @@ -14261,7 +14274,7 @@ }, "decode-uri-component": { "version": "0.2.2", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "resolved": false, "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==" }, "deep-extend": { @@ -14307,7 +14320,7 @@ }, "dezalgo": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "resolved": false, "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", "requires": { "asap": "^2.0.0", @@ -14359,7 +14372,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14374,7 +14387,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14383,7 +14396,7 @@ }, "ecc-jsbn": { "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "resolved": false, "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", "requires": { "jsbn": "~0.1.0", @@ -14418,7 +14431,7 @@ }, "env-paths": { "version": "2.2.1", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "resolved": false, "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==" }, "err-code": { @@ -14502,7 +14515,7 @@ }, "extsprintf": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "resolved": false, "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" }, "fast-json-stable-stringify": { @@ -14512,12 +14525,12 @@ }, "figgy-pudding": { "version": "3.5.2", - "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz", + "resolved": false, "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==" }, "filter-obj": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "resolved": false, "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==" }, "find-npm-prefix": { @@ -14550,7 +14563,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14565,7 +14578,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14574,12 +14587,12 @@ }, "forever-agent": { "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "resolved": false, "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" }, "form-data": { "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "resolved": false, "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", "requires": { "asynckit": "^0.4.0", @@ -14612,7 +14625,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14627,7 +14640,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14695,7 +14708,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14710,7 +14723,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -14719,7 +14732,7 @@ }, "fs.realpath": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "resolved": false, "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "function-bind": { @@ -14809,7 +14822,7 @@ }, "getpass": { "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "resolved": false, "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", "requires": { "assert-plus": "^1.0.0" @@ -14817,7 +14830,7 @@ }, "glob": { "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "resolved": false, "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "requires": { "fs.realpath": "^1.0.0", @@ -14830,7 +14843,7 @@ "dependencies": { "minimatch": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "resolved": false, "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" @@ -14873,12 +14886,12 @@ }, "graceful-fs": { "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "resolved": false, "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "har-schema": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "resolved": false, "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==" }, "har-validator": { @@ -14957,7 +14970,7 @@ }, "http-signature": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "resolved": false, "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", "requires": { "assert-plus": "^1.0.0", @@ -15084,7 +15097,7 @@ }, "is-cidr": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/is-cidr/-/is-cidr-3.1.1.tgz", + "resolved": false, "integrity": "sha512-Gx+oErgq1j2jAKCR2Kbq0b3wbH0vQKqZ0wOlHxm0o56nq51Cs/DZA8oz9dMDhbHyHEGgJ86eTeVudtgMMOx3Mw==", "requires": { "cidr-regex": "^2.0.10" @@ -15163,7 +15176,7 @@ }, "is-typedarray": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "resolved": false, "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" }, "isarray": { @@ -15178,12 +15191,12 @@ }, "isstream": { "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "resolved": false, "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "jsbn": { "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "resolved": false, "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" }, "json-parse-better-errors": { @@ -15193,7 +15206,7 @@ }, "json-parse-even-better-errors": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "resolved": false, "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" }, "json-schema": { @@ -15203,7 +15216,7 @@ }, "json-stringify-safe": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "resolved": false, "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, "jsonparse": { @@ -15421,7 +15434,7 @@ }, "lock-verify": { "version": "2.2.2", - "resolved": "https://registry.npmjs.org/lock-verify/-/lock-verify-2.2.2.tgz", + "resolved": false, "integrity": "sha512-2CUNtr1ZSVKJHcYP8uEzafmmuyauCB5zZimj8TvQd/Lflt9kXVZs+8S+EbAzZLaVUDn8CYGmeC3DFGdYfnCzeQ==", "requires": { "@iarna/cli": "^2.1.0", @@ -15550,7 +15563,7 @@ }, "meant": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/meant/-/meant-1.0.3.tgz", + "resolved": false, "integrity": "sha512-88ZRGcNxAq4EH38cQ4D85PM57pikCwS8Z99EWHODxN7KBY+UuPiqzRTtZzS8KTXO/ywSWbdjjJST2Hly/EQxLw==" }, "mime-db": { @@ -15568,7 +15581,7 @@ }, "minimatch": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "resolved": false, "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" @@ -15617,7 +15630,7 @@ }, "mkdirp": { "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "resolved": false, "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "requires": { "minimist": "^1.2.6" @@ -15665,7 +15678,7 @@ }, "node-gyp": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-5.1.1.tgz", + "resolved": false, "integrity": "sha512-WH0WKGi+a4i4DUt2mHnvocex/xPLp9pYt5R6M2JdFB7pJ7Z34hveZ4nDTGTiLXCkitA9T8HFZjhinBCiVHYcWw==", "requires": { "env-paths": "^2.2.0", @@ -16003,7 +16016,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16018,7 +16031,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16032,7 +16045,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": false, "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, "path-is-inside": { @@ -16052,7 +16065,7 @@ }, "performance-now": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "resolved": false, "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "pify": { @@ -16101,7 +16114,7 @@ }, "proto-list": { "version": "1.2.4", - "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "resolved": false, "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==" }, "protoduck": { @@ -16124,7 +16137,7 @@ }, "psl": { "version": "1.9.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "resolved": false, "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, "pump": { @@ -16164,12 +16177,12 @@ }, "qs": { "version": "6.5.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", + "resolved": false, "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==" }, "query-string": { "version": "6.14.1", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.1.tgz", + "resolved": false, "integrity": "sha512-XDxAeVmpfu1/6IjyT/gXHOl+S0vQ9owggJ30hhWKdHAsNPOcasn5o9BW0eejZqL2e4vMjhAxoW3jVHcD6mbcYw==", "requires": { "decode-uri-component": "^0.2.0", @@ -16180,7 +16193,7 @@ }, "qw": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/qw/-/qw-1.0.2.tgz", + "resolved": false, "integrity": "sha512-1PhZ/iLKwlVNq45dnerTMKFjMof49uqli7/0QsvPNbX5OJ3IZ8msa9lUpvPheVdP+IYYPrf6cOaVil7S35joVA==" }, "rc": { @@ -16226,7 +16239,7 @@ }, "read-package-json": { "version": "2.1.2", - "resolved": "https://registry.npmjs.org/read-package-json/-/read-package-json-2.1.2.tgz", + "resolved": false, "integrity": "sha512-D1KmuLQr6ZSJS0tW8hf3WGpRlwszJOXZ3E8Yd/DNRaM5d+1wVRZdHlpGBLAuovjr28LbWvjpWkBHMxpRGGjzNA==", "requires": { "glob": "^7.1.1", @@ -16285,7 +16298,7 @@ }, "request": { "version": "2.88.2", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "resolved": false, "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "requires": { "aws-sign2": "~0.7.0", @@ -16355,7 +16368,7 @@ }, "safe-buffer": { "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "resolved": false, "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safer-buffer": { @@ -16526,7 +16539,7 @@ }, "sshpk": { "version": "1.17.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", + "resolved": false, "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==", "requires": { "asn1": "~0.2.3", @@ -16582,7 +16595,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16597,7 +16610,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16611,7 +16624,7 @@ }, "strict-uri-encode": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "resolved": false, "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==" }, "string-width": { @@ -16767,7 +16780,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16782,7 +16795,7 @@ "dependencies": { "safe-buffer": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "resolved": false, "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } @@ -16801,7 +16814,7 @@ }, "tough-cookie": { "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "resolved": false, "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "requires": { "psl": "^1.1.28", @@ -16810,7 +16823,7 @@ "dependencies": { "punycode": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "resolved": false, "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" } } @@ -16825,7 +16838,7 @@ }, "tweetnacl": { "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "resolved": false, "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "typedarray": { @@ -16896,7 +16909,7 @@ }, "uri-js": { "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "resolved": false, "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "requires": { "punycode": "^2.1.0" @@ -16937,7 +16950,7 @@ }, "uuid": { "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "resolved": false, "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" }, "validate-npm-package-license": { @@ -16959,7 +16972,7 @@ }, "verror": { "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "resolved": false, "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", "requires": { "assert-plus": "^1.0.0", diff --git a/package.json b/package.json index 00ce356f9..7fa90e431 100644 --- a/package.json +++ b/package.json @@ -184,6 +184,8 @@ "cookie-session": "^2.0.0", "core-js": "^3.28.0", "cors": "^2.8.5", + "csv-parser": "^3.0.0", + "csv-stringify": "^6.3.0", "depcheck": "^0.9.2", "equation-editor-react": "github:bobzel/equation-editor-react#useLocally", "exif": "^0.6.0", diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 53e55c1c3..c3dadd821 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -5,6 +5,7 @@ import { msToTime } from '../ActionUtilities'; import * as bcrypt from 'bcrypt-nodejs'; import { Opt } from '../../fields/Doc'; import { WebSocket } from '../websocket'; +import { DashStats } from '../DashStats'; export const timeMap: { [id: string]: number } = {}; interface ActivityUnit { diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts new file mode 100644 index 000000000..13ee61b3a --- /dev/null +++ b/src/server/DashStats.ts @@ -0,0 +1,70 @@ +import { magenta } from 'colors'; +import { Request, Response } from 'express'; +import SocketIO from 'socket.io'; +import { Client } from './Client'; +import { WebSocket } from './websocket'; +const fs = require('fs'); +const csv = require('csv-parser'); +import { stringify } from 'csv-stringify/sync'; + +export namespace DashStats { + const statsCSVFilename = './src/server/stats/userLoginStats.csv'; + const columns = ['USERNAME', 'ACTION', 'TIME']; + + interface SocketPair { + socketId: string; + username: string; + } + + interface CSVStore { + USERNAME: string; + ACTION: string; + TIME: string; + } + + export function handleStatsView(res: Response) { + const results: CSVStore[] = []; + fs.createReadStream(statsCSVFilename) + .pipe(csv(columns)) + .on('data', (data: any) => results.push(data)) + .on('end', () => { + console.log(results); + }); + + // let newRow = stringify([{ USERNAME: 'hi', ACTION: 'hi', TIME: 'hi' }], { header: true, columns: columns }); + // console.log(newRow); + + let current = getCurrentConnections(); + res.json({ + message: 'welcome to stats', + currentConnections: current.length, + socketMap: current, + }); + } + + export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { + if (!(username === undefined)) { + let currentDate = new Date(); + console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + console.log('stringify -> '); + } + } + + export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { + if (!(username === undefined)) { + let currentDate = new Date(); + console.log(magenta(`User ${username.split(' ')[0]} logged out at: ${currentDate.toISOString()}`)); + } + } + + function getCurrentConnections(): SocketPair[] { + let socketPairs: SocketPair[] = []; + for (let [key, value] of WebSocket.socketMap) { + if (!key.disconnected) { + socketPairs.push({ socketId: key.id, username: value.split(' ')[0] }); + } + } + console.log(socketPairs); + return socketPairs; + } +} diff --git a/src/server/index.ts b/src/server/index.ts index 6e6bde3cb..0848d828e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -18,6 +18,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { DashSessionAgent } from './DashSession/DashSessionAgent'; import { AppliedSessionAgent } from './DashSession/Session/agents/applied_session_agent'; +import { DashStats } from './DashStats'; import { DashUploadUtils } from './DashUploadUtils'; import { Database } from './database'; import { Logger } from './ProcessFactory'; @@ -83,6 +84,12 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ res }) => res.send(true), }); + addSupervisedRoute({ + method: Method.GET, + subscription: '/stats', + secureHandler: ({ res }) => DashStats.handleStatsView(res), + }); + addSupervisedRoute({ method: Method.GET, subscription: '/resolvedPorts', diff --git a/src/server/stats/userLoginStats.csv b/src/server/stats/userLoginStats.csv new file mode 100644 index 000000000..ec33b01b2 --- /dev/null +++ b/src/server/stats/userLoginStats.csv @@ -0,0 +1,2 @@ +USERNAME,ACTION,TIME +boo15869,loggedIn,2023-01 \ No newline at end of file diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 9b91a35a6..e556ecc17 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -1,4 +1,4 @@ -import { blue } from 'colors'; +import { blue, magenta } from 'colors'; import * as express from 'express'; import { createServer, Server } from 'https'; import { networkInterfaces } from 'os'; @@ -12,6 +12,7 @@ import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; import YoutubeApi from './apis/youtube/youtubeApiSample'; import { initializeGuest } from './authentication/DashUserModel'; import { Client } from './Client'; +import { DashStats } from './DashStats'; import { Database } from './database'; import { DocumentsCollection } from './IDatabase'; import { Diff, GestureContent, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, Transferable, Types, UpdateMobileInkOverlayPositionContent, YoutubeQueryInput, YoutubeQueryTypes } from './Message'; @@ -20,7 +21,7 @@ import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { export let _socket: Socket; - const clients: { [key: string]: Client } = {}; + export const clients: { [key: string]: Client } = {}; export const socketMap = new Map(); export let disconnect: Function; @@ -97,6 +98,13 @@ export namespace WebSocket { console.log('received bye'); }); + socket.on('disconnect', function () { + let currentUser = socketMap.get(socket); + if (!(currentUser === undefined)) { + DashStats.logUserLogout(socketMap.get(socket), socket); + } + }); + Utils.Emit(socket, MessageStore.Foo, 'handshooken'); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -176,7 +184,12 @@ export namespace WebSocket { const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); + console.log(magenta(`currently connected: ${JSON.stringify(clients)}`)); + // console.log(magenta('socket map below')); + // console.log([...socketMap.entries()]); + printActiveUsers(); socketMap.set(socket, userEmail + ' at ' + datetime); + DashStats.logUserLogin(socketMap.get(socket), socket); } function getField([id, callback]: [string, (result?: Transferable) => void]) { -- cgit v1.2.3-70-g09d2 From 0e55893d0f7f2a0aa5098df73d0ece5a7f1a4ddf Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 15 Mar 2023 22:33:22 -0400 Subject: fixed up Clone() and export/import collection to work with links, presentations, and contexts better. --- .eslintrc.json | 4 ++ package-lock.json | 78 +++++++++++++--------- package.json | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 6 +- src/client/views/nodes/DocumentView.tsx | 1 + src/client/views/nodes/trails/PresBox.tsx | 7 +- src/decycler/decycler.d.ts | 2 + src/decycler/decycler.js | 51 ++++++++++++++ src/fields/Doc.ts | 75 ++++++++++++--------- src/server/ApiManagers/UploadManager.ts | 24 +++---- tsconfig.json | 24 ++----- 11 files changed, 170 insertions(+), 104 deletions(-) create mode 100644 src/decycler/decycler.d.ts create mode 100644 src/decycler/decycler.js (limited to 'src/server/ApiManagers') diff --git a/.eslintrc.json b/.eslintrc.json index b9f8e1b7a..43bb53566 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -10,5 +10,9 @@ "object-shorthand": "off", "class-methods-use-this": "off", "single-quote": "off" + }, + "parserOptions": { + "ecmaVersion": 11, + "sourceType": "module" } } diff --git a/package-lock.json b/package-lock.json index 4695adf40..cc51ad9e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -590,15 +590,30 @@ "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz", "integrity": "sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA==" }, + "@eslint-community/eslint-utils": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.2.0.tgz", + "integrity": "sha512-gB8T4H4DEfX2IV9zGDJPOBgP1e/DbfCPDTtEqUMckpvzS1OYtva8JdFYBqMwYk7xAQ429WGF/UPqn8uQ//h2vQ==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^3.3.0" + } + }, + "@eslint-community/regexpp": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.4.0.tgz", + "integrity": "sha512-A9983Q0LnDGdLPjxyXQ00sbV+K+O+ko2Dr+CZigbHWtX9pNfxlaBkMR8X1CztI73zuEyEBXTVjx7CE+/VSwDiQ==", + "dev": true + }, "@eslint/eslintrc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz", - "integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.1.tgz", + "integrity": "sha512-eFRmABvW2E5Ho6f5fHLqgena46rOj7r7OKHYfLElqcBfGFHHpjBhivyi5+jOEQuSpdc/1phIZJlbC2te+tZNIw==", "dev": true, "requires": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.4.0", + "espree": "^9.5.0", "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", @@ -660,6 +675,12 @@ } } }, + "@eslint/js": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.36.0.tgz", + "integrity": "sha512-lxJ9R5ygVm8ZWgYdUweoq5ownDlJ4upvoWmO4eLxBYHdMo+vZ/Rx0EN6MbKWDJOSUGrqJy2Gt+Dyv/VKml0fjg==", + "dev": true + }, "@ffmpeg/core": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz", @@ -6683,12 +6704,15 @@ } }, "eslint": { - "version": "8.34.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.34.0.tgz", - "integrity": "sha512-1Z8iFsucw+7kSqXNZVslXS8Ioa4u2KM7GPwuKtkTFAqZ/cHMcEaR+1+Br0wLlot49cNxIiZk5wp8EAbPcYZxTg==", + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.36.0.tgz", + "integrity": "sha512-Y956lmS7vDqomxlaaQAHVmeb4tNMp2FWIvU/RnU5BD3IKMD/MJPr76xdyr68P8tV1iNMvN2mRK0yy3c+UjL+bw==", "dev": true, "requires": { - "@eslint/eslintrc": "^1.4.1", + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.4.0", + "@eslint/eslintrc": "^2.0.1", + "@eslint/js": "8.36.0", "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", @@ -6699,10 +6723,9 @@ "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", "eslint-scope": "^7.1.1", - "eslint-utils": "^3.0.0", "eslint-visitor-keys": "^3.3.0", - "espree": "^9.4.0", - "esquery": "^1.4.0", + "espree": "^9.5.0", + "esquery": "^1.4.2", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", @@ -6723,7 +6746,6 @@ "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", - "regexpp": "^3.2.0", "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", "text-table": "^0.2.0" @@ -6811,6 +6833,15 @@ "estraverse": "^5.2.0" } }, + "esquery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "dev": true, + "requires": { + "estraverse": "^5.1.0" + } + }, "estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", @@ -7852,23 +7883,6 @@ "estraverse": "^4.1.1" } }, - "eslint-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", - "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^2.0.0" - }, - "dependencies": { - "eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true - } - } - }, "eslint-visitor-keys": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", @@ -7876,9 +7890,9 @@ "dev": true }, "espree": { - "version": "9.4.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", - "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.0.tgz", + "integrity": "sha512-JPbJGhKc47++oo4JkEoTe2wjy4fmMwvFpgJT9cQzmfXKp22Dr6Hf1tdCteLz1h0P3t+mGvWZ+4Uankvh8+c6zw==", "dev": true, "requires": { "acorn": "^8.8.0", diff --git a/package.json b/package.json index 00ce356f9..2c4c41917 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,7 @@ "cross-env": "^5.2.1", "css-loader": "^2.1.1", "dotenv": "^8.6.0", - "eslint": "^8.18.0", + "eslint": "^8.36.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-node": "^4.1.0", "eslint-config-prettier": "^8.5.0", diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 15d8144fc..f0c140ef1 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1796,11 +1796,7 @@ export class CollectionFreeFormView extends CollectionSubView Doc.Zip(this.props.Document) }); - moreItems.push({ description: 'Import exported collection', icon: 'upload', event: ({ x, y }) => this.importDocument(e.clientX, e.clientY) }); - } + moreItems.push({ description: 'Import exported collection', icon: 'upload', event: ({ x, y }) => this.importDocument(e.clientX, e.clientY) }); !mores && ContextMenu.Instance.addItem({ description: 'More...', subitems: moreItems, icon: 'eye' }); }; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 02af30d0c..b7a760c1e 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -1002,6 +1002,7 @@ export class DocumentViewInternal extends DocComponent Utils.CopyText(Doc.globalServerPath(this.props.Document)), icon: 'fingerprint' }); } + moreItems.push({ description: 'Export collection', icon: 'download', event: async () => Doc.Zip(this.props.Document) }); } if (this.props.removeDocument && !Doc.IsSystem(this.rootDoc) && Doc.ActiveDashboard !== this.props.Document) { diff --git a/src/client/views/nodes/trails/PresBox.tsx b/src/client/views/nodes/trails/PresBox.tsx index be40b3592..e79e7472a 100644 --- a/src/client/views/nodes/trails/PresBox.tsx +++ b/src/client/views/nodes/trails/PresBox.tsx @@ -570,9 +570,9 @@ export class PresBox extends ViewBoxBaseComponent() { } } else { if (bestTarget._panX !== activeItem.presPanX || bestTarget._panY !== activeItem.presPanY || bestTarget._viewScale !== activeItem.presViewScale) { - bestTarget._panX = activeItem.presPanX; - bestTarget._panY = activeItem.presPanY; - bestTarget._viewScale = activeItem.presViewScale; + bestTarget._panX = activeItem.presPanX ?? bestTarget._panX; + bestTarget._panY = activeItem.presPanY ?? bestTarget._panY; + bestTarget._viewScale = activeItem.presViewScale ?? bestTarget._viewScale; changed = true; } } @@ -717,6 +717,7 @@ export class PresBox extends ViewBoxBaseComponent() { zoomTime: activeItem.presMovement === PresMovement.Jump ? 0 : Math.min(Math.max(effect ? 750 : 500, (effect ? 0.2 : 1) * presTime), presTime), effect: activeItem, noSelect: true, + openLocation: OpenWhere.addLeft, anchorDoc: activeItem, easeFunc: StrCast(activeItem.presEaseFunc, 'ease') as any, zoomTextSelections: BoolCast(activeItem.presZoomText), diff --git a/src/decycler/decycler.d.ts b/src/decycler/decycler.d.ts new file mode 100644 index 000000000..84620f79c --- /dev/null +++ b/src/decycler/decycler.d.ts @@ -0,0 +1,2 @@ +export declare const decycle: Function; +export declare const retrocycle: Function; diff --git a/src/decycler/decycler.js b/src/decycler/decycler.js new file mode 100644 index 000000000..7fb8a45c7 --- /dev/null +++ b/src/decycler/decycler.js @@ -0,0 +1,51 @@ +/// +/// this is a modified copy of the npm project: https://www.npmjs.com/package/json-decycle +/// the original code is used as replacer when stringifying JSON objects that have cycles. +/// However, we want an additional replacer that stringifies Dash Fields and Docs in a custom way. +/// So this modified code allows for a custom replacer to be added to this object that will run before this replacer +/// + +const g = e => typeof e === 'object' && e != null && !(e instanceof Boolean) && !(e instanceof Date) && !(e instanceof Number) && !(e instanceof RegExp) && !(e instanceof String); +const b = e => String('#') + e.map(t => String(t).replace(/~/g, '~0').replace(/\//g, '~1')).join('/'); +// eslint-disable-next-line node/no-unsupported-features/es-syntax +export function decycle(replacer) { + const e = new WeakMap(); + return function (n, rr) { + const r = replacer(n, rr); + if (n !== '$ref' && g(r)) { + if (e.has(r)) return { $ref: b(e.get(r)) }; + e.set(r, [...(e.get(this) === undefined ? [] : e.get(this)), n]); + } + return r; + }; +} +// eslint-disable-next-line node/no-unsupported-features/es-syntax +export function retrocycle() { + const e = new WeakMap(); + const t = new WeakMap(); + const n = new Set(); + function r(o) { + const c = o.$ref.slice(1).split('/'); + let s; + let a = this; + // eslint-disable-next-line no-plusplus + for (let p = 0; p < c.length; p++) { + s = c[p].replace(/~1/g, '/').replace(/~0/g, '~'); + a = a[s]; + } + const f = e.get(o); + f[t.get(o)] = a; + } + return function (c, s) { + if (c === '$ref') n.add(this); + else if (g(s)) { + const f = c === '' && Object.keys(this).length === 1; + if (f) n.forEach(r, this); + else { + e.set(s, this); + t.set(s, c); + } + } + return s; + }; +} diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index de94ed5db..168e29dd5 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -10,6 +10,7 @@ import { scriptingGlobal, ScriptingGlobals } from '../client/util/ScriptingGloba import { SelectionManager } from '../client/util/SelectionManager'; import { afterDocDeserialize, autoObject, Deserializable, SerializationHelper } from '../client/util/SerializationHelper'; import { UndoManager } from '../client/util/UndoManager'; +import { decycle } from '../decycler/decycler'; import { DashColor, incrementTitleCopy, intersectRect, Utils } from '../Utils'; import { DateField } from './DateField'; import { Copy, HandleUpdate, Id, OnUpdate, Parent, Self, SelfProxy, ToScriptString, ToString, Update } from './FieldSymbols'; @@ -25,7 +26,6 @@ import { Cast, DocCast, FieldValue, NumCast, StrCast, ToConstructor } from './Ty import { AudioField, ImageField, MapField, PdfField, VideoField, WebField } from './URLField'; import { deleteProperty, GetEffectiveAcl, getField, getter, makeEditable, makeReadOnly, normalizeEmail, setter, SharingPermissions, updateFunction } from './util'; import JSZip = require('jszip'); - export namespace Field { export function toKeyValueString(doc: Doc, key: string): string { const onDelegate = Object.keys(doc).includes(key); @@ -701,22 +701,12 @@ export namespace Doc { return bestAlias ?? Doc.MakeAlias(doc); } - export async function makeClone( - doc: Doc, - cloneMap: Map, - linkMap: Map, - rtfs: { copy: Doc; key: string; field: RichTextField }[], - exclusions: string[], - topLevelExclusions: string[], - dontCreate: boolean, - asBranch: boolean - ): Promise { + export async function makeClone(doc: Doc, cloneMap: Map, linkMap: Map, rtfs: { copy: Doc; key: string; field: RichTextField }[], exclusions: string[], dontCreate: boolean, asBranch: boolean): Promise { if (Doc.IsBaseProto(doc)) return doc; if (cloneMap.get(doc[Id])) return cloneMap.get(doc[Id])!; - const copy = dontCreate ? (asBranch ? Cast(doc.branchMaster, Doc, null) || doc : doc) : new Doc(undefined, true); + const copy = dontCreate ? (asBranch ? Cast(doc.branchMaster, Doc, null) ?? doc : doc) : new Doc(undefined, true); cloneMap.set(doc[Id], copy); - const fieldExclusions = doc.type === DocumentType.MARKER ? exclusions.filter(ex => ex !== 'annotationOn') : exclusions; - const filter = [...fieldExclusions, ...topLevelExclusions, ...Cast(doc.cloneFieldFilter, listSpec('string'), [])]; + const filter = [...exclusions, ...Cast(doc.cloneFieldFilter, listSpec('string'), [])]; await Promise.all( Object.keys(doc).map(async key => { if (filter.includes(key)) return; @@ -727,10 +717,10 @@ export namespace Doc { const list = await Cast(doc[key], listSpec(Doc)); const docs = list && (await DocListCastAsync(list))?.filter(d => d instanceof Doc); if (docs !== undefined && docs.length) { - const clones = await Promise.all(docs.map(async d => Doc.makeClone(d, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch))); + const clones = await Promise.all(docs.map(async d => Doc.makeClone(d, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch))); !dontCreate && assignKey(new List(clones)); } else if (doc[key] instanceof Doc) { - assignKey(key.includes('layout[') ? undefined : key.startsWith('layout') ? (doc[key] as Doc) : await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch)); // reference documents except copy documents that are expanded template fields + assignKey(key.includes('layout[') ? undefined : key.startsWith('layout') ? (doc[key] as Doc) : await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); // reference documents except copy documents that are expanded template fields } else { !dontCreate && assignKey(ObjectField.MakeCopy(field)); if (field instanceof RichTextField) { @@ -740,13 +730,12 @@ export namespace Doc { } } }; - if (key === 'proto') { - if (doc[key] instanceof Doc) { - assignKey(await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch)); - } - } else if (key === 'anchor1' || key === 'anchor2') { - if (doc[key] instanceof Doc) { - assignKey(await Doc.makeClone(doc[key] as Doc, cloneMap, linkMap, rtfs, exclusions, [], true, asBranch)); + const docAtKey = doc[key]; + if (docAtKey instanceof Doc) { + if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || (key === 'proto' && cloneMap.has(doc[Id])) || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); + } else { + assignKey(docAtKey); } } else { if (field instanceof RefField) { @@ -765,8 +754,8 @@ export namespace Doc { }) ); for (const link of Array.from(doc[DirectLinksSym])) { - const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, [], dontCreate, asBranch); - linkMap.set(link, linkClone); + const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch); + linkMap.set(link[Id], linkClone); } if (!dontCreate) { Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); @@ -779,11 +768,29 @@ export namespace Doc { Doc.AddFileOrphan(copy); return copy; } + export function repairClone(doc: Doc, cloned: Doc[], visited: Set) { + if (visited.has(doc)) return; + visited.add(doc); + Object.keys(doc).map(key => { + const docAtKey = DocCast(doc[key]); + if (docAtKey && !Doc.IsSystem(docAtKey)) { + if (!cloned.includes(docAtKey)) { + doc[key] = undefined; + } else { + repairClone(docAtKey, cloned, visited); + } + } + }); + } export async function MakeClone(doc: Doc, dontCreate: boolean = false, asBranch = false, cloneMap: Map = new Map()) { - const linkMap = new Map(); + const linkMap = new Map(); const rtfMap: { copy: Doc; key: string; field: RichTextField }[] = []; - const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf', 'branches', 'branchOf'], ['context'], dontCreate, asBranch); - Array.from(linkMap.entries()).map((links: Doc[]) => LinkManager.Instance.addLink(links[1], true)); + const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf', 'branches', 'branchOf'], dontCreate, asBranch); + const repaired = new Set(); + const linkedDocs = Array.from(linkMap.values()); + const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; + clonedDocs.map(clone => Doc.repairClone(clone, Array.from(cloneMap.values()), repaired)); + linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true)); rtfMap.map(({ copy, key, field }) => { const replacer = (match: any, attr: string, id: string, offset: any, string: any) => { const mapped = cloneMap.get(id); @@ -797,7 +804,7 @@ export namespace Doc { const re = new RegExp(regex, 'g'); copy[key] = new RichTextField(field.Data.replace(/("textId":|"audioId":|"anchorId":)"([^"]+)"/g, replacer).replace(re, replacer2), field.Text); }); - return { clone: copy, map: cloneMap }; + return { clone: copy, map: cloneMap, linkMap }; } export async function Zip(doc: Doc) { @@ -806,9 +813,10 @@ export namespace Doc { // a.href = url; // a.download = `DocExport-${this.props.Document[Id]}.zip`; // a.click(); - const { clone, map } = await Doc.MakeClone(doc, true); + const { clone, map, linkMap } = await Doc.MakeClone(doc, true); + clone.LINKS = new List(Array.from(linkMap.values())); function replacer(key: any, value: any) { - if (['branchOf', 'cloneOf', 'context', 'cursors'].includes(key)) return undefined; + if (['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; else if (value instanceof Doc) { if (key !== 'field' && Number.isNaN(Number(key))) { const __fields = value[FieldsSym](); @@ -833,7 +841,7 @@ export namespace Doc { const docs: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const docString = JSON.stringify({ id: doc[Id], docs }, replacer); + const docString = JSON.stringify({ id: doc[Id], docs }, decycle(replacer)); const zip = new JSZip(); @@ -1520,7 +1528,8 @@ export namespace Doc { const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); if (json !== 'error') { - const doc = await DocServer.GetRefField(json); + const doc = DocCast(await DocServer.GetRefField(json)); + (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index fe4c475c9..9bacbd5c8 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -1,19 +1,19 @@ -import ApiManager, { Registration } from './ApiManager'; -import { Method, _success } from '../RouteManager'; import * as formidable from 'formidable'; -import v4 = require('uuid/v4'); -const AdmZip = require('adm-zip'); -import { extname, basename, dirname } from 'path'; import { createReadStream, createWriteStream, unlink, writeFile } from 'fs'; -import { publicDirectory, filesDirectory } from '..'; -import { Database } from '../database'; -import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; +import { basename, dirname, extname, normalize } from 'path'; import * as sharp from 'sharp'; -import { AcceptableMedia, Upload } from '../SharedMediaTypes'; -import { normalize } from 'path'; +import { filesDirectory, publicDirectory } from '..'; +import { retrocycle } from '../../decycler/decycler'; +import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; +import { Database } from '../database'; +import { Method, _success } from '../RouteManager'; import RouteSubscriber from '../RouteSubscriber'; -const imageDataUri = require('image-data-uri'); +import { AcceptableMedia, Upload } from '../SharedMediaTypes'; +import ApiManager, { Registration } from './ApiManager'; import { SolrManager } from './SearchManager'; +import v4 = require('uuid/v4'); +const AdmZip = require('adm-zip'); +const imageDataUri = require('image-data-uri'); const fs = require('fs'); export enum Directory { @@ -252,7 +252,7 @@ export default class UploadManager extends ApiManager { }); const json = zip.getEntry('doc.json'); try { - const data = JSON.parse(json.getData().toString('utf8')); + const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); const datadocs = data.docs; id = getId(data.id); const docs = Object.keys(datadocs).map(key => datadocs[key]); diff --git a/tsconfig.json b/tsconfig.json index 993ab13b9..bff9255db 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,34 +2,22 @@ "compilerOptions": { "target": "es5", "downlevelIteration": true, - // "module": "system", "removeComments": true, "experimentalDecorators": true, "allowSyntheticDefaultImports": true, + "moduleDetection": "auto", "strict": true, "jsx": "react", "allowJs": true, "sourceMap": true, "outDir": "dist", - "lib": [ - "dom", - "es2015" - ], - "typeRoots": [ - "node_modules/@types", - "./src/typings" - ], - "types": [ - "youtube", - "node" - ] + "lib": ["dom", "es2015"], + "typeRoots": ["node_modules/@types", "./src/typings"], + "types": ["youtube", "node"] }, // "exclude": [ // "node_modules", // "static" // ], - "typeRoots": [ - "./node_modules/@types", - "./src/typings" - ] -} \ No newline at end of file + "typeRoots": ["./node_modules/@types", "./src/typings"] +} -- cgit v1.2.3-70-g09d2 From d2bca182a311e95515bbff8fb378b29918fe99d7 Mon Sep 17 00:00:00 2001 From: geireann Date: Thu, 16 Mar 2023 12:00:59 -0400 Subject: fixed export/import collectoin --- src/fields/Doc.ts | 34 +++++++++++++++++++-------------- src/server/ApiManagers/UploadManager.ts | 4 +++- 2 files changed, 23 insertions(+), 15 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 168e29dd5..6543679ad 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -732,7 +732,7 @@ export namespace Doc { }; const docAtKey = doc[key]; if (docAtKey instanceof Doc) { - if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || (key === 'proto' && cloneMap.has(doc[Id])) || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || key === 'proto'|| ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); } else { assignKey(docAtKey); @@ -757,8 +757,8 @@ export namespace Doc { const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch); linkMap.set(link[Id], linkClone); } - if (!dontCreate) { Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); + if (!dontCreate) { asBranch ? (copy.branchOf = doc) : (copy.cloneOf = doc); if (!Doc.IsPrototype(copy)) { Doc.AddDocToList(doc, 'branches', Doc.GetProto(copy)); @@ -768,16 +768,19 @@ export namespace Doc { Doc.AddFileOrphan(copy); return copy; } - export function repairClone(doc: Doc, cloned: Doc[], visited: Set) { - if (visited.has(doc)) return; - visited.add(doc); - Object.keys(doc).map(key => { - const docAtKey = DocCast(doc[key]); + export function repairClone(clone: Doc, cloneMap: Map, visited: Set) { + if (visited.has(clone)) return; + visited.add(clone); + Object.keys(clone).filter(key => key !== "cloneOf").map(key => { + const docAtKey = DocCast(clone[key]); if (docAtKey && !Doc.IsSystem(docAtKey)) { - if (!cloned.includes(docAtKey)) { - doc[key] = undefined; + if (!Array.from(cloneMap.values()).includes(docAtKey)) { + if (cloneMap.has(docAtKey[Id])) { + clone[key] = cloneMap.get(docAtKey[Id]); + } + else clone[key] = undefined; } else { - repairClone(docAtKey, cloned, visited); + repairClone(docAtKey, cloneMap, visited); } } }); @@ -789,7 +792,7 @@ export namespace Doc { const repaired = new Set(); const linkedDocs = Array.from(linkMap.values()); const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; - clonedDocs.map(clone => Doc.repairClone(clone, Array.from(cloneMap.values()), repaired)); + clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true)); rtfMap.map(({ copy, key, field }) => { const replacer = (match: any, attr: string, id: string, offset: any, string: any) => { @@ -813,7 +816,7 @@ export namespace Doc { // a.href = url; // a.download = `DocExport-${this.props.Document[Id]}.zip`; // a.click(); - const { clone, map, linkMap } = await Doc.MakeClone(doc, true); + const { clone, map, linkMap } = await Doc.MakeClone(doc, false); clone.LINKS = new List(Array.from(linkMap.values())); function replacer(key: any, value: any) { if (['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; @@ -841,7 +844,7 @@ export namespace Doc { const docs: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const docString = JSON.stringify({ id: doc[Id], docs }, decycle(replacer)); + const docString = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); const zip = new JSZip(); @@ -1527,8 +1530,11 @@ export namespace Doc { formData.append('remap', 'true'); const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); + console.log(json) if (json !== 'error') { - const doc = DocCast(await DocServer.GetRefField(json)); + await DocServer.GetRefFields(json.docids as string[]); + const doc = DocCast(await DocServer.GetRefField(json.id)); + console.log("Doc = ", doc, doc?.title); (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); return doc; } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 9bacbd5c8..5da3dfd3f 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -228,6 +228,7 @@ export default class UploadManager extends ApiManager { form.parse(req, async (_err, fields, files) => { remap = fields.remap !== 'false'; let id: string = ''; + let docids: string[] = []; try { for (const name in files) { const f = files[name]; @@ -257,6 +258,7 @@ export default class UploadManager extends ApiManager { id = getId(data.id); const docs = Object.keys(datadocs).map(key => datadocs[key]); docs.forEach(mapFn); + docids = docs.map(doc => doc.id) await Promise.all( docs.map( (doc: any) => @@ -279,7 +281,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify(id || 'error')); + res.send(JSON.stringify({id, docids} || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From de0df48cba8e89256a3208fbadfd5afaaa9e22d3 Mon Sep 17 00:00:00 2001 From: bobzel Date: Thu, 16 Mar 2023 21:10:23 -0400 Subject: added importing and exporting of assets (images, pdfs, etc) to collection importer/exporter --- src/JSZipUtils.js | 142 ++++++++++++++++++++++++++++++++ src/fields/Doc.ts | 82 ++++++++++++------ src/server/ApiManagers/UploadManager.ts | 28 ++++--- 3 files changed, 216 insertions(+), 36 deletions(-) create mode 100644 src/JSZipUtils.js (limited to 'src/server/ApiManagers') diff --git a/src/JSZipUtils.js b/src/JSZipUtils.js new file mode 100644 index 000000000..5ce1bd471 --- /dev/null +++ b/src/JSZipUtils.js @@ -0,0 +1,142 @@ +var JSZipUtils = {}; +// just use the responseText with xhr1, response with xhr2. +// The transformation doesn't throw away high-order byte (with responseText) +// because JSZip handles that case. If not used with JSZip, you may need to +// do it, see https://developer.mozilla.org/En/Using_XMLHttpRequest#Handling_binary_data +JSZipUtils._getBinaryFromXHR = function (xhr) { + // for xhr.responseText, the 0xFF mask is applied by JSZip + return xhr.response || xhr.responseText; +}; + +// taken from jQuery +function createStandardXHR() { + try { + return new window.XMLHttpRequest(); + } catch (e) {} +} + +function createActiveXHR() { + try { + return new window.ActiveXObject('Microsoft.XMLHTTP'); + } catch (e) {} +} + +// Create the request object +var createXHR = + typeof window !== 'undefined' && window.ActiveXObject + ? /* Microsoft failed to properly + * implement the XMLHttpRequest in IE7 (can't request local files), + * so we use the ActiveXObject when it is available + * Additionally XMLHttpRequest can be disabled in IE7/IE8 so + * we need a fallback. + */ + function () { + return createStandardXHR() || createActiveXHR(); + } + : // For all other browsers, use the standard XMLHttpRequest object + createStandardXHR; + +/** + * @param {string} path The path to the resource to GET. + * @param {function|{callback: function, progress: function}} options + * @return {Promise|undefined} If no callback is passed then a promise is returned + */ +JSZipUtils.getBinaryContent = function (path, options) { + var promise, resolve, reject; + var callback; + + if (!options) { + options = {}; + } + + // backward compatible callback + if (typeof options === 'function') { + callback = options; + options = {}; + } else if (typeof options.callback === 'function') { + // callback inside options object + callback = options.callback; + } + + if (!callback && typeof Promise !== 'undefined') { + promise = new Promise(function (_resolve, _reject) { + resolve = _resolve; + reject = _reject; + }); + } else { + resolve = function (data) { + callback(null, data); + }; + reject = function (err) { + callback(err, null); + }; + } + + /* + * Here is the tricky part : getting the data. + * In firefox/chrome/opera/... setting the mimeType to 'text/plain; charset=x-user-defined' + * is enough, the result is in the standard xhr.responseText. + * cf https://developer.mozilla.org/En/XMLHttpRequest/Using_XMLHttpRequest#Receiving_binary_data_in_older_browsers + * In IE <= 9, we must use (the IE only) attribute responseBody + * (for binary data, its content is different from responseText). + * In IE 10, the 'charset=x-user-defined' trick doesn't work, only the + * responseType will work : + * http://msdn.microsoft.com/en-us/library/ie/hh673569%28v=vs.85%29.aspx#Binary_Object_upload_and_download + * + * I'd like to use jQuery to avoid this XHR madness, but it doesn't support + * the responseType attribute : http://bugs.jquery.com/ticket/11461 + */ + try { + var xhr = createXHR(); + + xhr.open('GET', path, true); + + // recent browsers + if ('responseType' in xhr) { + xhr.responseType = 'arraybuffer'; + } + + // older browser + if (xhr.overrideMimeType) { + xhr.overrideMimeType('text/plain; charset=x-user-defined'); + } + + xhr.onreadystatechange = function (event) { + // use `xhr` and not `this`... thanks IE + if (xhr.readyState === 4) { + if (xhr.status === 200 || xhr.status === 0) { + try { + resolve(JSZipUtils._getBinaryFromXHR(xhr)); + } catch (err) { + reject(new Error(err)); + } + } else { + reject(new Error('Ajax error for ' + path + ' : ' + this.status + ' ' + this.statusText)); + } + } + }; + + if (options.progress) { + xhr.onprogress = function (e) { + options.progress({ + path: path, + originalEvent: e, + percent: (e.loaded / e.total) * 100, + loaded: e.loaded, + total: e.total, + }); + }; + } + + xhr.send(); + } catch (e) { + reject(new Error(e), null); + } + + // returns a promise or undefined depending on whether a callback was + // provided + return promise; +}; + +// export +module.exports = JSZipUtils; diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 6543679ad..deda8aa1f 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -26,6 +26,7 @@ import { Cast, DocCast, FieldValue, NumCast, StrCast, ToConstructor } from './Ty import { AudioField, ImageField, MapField, PdfField, VideoField, WebField } from './URLField'; import { deleteProperty, GetEffectiveAcl, getField, getter, makeEditable, makeReadOnly, normalizeEmail, setter, SharingPermissions, updateFunction } from './util'; import JSZip = require('jszip'); +import * as JSZipUtils from '../JSZipUtils'; export namespace Field { export function toKeyValueString(doc: Doc, key: string): string { const onDelegate = Object.keys(doc).includes(key); @@ -732,7 +733,7 @@ export namespace Doc { }; const docAtKey = doc[key]; if (docAtKey instanceof Doc) { - if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || key === 'proto'|| ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + if (!Doc.IsSystem(docAtKey) && (key === 'annotationOn' || key === 'proto' || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch)); } else { assignKey(docAtKey); @@ -757,7 +758,7 @@ export namespace Doc { const linkClone = await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, dontCreate, asBranch); linkMap.set(link[Id], linkClone); } - Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); + Doc.SetInPlace(copy, 'title', (asBranch ? 'BRANCH: ' : 'CLONE: ') + doc.title, true); if (!dontCreate) { asBranch ? (copy.branchOf = doc) : (copy.cloneOf = doc); if (!Doc.IsPrototype(copy)) { @@ -768,22 +769,23 @@ export namespace Doc { Doc.AddFileOrphan(copy); return copy; } - export function repairClone(clone: Doc, cloneMap: Map, visited: Set) { + export function repairClone(clone: Doc, cloneMap: Map, visited: Set) { if (visited.has(clone)) return; visited.add(clone); - Object.keys(clone).filter(key => key !== "cloneOf").map(key => { - const docAtKey = DocCast(clone[key]); - if (docAtKey && !Doc.IsSystem(docAtKey)) { - if (!Array.from(cloneMap.values()).includes(docAtKey)) { - if (cloneMap.has(docAtKey[Id])) { - clone[key] = cloneMap.get(docAtKey[Id]); + Object.keys(clone) + .filter(key => key !== 'cloneOf') + .map(key => { + const docAtKey = DocCast(clone[key]); + if (docAtKey && !Doc.IsSystem(docAtKey)) { + if (!Array.from(cloneMap.values()).includes(docAtKey)) { + if (cloneMap.has(docAtKey[Id])) { + clone[key] = cloneMap.get(docAtKey[Id]); + } else clone[key] = undefined; + } else { + repairClone(docAtKey, cloneMap, visited); } - else clone[key] = undefined; - } else { - repairClone(docAtKey, cloneMap, visited); } - } - }); + }); } export async function MakeClone(doc: Doc, dontCreate: boolean = false, asBranch = false, cloneMap: Map = new Map()) { const linkMap = new Map(); @@ -818,8 +820,9 @@ export namespace Doc { // a.click(); const { clone, map, linkMap } = await Doc.MakeClone(doc, false); clone.LINKS = new List(Array.from(linkMap.values())); + const proms = [] as string[]; function replacer(key: any, value: any) { - if (['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; + if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; else if (value instanceof Doc) { if (key !== 'field' && Number.isNaN(Number(key))) { const __fields = value[FieldsSym](); @@ -829,9 +832,14 @@ export namespace Doc { } } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; - else if (value instanceof ImageField) return { url: value.url.href, __type: 'image' }; - else if (value instanceof PdfField) return { url: value.url.href, __type: 'pdf' }; - else if (value instanceof AudioField) return { url: value.url.href, __type: 'audio' }; + else if (value instanceof ImageField) { + const extension = value.url.href.replace(/.*\./, ''); + proms.push(value.url.href.replace('.' + extension, '_o.' + extension)); + return { url: value.url.href, __type: 'image' }; + } else if (value instanceof PdfField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'pdf' }; + } else if (value instanceof AudioField) return { url: value.url.href, __type: 'audio' }; else if (value instanceof VideoField) return { url: value.url.href, __type: 'video' }; else if (value instanceof WebField) return { url: value.url.href, __type: 'web' }; else if (value instanceof MapField) return { url: value.url.href, __type: 'map' }; @@ -846,6 +854,32 @@ export namespace Doc { Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); const docString = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); + let generateZIP = (proms: string[]) => { + var zip = new JSZip(); + var count = 0; + var zipFilename = 'dashExport.zip'; + + proms + .filter(url => url.startsWith(window.location.origin)) + .forEach((url, i) => { + var filename = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); + // loading a file and add it in a zip file + JSZipUtils.getBinaryContent(url, function (err: any, data: any) { + if (err) { + throw err; // or handle the error + } + zip.file(filename, data, { binary: true }); + count++; + if (count == proms.length) { + zip.file('doc.json', docString); + zip.generateAsync({ type: 'blob' }).then(function (content) { + saveAs(content, zipFilename); + }); + } + }); + }); + }; + generateZIP(proms); const zip = new JSZip(); zip.file('doc.json', docString); @@ -857,10 +891,10 @@ export namespace Doc { // img.file("smile.gif", imgData, {base64: true}); // Generate the zip file asynchronously - zip.generateAsync({ type: 'blob' }).then((content: any) => { - // Force down of the Zip file - saveAs(content, doc.title + '.zip'); // glr: Possibly change the name of the document to match the title? - }); + // zip.generateAsync({ type: 'blob' }).then((content: any) => { + // // Force down of the Zip file + // saveAs(content, doc.title + '.zip'); // glr: Possibly change the name of the document to match the title? + // }); } // // Determines whether the layout needs to be expanded (as a template). @@ -1530,12 +1564,12 @@ export namespace Doc { formData.append('remap', 'true'); const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); - console.log(json) + console.log(json); if (json !== 'error') { await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); - console.log("Doc = ", doc, doc?.title); (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); + doc.LINKS = undefined; return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 5da3dfd3f..6e28268a9 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -235,18 +235,22 @@ export default class UploadManager extends ApiManager { const path_2 = Array.isArray(f) ? '' : f.path; const zip = new AdmZip(path_2); zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith('files/')) return; - let directory = dirname(entry.entryName) + '/'; - const extension = extname(entry.entryName); - const base = basename(entry.entryName).split('.')[0]; + let entryName = entry.entryName.replace(/%%%/g, '/'); + if (!entryName.startsWith('files/')) { + return; + } + const extension = extname(entryName); + const pathname = publicDirectory + '/' + entry.entryName; + const targetname = publicDirectory + '/' + entryName; try { zip.extractEntryTo(entry.entryName, publicDirectory, true, false); - directory = '/' + directory; - - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_o' + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_s' + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_m' + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_l' + extension)); + createReadStream(pathname).pipe(createWriteStream(targetname)); + if (extension !== '.pdf') { + createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_s' + extension))); + createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_m' + extension))); + createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_l' + extension))); + } + unlink(pathname, () => {}); } catch (e) { console.log(e); } @@ -258,7 +262,7 @@ export default class UploadManager extends ApiManager { id = getId(data.id); const docs = Object.keys(datadocs).map(key => datadocs[key]); docs.forEach(mapFn); - docids = docs.map(doc => doc.id) + docids = docs.map(doc => doc.id); await Promise.all( docs.map( (doc: any) => @@ -281,7 +285,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify({id, docids} || 'error')); + res.send(JSON.stringify({ id, docids } || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From f83e5d34794ef675d4627ecef2ed7042b17b1b06 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:05:01 -0400 Subject: cleaning up zip/unzip of files --- src/client/views/collections/CollectionView.tsx | 13 ++- src/client/views/nodes/DocumentView.tsx | 2 +- .../views/nodes/formattedText/DashFieldView.tsx | 6 +- src/fields/Doc.ts | 128 +++++++++------------ src/server/ApiManagers/UploadManager.ts | 9 +- 5 files changed, 67 insertions(+), 91 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index cfb9310b6..790aa765d 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -156,12 +156,13 @@ export class CollectionView extends ViewBoxAnnotatableComponent { - const newRendition = Doc.MakeAlias(this.rootDoc); - newRendition._viewType = vtype; - this.props.addDocTab(newRendition, OpenWhere.addRight); - return newRendition; - }); + !Doc.noviceMode && + this.setupViewTypes('UI Controls...', vtype => { + const newRendition = Doc.MakeAlias(this.rootDoc); + newRendition._viewType = vtype; + this.props.addDocTab(newRendition, OpenWhere.addRight); + return newRendition; + }); const options = cm.findByDescription('Options...'); const optionItems = options && 'subitems' in options ? options.subitems : []; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 7107707d1..6686f142f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -693,7 +693,7 @@ export class DocumentViewInternal extends DocComponent this.props.addDocTab(templateDoc, OpenWhere.addRight), icon: 'eye' }); !appearance && appearanceItems.length && cm.addItem({ description: 'UI Controls...', subitems: appearanceItems, icon: 'compass' }); - if (!Doc.IsSystem(this.rootDoc) && this.rootDoc._viewType !== CollectionViewType.Docking && this.props.docViewPath().lastElement()?.rootDoc?._viewType !== CollectionViewType.Tree) { + if (!Doc.IsSystem(this.rootDoc) && this.rootDoc.type !== DocumentType.PRES && ![CollectionViewType.Docking, CollectionViewType.Tree].includes(this.rootDoc._viewType as any)) { const existingOnClick = cm.findByDescription('OnClick...'); const onClicks: ContextMenuProps[] = existingOnClick && 'subitems' in existingOnClick ? existingOnClick.subitems : []; diff --git a/src/client/views/nodes/formattedText/DashFieldView.tsx b/src/client/views/nodes/formattedText/DashFieldView.tsx index c43206629..72e8aedac 100644 --- a/src/client/views/nodes/formattedText/DashFieldView.tsx +++ b/src/client/views/nodes/formattedText/DashFieldView.tsx @@ -101,11 +101,7 @@ export class DashFieldViewInternal extends React.Component { - dashDoc instanceof Doc && (this._dashDoc = dashDoc); - }) - ); + DocServer.GetRefField(this.props.docId).then(action(dashDoc => dashDoc instanceof Doc && (this._dashDoc = dashDoc))); } else { this._dashDoc = this.props.tbox.rootDoc; } diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 6c808c145..c5af45262 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -705,7 +705,7 @@ export namespace Doc { if (cloneMap.get(doc[Id])) return cloneMap.get(doc[Id])!; const copy = new Doc(undefined, true); cloneMap.set(doc[Id], copy); - const filter = [...exclusions, ...Cast(doc.cloneFieldFilter, listSpec('string'), [])]; + const filter = [...exclusions, ...StrListCast(doc.cloneFieldFilter)]; await Promise.all( Object.keys(doc).map(async key => { if (filter.includes(key)) return; @@ -725,15 +725,13 @@ export namespace Doc { const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => '(' + exp + ')').join('|') + ')":"([a-z-A-Z0-9_]*)"', 'g'); const rawdocids = field.Data.match(docidsearch); const docids = rawdocids?.map((str: string) => - DocsInTextFieldIds.reduce((output, exp) => { - return output.replace(new RegExp(`${exp}":`, 'g'), ''); - }, str) + DocsInTextFieldIds.reduce((output, exp) => output.replace(new RegExp(`${exp}":`, 'g'), ''), str) .replace(/"/g, '') .trim() ); const results = docids && (await DocServer.GetRefFields(docids)); const docs = results && Array.from(Object.keys(results)).map(key => DocCast(results[key])); - docs && docs.map(doc => Doc.makeClone(doc, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); + docs?.map(doc => doc && Doc.makeClone(doc, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); rtfs.push({ copy, key, field }); } } @@ -741,7 +739,7 @@ export namespace Doc { }; const docAtKey = doc[key]; if (docAtKey instanceof Doc) { - if (!Doc.IsSystem(docAtKey) && (key.startsWith('layout') || key === 'annotationOn' || key === 'proto' || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { + if (!Doc.IsSystem(docAtKey) && (key.startsWith('layout') || ['context', 'annotationOn', 'proto'].includes(key) || ((key === 'anchor1' || key === 'anchor2') && doc.author === Doc.CurrentUserEmail))) { assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, cloneLinks)); } else { assignKey(docAtKey); @@ -749,8 +747,7 @@ export namespace Doc { } else if (field instanceof RefField) { assignKey(field); } else if (cfield instanceof ComputedField) { - assignKey(cfield[Copy]()); - // ComputedField.MakeFunction(cfield.script.originalScript)); + assignKey(cfield[Copy]()); // ComputedField.MakeFunction(cfield.script.originalScript)); } else if (field instanceof ObjectField) { await copyObjectField(field); } else if (field instanceof Promise) { @@ -804,8 +801,6 @@ export namespace Doc { const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf'], cloneLinks); const repaired = new Set(); const linkedDocs = Array.from(linkMap.values()); - const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; - clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true)); rtfMap.map(({ copy, key, field }) => { const replacer = (match: any, attr: string, id: string, offset: any, string: any) => { @@ -816,20 +811,16 @@ export namespace Doc { const mapped = cloneMap.get(id); return href + (mapped ? mapped[Id] : id); }; - const regex = `(${Doc.localServerPath()})([^"]*)`; - const re = new RegExp(regex, 'g'); + const re = new RegExp(`(${Doc.localServerPath()})([^"]*)`, 'g'); const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => `"${exp}":`).join('|') + ')"([^"]+)"', 'g'); copy[key] = new RichTextField(field.Data.replace(docidsearch, replacer).replace(re, replacer2), field.Text); }); + const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs]; + clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired)); return { clone: copy, map: cloneMap, linkMap }; } - export async function Zip(doc: Doc) { - // const a = document.createElement("a"); - // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); - // a.href = url; - // a.download = `DocExport-${this.props.Document[Id]}.zip`; - // a.click(); + export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') { const { clone, map, linkMap } = await Doc.MakeClone(doc); clone.LINKS = new List(Array.from(linkMap.values())); const proms = [] as string[]; @@ -837,76 +828,61 @@ export namespace Doc { if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; else if (value instanceof Doc) { if (key !== 'field' && Number.isNaN(Number(key))) { - const __fields = value[FieldsSym](); - return { id: value[Id], __type: 'Doc', fields: __fields }; - } else { - return { fieldId: value[Id], __type: 'proxy' }; + return { id: value[Id], __type: 'Doc', fields: value[FieldsSym]() }; } - } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; - else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; - else if (value instanceof ImageField) { + return { fieldId: value[Id], __type: 'proxy' }; + } else if (value instanceof ImageField) { const extension = value.url.href.replace(/.*\./, ''); proms.push(value.url.href.replace('.' + extension, '_o.' + extension)); return { url: value.url.href, __type: 'image' }; } else if (value instanceof PdfField) { proms.push(value.url.href); return { url: value.url.href, __type: 'pdf' }; - } else if (value instanceof AudioField) return { url: value.url.href, __type: 'audio' }; - else if (value instanceof VideoField) return { url: value.url.href, __type: 'video' }; + } else if (value instanceof AudioField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'audio' }; + } else if (value instanceof VideoField) { + proms.push(value.url.href); + return { url: value.url.href, __type: 'video' }; + } else if (value instanceof ScriptField) return { script: value.script, __type: 'script' }; + else if (value instanceof RichTextField) return { Data: value.Data, Text: value.Text, __type: 'RichTextField' }; else if (value instanceof WebField) return { url: value.url.href, __type: 'web' }; else if (value instanceof MapField) return { url: value.url.href, __type: 'map' }; else if (value instanceof DateField) return { date: value.toString(), __type: 'date' }; else if (value instanceof ProxyField) return { fieldId: value.fieldId, __type: 'proxy' }; else if (value instanceof Array && key !== 'fields') return { fields: value, __type: 'list' }; else if (value instanceof ComputedField) return { script: value.script, __type: 'computed' }; - else return value; + return value; } const docs: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const docString = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); - - let generateZIP = (proms: string[]) => { - var zip = new JSZip(); - var count = 0; - var zipFilename = 'dashExport.zip'; - - proms - .filter(url => url.startsWith(window.location.origin)) - .forEach((url, i) => { - var filename = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); - // loading a file and add it in a zip file - JSZipUtils.getBinaryContent(url, function (err: any, data: any) { - if (err) { - throw err; // or handle the error - } - zip.file(filename, data, { binary: true }); - count++; - if (count == proms.length) { - zip.file('doc.json', docString); - zip.generateAsync({ type: 'blob' }).then(function (content) { - saveAs(content, zipFilename); - }); - } - }); - }); - }; - generateZIP(proms); - const zip = new JSZip(); - - zip.file('doc.json', docString); - - // // Generate a directory within the Zip file structure - // var img = zip.folder("images"); - - // // Add a file to the directory, in this case an image with data URI as contents - // img.file("smile.gif", imgData, {base64: true}); + const jsonDocs = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); - // Generate the zip file asynchronously - zip.generateAsync({ type: 'blob' }).then((content: any) => { - // Force down of the Zip file - saveAs(content, doc.title + '.zip'); // glr: Possibly change the name of the document to match the title? - }); + const zip = new JSZip(); + var count = 0; + proms + .filter(url => url.startsWith(window.location.origin)) + .forEach((url, i) => { + // loading a file and add it in a zip file + JSZipUtils.getBinaryContent(url, (err: any, data: any) => { + if (err) throw err; // or handle the error + // // Generate a directory within the Zip file structure + // const assets = zip.folder("assets"); + // assets.file(filename, data, {binary: true}); + const assetPathOnServer = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); + zip.file(assetPathOnServer, data, { binary: true }); + if (++count == proms.length) { + zip.file('doc.json', jsonDocs); + zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename)); + // const a = document.createElement("a"); + // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); + // a.href = url; + // a.download = `DocExport-${this.props.Document[Id]}.zip`; + // a.click(); + } + }); + }); } const _pendingMap: Map = new Map(); @@ -963,7 +939,7 @@ export namespace Doc { // otherwise, it just returns the childDoc export function GetLayoutDataDocPair(containerDoc: Doc, containerDataDoc: Opt, childDoc: Doc) { if (!childDoc || childDoc instanceof Promise || !Doc.GetProto(childDoc)) { - console.log('No, no, no!'); + console.log('Warning: GetLayoutDataDocPair childDoc not defined'); return { layout: childDoc, data: childDoc }; } const resolvedDataDoc = Doc.AreProtosEqual(containerDataDoc, containerDoc) || (!childDoc.isTemplateDoc && !childDoc.isTemplateForField) ? undefined : containerDataDoc; @@ -1553,15 +1529,19 @@ export namespace Doc { } } - export async function importDocument(file: File) { + /// + // imports a previously exported zip file which contains a set of documents and their assets (eg, images, videos) + // the 'remap' parameter determines whether the ids of the documents loaded should be kept as they were, or remapped to new ids + // If they are not remapped, loading the file will overwrite any existing documents with those ids + // + export async function importDocument(file: File, remap = false) { const upload = Utils.prepend('/uploadDoc'); const formData = new FormData(); if (file) { formData.append('file', file); - formData.append('remap', 'true'); + formData.append('remap', remap.toString()); const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); - console.log(json); if (json !== 'error') { await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 6e28268a9..2cc2ac46c 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -184,9 +184,8 @@ export default class UploadManager extends ApiManager { if (id.endsWith('Proto')) return id; if (id in ids) { return ids[id]; - } else { - return (ids[id] = v4()); } + return (ids[id] = v4()); }; const mapFn = (doc: any) => { if (doc.id) { @@ -266,7 +265,7 @@ export default class UploadManager extends ApiManager { await Promise.all( docs.map( (doc: any) => - new Promise(res => { + new Promise(res => Database.Instance.replace( doc.id, doc, @@ -275,8 +274,8 @@ export default class UploadManager extends ApiManager { res(); }, true - ); - }) + ) + ) ) ); } catch (e) { -- cgit v1.2.3-70-g09d2 From 237e39029201aba23249de1ab5c405197acffe5a Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:50:28 -0400 Subject: more cleanup of import/export --- src/fields/Doc.ts | 15 +++++++++------ src/server/ApiManagers/UploadManager.ts | 34 +++++++++++++++------------------ 2 files changed, 24 insertions(+), 25 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index c5af45262..35f9a1032 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -822,7 +822,6 @@ export namespace Doc { export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') { const { clone, map, linkMap } = await Doc.MakeClone(doc); - clone.LINKS = new List(Array.from(linkMap.values())); const proms = [] as string[]; function replacer(key: any, value: any) { if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; @@ -856,8 +855,10 @@ export namespace Doc { } const docs: { [id: string]: any } = {}; + const links: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const jsonDocs = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); + Array.from(linkMap.entries()).forEach(l => (links[l[0]] = l[1])); + const jsonDocs = JSON.stringify({ id: clone[Id], docs, links }, decycle(replacer)); const zip = new JSZip(); var count = 0; @@ -873,7 +874,7 @@ export namespace Doc { const assetPathOnServer = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); zip.file(assetPathOnServer, data, { binary: true }); if (++count == proms.length) { - zip.file('doc.json', jsonDocs); + zip.file('docs.json', jsonDocs); zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename)); // const a = document.createElement("a"); // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); @@ -1543,10 +1544,12 @@ export namespace Doc { const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); if (json !== 'error') { - await DocServer.GetRefFields(json.docids as string[]); + const docs = await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); - (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); - doc.LINKS = undefined; + const links = await DocServer.GetRefFields(json.linkids as string[]); + Array.from(Object.keys(links)) + .map(key => links[key]) + .forEach(link => link instanceof Doc && LinkManager.Instance.addLink(link)); return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2cc2ac46c..1f72d72dc 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -228,6 +228,7 @@ export default class UploadManager extends ApiManager { remap = fields.remap !== 'false'; let id: string = ''; let docids: string[] = []; + let linkids: string[] = []; try { for (const name in files) { const f = files[name]; @@ -254,28 +255,23 @@ export default class UploadManager extends ApiManager { console.log(e); } }); - const json = zip.getEntry('doc.json'); + const json = zip.getEntry('docs.json'); try { const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); - const datadocs = data.docs; + const { docs, links } = data; id = getId(data.id); - const docs = Object.keys(datadocs).map(key => datadocs[key]); - docs.forEach(mapFn); - docids = docs.map(doc => doc.id); + const rdocs = Object.keys(docs).map(key => docs[key]); + const ldocs = Object.keys(links).map(key => links[key]); + [...rdocs, ...ldocs].forEach(mapFn); + docids = rdocs.map(doc => doc.id); + linkids = ldocs.map(link => link.id); await Promise.all( - docs.map( - (doc: any) => - new Promise(res => - Database.Instance.replace( - doc.id, - doc, - (err, r) => { - err && console.log(err); - res(); - }, - true - ) - ) + [...rdocs, ...ldocs].map( + doc => + new Promise(res => { + // overwrite mongo doc with json doc contents + Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true); + }) ) ); } catch (e) { @@ -284,7 +280,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify({ id, docids } || 'error')); + res.send(JSON.stringify({ id, docids, linkids } || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From 16a33f53c2a031f791ed8c98d539409e1a7c6bb4 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:52:43 -0400 Subject: from last --- src/server/ApiManagers/UploadManager.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 1f72d72dc..74c06b4a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -180,11 +180,8 @@ export default class UploadManager extends ApiManager { const ids: { [id: string]: string } = {}; let remap = true; const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith('Proto')) return id; - if (id in ids) { - return ids[id]; - } + if (!remap || id.endsWith('Proto')) return id; + if (id in ids) return ids[id]; return (ids[id] = v4()); }; const mapFn = (doc: any) => { -- cgit v1.2.3-70-g09d2