diff options
| author | mehekj <mehek.jethani@gmail.com> | 2022-04-12 18:11:13 -0400 |
|---|---|---|
| committer | mehekj <mehek.jethani@gmail.com> | 2022-04-12 18:11:13 -0400 |
| commit | 82272cc4735a9da78a1c4592bdf460f1ab6e644d (patch) | |
| tree | b2980350c566102048cee05d5d153d78a49a4d89 | |
| parent | b3424535cb746ff9fba35375d9abf07ba174dcf0 (diff) | |
| parent | ab0e1ac6f5b21f2e10bdce428c882482a0f159b4 (diff) | |
Merge branch 'inkocr-mehek' into inking-naafi-mehek
44 files changed, 2272 insertions, 2220 deletions
diff --git a/package-lock.json b/package-lock.json index 78b64b92a..1f8b1b8f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2044,7 +2044,6 @@ "version": "6.8.9", "resolved": "https://registry.npmjs.org/@types/react-table/-/react-table-6.8.9.tgz", "integrity": "sha512-fVQXjy/EYDbgraScgjDONA291McKqGrw0R0NeK639fx2bS4T19TnXMjg3FjOPlkI3qYTQtFTPADlRYysaQIMpA==", - "dev": true, "requires": { "@types/react": "*" } @@ -3456,11 +3455,6 @@ "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==" }, - "bindings": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz", - "integrity": "sha512-i47mqjF9UbjxJhxGf+pZ6kSxrnI3wBLlnGI2ArWJ4r0VrvDS7ZYXkprq/pLaBWYq4GM0r4zdHY+NNRqEMU7uew==" - }, "bl": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", @@ -4585,6 +4579,16 @@ "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=" }, + "clipboard": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-1.7.1.tgz", + "integrity": "sha1-Ng1taUbpmnof7zleQrqStem1oWs=", + "requires": { + "good-listener": "^1.2.2", + "select": "^1.1.2", + "tiny-emitter": "^2.0.0" + } + }, "cliss": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/cliss/-/cliss-0.0.2.tgz", @@ -5313,6 +5317,11 @@ } } }, + "crypto-js": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-3.3.0.tgz", + "integrity": "sha512-DIT51nX0dCfKltpRiXV+/TVZq+Qq2NgF4644+K7Ttnla7zEzqc+kjJyiB96BHNyUTBxyjzRcZYpUdZa+QAqi6Q==" + }, "crypto-random-string": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", @@ -5501,16 +5510,6 @@ "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=", "dev": true }, - "d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "dev": true, - "requires": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, "d3-array": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-1.2.4.tgz", @@ -5901,6 +5900,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "delegate": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -6526,8 +6530,8 @@ "integrity": "sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==" }, "equation-editor-react": { - "version": "github:bobzel/equation-editor-react#75915e852b4b36a6a4cd3e1cbc80598da6b65227", - "from": "github:bobzel/equation-editor-react#useLocally", + "version": "git+ssh://git@github.com/bobzel/equation-editor-react.git#75915e852b4b36a6a4cd3e1cbc80598da6b65227", + "from": "equation-editor-react@github:bobzel/equation-editor-react#useLocally", "requires": { "jquery": "^3.4.1", "mathquill": "^0.10.1-a" @@ -6585,14 +6589,14 @@ } }, "es5-ext": { - "version": "0.10.53", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", - "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", + "version": "0.10.59", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.59.tgz", + "integrity": "sha512-cOgyhW0tIJyQY1Kfw6Kr0viu9ZlUctVchRMZ7R0HiH3dxTSp5zJDLecwxUqPUrGKMsgBI1wd1FL+d9Jxfi4cLw==", "dev": true, "requires": { - "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.3", - "next-tick": "~1.0.0" + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.3", + "next-tick": "^1.1.0" } }, "es6-iterator": { @@ -6604,6 +6608,18 @@ "d": "1", "es5-ext": "^0.10.35", "es6-symbol": "^3.1.1" + }, + "dependencies": { + "d": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", + "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "dev": true, + "requires": { + "es5-ext": "^0.10.50", + "type": "^1.0.1" + } + } } }, "es6-promise": { @@ -6619,6 +6635,18 @@ "requires": { "d": "^1.0.1", "ext": "^1.1.2" + }, + "dependencies": { + "d": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", + "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "dev": true, + "requires": { + "es5-ext": "^0.10.50", + "type": "^1.0.1" + } + } } }, "escalade": { @@ -7501,14 +7529,6 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, - "fs-minipass": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", - "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", - "requires": { - "minipass": "^2.6.0" - } - }, "fs-monkey": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.3.tgz", @@ -7846,6 +7866,14 @@ "jquery": "*" } }, + "good-listener": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.2.tgz", + "integrity": "sha1-1TswzfkxPf+33JoNR3CWqm0UXFA=", + "requires": { + "delegate": "^3.1.2" + } + }, "google-auth-library": { "version": "4.2.6", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-4.2.6.tgz", @@ -8462,12 +8490,27 @@ "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=" }, - "ignore-walk": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", - "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "iink-js": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/iink-js/-/iink-js-1.5.4.tgz", + "integrity": "sha512-WcjMmyXbSo4GY56AVVTKRyxlguh2KAGFyoH24+Zmm87ZWII12or2uLiovAyOK4IE+VVz7m0U5RMuv8i/RV/3Nw==", "requires": { - "minimatch": "^3.0.4" + "@babel/runtime": "^7.9.2", + "clipboard": "^1.7.1", + "crypto-js": "^3.3.0", + "d3-selection": "^1.4.1", + "json-css": "^1.5.6", + "lodash.merge": "^4.6.2", + "loglevel": "^1.6.8", + "perfect-scrollbar": "^1.5.0", + "uuid-js": "^0.7.5" + }, + "dependencies": { + "d3-selection": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.4.2.tgz", + "integrity": "sha512-SJ0BqYihzOjDnnlfyeHT0e30k0K1+5sR3d5fNueCNeuhZTnGw4M4o8mqJchSwgKMXCNFo+e2VTChiSJ0vYtXkg==" + } } }, "image-data-uri": { @@ -9532,6 +9575,11 @@ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" }, + "json-css": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/json-css/-/json-css-1.5.6.tgz", + "integrity": "sha1-65ZPg0ouTqobwvaY/12wB6JsfAA=" + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -9819,43 +9867,6 @@ "type-check": "~0.3.2" } }, - "libxmljs": { - "version": "0.19.7", - "resolved": "https://registry.npmjs.org/libxmljs/-/libxmljs-0.19.7.tgz", - "integrity": "sha512-lFJyG9T1mVwTzNTw6ZkvIt0O+NsIR+FTE+RcC2QDFGU8YMnQrnyEOGrj6HWSe1AdwQK7s37BOp4NL+pcAqfK2g==", - "requires": { - "bindings": "~1.3.0", - "nan": "~2.14.0", - "node-pre-gyp": "~0.11.0" - }, - "dependencies": { - "node-pre-gyp": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", - "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", - "requires": { - "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.1", - "nopt": "^4.0.1", - "npm-packlist": "^1.1.6", - "npmlog": "^4.0.2", - "rc": "^1.2.7", - "rimraf": "^2.6.1", - "semver": "^5.3.0", - "tar": "^4" - } - }, - "rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "requires": { - "glob": "^7.1.3" - } - } - } - }, "lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -10041,8 +10052,7 @@ "loglevel": { "version": "1.6.8", "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.8.tgz", - "integrity": "sha512-bsU7+gc9AJ2SqpzxwU3+1fedl8zAntbtC5XYlt3s2j1hJcn2PsXSmgN8TaLG/J1/2mod4+cE/3vNL70/c1RNCA==", - "dev": true + "integrity": "sha512-bsU7+gc9AJ2SqpzxwU3+1fedl8zAntbtC5XYlt3s2j1hJcn2PsXSmgN8TaLG/J1/2mod4+cE/3vNL70/c1RNCA==" }, "loglevelnext": { "version": "1.0.5", @@ -10437,23 +10447,6 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, - "minipass": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", - "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", - "requires": { - "safe-buffer": "^5.1.2", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", - "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", - "requires": { - "minipass": "^2.9.0" - } - }, "mississippi": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-2.0.0.tgz", @@ -10865,16 +10858,6 @@ "resolved": "https://registry.npmjs.org/native-or-bluebird/-/native-or-bluebird-1.2.0.tgz", "integrity": "sha1-OcR7/Xgl0fuf+tMiEK4l2q3xAck=" }, - "needle": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/needle/-/needle-2.4.1.tgz", - "integrity": "sha512-x/gi6ijr4B7fwl6WYL9FwlCvRQKGlUNvnceho8wxkwXqN8jvVmmmATTmZPRRG7b/yC1eode26C2HO9jl78Du9g==", - "requires": { - "debug": "^3.2.6", - "iconv-lite": "^0.4.4", - "sax": "^1.2.4" - } - }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -10887,9 +10870,9 @@ "dev": true }, "next-tick": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" }, "nextafter": { "version": "1.0.0", @@ -11120,15 +11103,6 @@ "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=" }, - "nopt": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", - "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", - "requires": { - "abbrev": "1", - "osenv": "^0.1.4" - } - }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -14257,29 +14231,6 @@ } } }, - "npm-bundled": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", - "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", - "requires": { - "npm-normalize-package-bin": "^1.0.1" - } - }, - "npm-normalize-package-bin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", - "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" - }, - "npm-packlist": { - "version": "1.4.8", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz", - "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", - "requires": { - "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1", - "npm-normalize-package-bin": "^1.0.1" - } - }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -14897,6 +14848,11 @@ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, + "perfect-scrollbar": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/perfect-scrollbar/-/perfect-scrollbar-1.5.5.tgz", + "integrity": "sha512-dzalfutyP3e/FOpdlhVryN4AJ5XDVauVWxybSkLZmakFE2sS3y3pc4JnSprw8tGmHvkaG5Edr5T7LBTZ+WWU2g==" + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -16152,9 +16108,14 @@ } }, "react-table": { - "version": "7.7.0", - "resolved": "https://registry.npmjs.org/react-table/-/react-table-7.7.0.tgz", - "integrity": "sha512-jBlj70iBwOTvvImsU9t01LjFjy4sXEtclBovl3mTiqjz23Reu0DKnRza4zlLtOPACx6j2/7MrQIthIK1Wi+LIA==" + "version": "6.11.5", + "resolved": "https://registry.npmjs.org/react-table/-/react-table-6.11.5.tgz", + "integrity": "sha512-LM+AS9v//7Y7lAlgTWW/cW6Sn5VOb3EsSkKQfQTzOW8FngB1FUskLLNEVkAYsTX9LjOWR3QlGjykJqCE6eXT/g==", + "requires": { + "@types/react-table": "^6.8.5", + "classnames": "^2.2.5", + "react-is": "^16.8.1" + } }, "react-themeable": { "version": "1.1.0", @@ -16826,11 +16787,6 @@ } } }, - "sax": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" - }, "saxes": { "version": "3.1.11", "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz", @@ -16895,6 +16851,11 @@ "resolved": "https://registry.npmjs.org/section-iterator/-/section-iterator-2.0.0.tgz", "integrity": "sha1-v0RNev7rlK1Dw5rS+yYVFifMuio=" }, + "select": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", + "integrity": "sha1-DnNQrN7ICxEIUoeG7B1EGNEbOW0=" + }, "select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", @@ -18116,20 +18077,6 @@ "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==", "dev": true }, - "tar": { - "version": "4.4.13", - "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", - "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", - "requires": { - "chownr": "^1.1.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.8.6", - "minizlib": "^1.2.1", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.2", - "yallist": "^3.0.3" - } - }, "tar-fs": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", @@ -18326,6 +18273,11 @@ "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" }, + "tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "tiny-inflate": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", @@ -19303,6 +19255,11 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" }, + "uuid-js": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/uuid-js/-/uuid-js-0.7.5.tgz", + "integrity": "sha1-bIhtAqU9LUDc8l2RoXC0p7JblNA=" + }, "valid-url": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", diff --git a/package.json b/package.json index cc15060e1..c8455c8a3 100644 --- a/package.json +++ b/package.json @@ -196,13 +196,13 @@ "https": "^1.0.0", "https-browserify": "^1.0.0", "i": "^0.3.7", + "iink-js": "^1.5.4", "image-data-uri": "^2.0.1", "image-size": "^0.7.5", "image-size-stream": "^1.1.0", "js-datepicker": "^4.6.6", "jsonschema": "^1.4.0", "jszip": "^3.7.1", - "libxmljs": "^0.19.7", "lodash": "^4.17.21", "material-ui": "^0.20.2", "memorystream": "^0.3.1", @@ -263,7 +263,7 @@ "react-resizable-rotatable-draggable": "^0.2.0", "react-reveal": "^1.2.2", "react-select": "^3.2.0", - "react-table": "^7.7.0", + "react-table": "^6.11.5", "readline": "^1.3.0", "request": "^2.88.2", "request-promise": "^4.2.6", diff --git a/src/Utils.ts b/src/Utils.ts index 8d71ca6eb..8c0e8c7c0 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -521,6 +521,26 @@ export function smoothScroll(duration: number, element: HTMLElement | HTMLElemen }; animateScroll(); } + +export function smoothScrollHorizontal(duration: number, element: HTMLElement | HTMLElement[], to: number) { + const elements = (element instanceof HTMLElement ? [element] : element); + const starts = elements.map(element => element.scrollLeft); + const startDate = new Date().getTime(); + + const animateScroll = () => { + const currentDate = new Date().getTime(); + const currentTime = currentDate - startDate; + elements.map((element, i) => element.scrollLeft = easeInOutQuad(currentTime, starts[i], to - starts[i], duration)); + + if (currentTime < duration) { + requestAnimationFrame(animateScroll); + } else { + elements.forEach(element => element.scrollLeft = to); + } + }; + animateScroll(); +} + export function addStyleSheet(styleType: string = "text/css") { const style = document.createElement("style"); style.type = styleType; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index e2f82fc62..884cc781d 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -4,7 +4,7 @@ import { DateField } from "../../fields/DateField"; import { Doc, DocListCast, DocListCastAsync, Field, HeightSym, Initializing, Opt, updateCachedAcls, WidthSym } from "../../fields/Doc"; import { Id } from "../../fields/FieldSymbols"; import { HtmlField } from "../../fields/HtmlField"; -import { InkField } from "../../fields/InkField"; +import { InkField, PointData } from "../../fields/InkField"; import { List } from "../../fields/List"; import { ProxyField } from "../../fields/Proxy"; import { RichTextField } from "../../fields/RichTextField"; @@ -179,7 +179,7 @@ export class DocumentOptions { layout?: string | Doc; // default layout string for a document contentPointerEvents?: string; // pointer events allowed for content of a document view. eg. set to "none" in menuSidebar for sharedDocs so that you can select a document, but not interact with its contents childLimitHeight?: number; // whether to limit the height of collection children. 0 - means height can be no bigger than width - childLayoutTemplate?: Doc; // template for collection to use to render its children (see PresBox or Buxton layout in tree view) + childLayoutTemplate?: Doc; // template for collection to use to render its children (see PresBox layout in tree view) childLayoutString?: string; // template string for collection to use to render its children childDontRegisterViews?: boolean; childHideLinkButton?: boolean; // hide link buttons on all children @@ -738,7 +738,7 @@ export namespace Docs { return linkDoc; } - export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: { X: number, Y: number }[], options: DocumentOptions = {}) { + export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: PointData[], options: DocumentOptions = {}) { const I = new Doc(); I[Initializing] = true; I.type = DocumentType.INK; @@ -1161,6 +1161,7 @@ export namespace DocUtils { created = Docs.Create.AudioDocument((field).url.href, resolved); layout = AudioBox.LayoutString; } else if (field instanceof InkField) { + console.log("Documents " + field.inkData) created = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), (field).inkData, resolved); layout = InkingStroke.LayoutString; } else if (field instanceof List && field[0] instanceof Doc) { diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index f08f13bbc..db20d9190 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -1,4 +1,5 @@ import { computed, observable, reaction } from "mobx"; +import * as rp from 'request-promise'; import { DataSym, Doc, DocListCast, DocListCastAsync } from "../../fields/Doc"; import { Id } from "../../fields/FieldSymbols"; import { InkTool } from "../../fields/InkField"; @@ -6,7 +7,6 @@ import { List } from "../../fields/List"; import { PrefetchProxy } from "../../fields/Proxy"; import { RichTextField } from "../../fields/RichTextField"; import { listSpec } from "../../fields/Schema"; -import { SchemaHeaderField } from "../../fields/SchemaHeaderField"; import { ComputedField, ScriptField } from "../../fields/ScriptField"; import { BoolCast, Cast, DateCast, NumCast, PromiseValue, StrCast } from "../../fields/Types"; import { nullAudio } from "../../fields/URLField"; @@ -36,7 +36,6 @@ import { ColorScheme } from "./SettingsManager"; import { SharingManager } from "./SharingManager"; import { SnappingManager } from "./SnappingManager"; import { UndoManager } from "./UndoManager"; -import * as rp from 'request-promise'; interface Button { title?: string; @@ -191,63 +190,9 @@ export class CurrentUserUtils { // }); // } - if (doc["template-button-detail"] === undefined) { - const { TextDocument, MasonryDocument, CarouselDocument } = Docs.Create; - - const openInTarget = ScriptField.MakeScript("openOnRight(self.doubleClickView)"); - const carousel = CarouselDocument([], { - title: "data", _height: 350, _itemIndex: 0, "_carousel-caption-xMargin": 10, "_carousel-caption-yMargin": 10, - onChildDoubleClick: openInTarget, backgroundColor: "#9b9b9b3F", system: true - }); - - const details = TextDocument("", { title: "details", _height: 200, _autoHeight: true, system: true }); - const short = TextDocument("", { title: "shortDescription", treeViewOpen: true, treeViewExpandedView: "layout", _height: 75, _autoHeight: true, system: true }); - const long = TextDocument("", { title: "longDescription", treeViewOpen: false, treeViewExpandedView: "layout", _height: 150, _autoHeight: true, system: true }); - - const buxtonFieldKeys = ["year", "originalPrice", "degreesOfFreedom", "company", "attribute", "primaryKey", "secondaryKey", "dimensions"]; - const detailedTemplate = { - doc: { - type: "doc", content: buxtonFieldKeys.map(fieldKey => ({ - type: "paragraph", - content: [{ type: "dashField", attrs: { fieldKey } }] - })) - }, - selection: { type: "text", anchor: 1, head: 1 }, - storedMarks: [] - }; - details.text = new RichTextField(JSON.stringify(detailedTemplate), buxtonFieldKeys.join(" ")); - - const shared = { _autoHeight: true, _xMargin: 0 }; - const detailViewOpts = { title: "detailView", _width: 300, _fontFamily: "Arial", _fontSize: "12px" }; - const descriptionWrapperOpts = { title: "descriptions", _height: 300, _columnWidth: -1, treeViewHideTitle: true, _pivotField: "title", system: true }; - - const descriptionWrapper = MasonryDocument([details, short, long], { ...shared, ...descriptionWrapperOpts }); - descriptionWrapper._columnHeaders = new List<SchemaHeaderField>([ - new SchemaHeaderField("[A Short Description]", "dimgray", undefined, undefined, undefined, false), - new SchemaHeaderField("[Long Description]", "dimgray", undefined, undefined, undefined, true), - new SchemaHeaderField("[Details]", "dimgray", undefined, undefined, undefined, true), - ]); - const detailView = Docs.Create.StackingDocument([carousel, descriptionWrapper], { ...shared, ...detailViewOpts, _chromeHidden: true, system: true }); - detailView.isTemplateDoc = makeTemplate(detailView); - - details.title = "Details"; - short.title = "A Short Description"; - long.title = "Long Description"; - - doc["template-button-detail"] = CurrentUserUtils.createToolButton({ - onDragStart: ScriptField.MakeFunction('copyDragFactory(this.dragFactory)'), - dragFactory: new PrefetchProxy(detailView) as any as Doc, - title: "detailView", - icon: "window-maximize", - system: true, - btnType: ButtonType.ToolButton, - }); - } - const requiredTypes = [ doc["template-button-slides"] as Doc, doc["template-mobile-button"] as Doc, - doc["template-button-detail"] as Doc, doc["template-button-link"] as Doc, //doc["template-button-switch"] as Doc] ]; diff --git a/src/client/views/AudioWaveform.tsx b/src/client/views/AudioWaveform.tsx index 8f3b7c2cd..525c0ce5a 100644 --- a/src/client/views/AudioWaveform.tsx +++ b/src/client/views/AudioWaveform.tsx @@ -1,126 +1,119 @@ import React = require("react"); import axios from "axios"; -import { action, computed } from "mobx"; +import { action, computed, IReactionDisposer, reaction } from "mobx"; import { observer } from "mobx-react"; import Waveform from "react-audio-waveform"; import { Doc } from "../../fields/Doc"; import { List } from "../../fields/List"; import { listSpec } from "../../fields/Schema"; -import { Cast, NumCast } from "../../fields/Types"; +import { Cast } from "../../fields/Types"; import { numberRange } from "../../Utils"; import "./AudioWaveform.scss"; import { Colors } from "./global/globalEnums"; + +/** + * AudioWaveform + * + * Used in CollectionStackedTimeline to render a canvas with a visual of an audio waveform for AudioBox and VideoBox documents. + * Uses react-audio-waveform package. + * Bins the audio data into audioBuckets which are passed to package to render the lines. + * Calculates new buckets each time a new zoom factor or new set of trim bounds is created and stores it in a field on the layout doc with a title indicating the bounds and zoom for that list (see audioBucketField) + */ + + export interface AudioWaveformProps { - duration: number; + duration: number; // length of media clip + rawDuration: number; // length of underlying media data mediaPath: string; layoutDoc: Doc; - trimming: boolean; - PanelHeight: () => number; + clipStart: number; + clipEnd: number; + zoomFactor: number; + PanelHeight: number; + PanelWidth: number; } @observer export class AudioWaveform extends React.Component<AudioWaveformProps> { - public static NUMBER_OF_BUCKETS = 100; - @computed get _waveHeight() { - return Math.max(50, this.props.PanelHeight()); + public static NUMBER_OF_BUCKETS = 100; // number of buckets data is divided into to draw waveform lines + + _disposer: IReactionDisposer | undefined; + + @computed get waveHeight() { return Math.max(50, this.props.PanelHeight); } + + @computed get clipStart() { return this.props.clipStart; } + @computed get clipEnd() { return this.props.clipEnd; } + @computed get zoomFactor() { return this.props.zoomFactor; } + + @computed get audioBuckets() { return Cast(this.props.layoutDoc[this.audioBucketField(this.clipStart, this.clipEnd, this.zoomFactor)], listSpec("number"), []); } + audioBucketField = (start: number, end: number, zoomFactor: number) => "audioBuckets/" + "/" + start.toFixed(2).replace(".", "_") + "/" + end.toFixed(2).replace(".", "_") + "/" + (zoomFactor * 10); + + + componentWillUnmount() { + this._disposer?.(); } + componentDidMount() { - const audioBuckets = Cast( - this.props.layoutDoc.audioBuckets, - listSpec("number"), - [] - ); - if (!audioBuckets.length) { - this.props.layoutDoc.audioBuckets = new List<number>([0, 0]); /// "lock" to prevent other views from computing the same data - setTimeout(this.createWaveformBuckets); - } + this._disposer = reaction(() => ({ clipStart: this.clipStart, clipEnd: this.clipEnd, fieldKey: this.audioBucketField(this.clipStart, this.clipEnd, this.zoomFactor), zoomFactor: this.props.zoomFactor }), + ({ clipStart, clipEnd, fieldKey, zoomFactor }) => { + if (!this.props.layoutDoc[fieldKey]) { + // setting these values here serves as a "lock" to prevent multiple attempts to create the waveform at nerly the same time. + const waveform = Cast(this.props.layoutDoc[this.audioBucketField(0, this.props.rawDuration, 1)], listSpec("number")); + this.props.layoutDoc[fieldKey] = waveform && new List<number>(waveform.slice(clipStart / this.props.rawDuration * waveform.length, clipEnd / this.props.rawDuration * waveform.length)); + setTimeout(() => this.createWaveformBuckets(fieldKey, clipStart, clipEnd, zoomFactor)); + } + }, { fireImmediately: true }); } // decodes the audio file into peaks for generating the waveform - createWaveformBuckets = async () => { + createWaveformBuckets = async (fieldKey: string, clipStart: number, clipEnd: number, zoomFactor: number) => { axios({ url: this.props.mediaPath, responseType: "arraybuffer" }).then( (response) => { const context = new window.AudioContext(); context.decodeAudioData( response.data, action((buffer) => { - const decodedAudioData = buffer.getChannelData(0); + const rawDecodedAudioData = buffer.getChannelData(0); + const startInd = clipStart / this.props.rawDuration; + const endInd = clipEnd / this.props.rawDuration; + const decodedAudioData = rawDecodedAudioData.slice(Math.floor(startInd * rawDecodedAudioData.length), Math.floor(endInd * rawDecodedAudioData.length)); + const numBuckets = Math.floor(AudioWaveform.NUMBER_OF_BUCKETS * zoomFactor); const bucketDataSize = Math.floor( - decodedAudioData.length / AudioWaveform.NUMBER_OF_BUCKETS + decodedAudioData.length / numBuckets ); const brange = Array.from(Array(bucketDataSize)); - this.props.layoutDoc.audioBuckets = new List<number>( - numberRange(AudioWaveform.NUMBER_OF_BUCKETS).map( - (i: number) => - brange.reduce( - (p, x, j) => - Math.abs( - Math.max(p, decodedAudioData[i * bucketDataSize + j]) - ), - 0 - ) / 2 - ) + const bucketList = numberRange(numBuckets).map( + (i: number) => + brange.reduce( + (p, x, j) => + Math.abs( + Math.max(p, decodedAudioData[i * bucketDataSize + j]) + ), + 0 + ) / 2 ); + this.props.layoutDoc[fieldKey] = new List<number>(bucketList); }) ); } ); } - @action - createTrimBuckets = () => { - const audioBuckets = Cast( - this.props.layoutDoc.audioBuckets, - listSpec("number"), - [] - ); - - const start = Math.floor( - (NumCast(this.props.layoutDoc.clipStart) / this.props.duration) * 100 - ); - const end = Math.floor( - (NumCast(this.props.layoutDoc.clipEnd) / this.props.duration) * 100 - ); - return audioBuckets.slice(start, end); - } - render() { - const audioBuckets = Cast( - this.props.layoutDoc.audioBuckets, - listSpec("number"), - [] - ); - return ( <div className="audioWaveform"> - {this.props.trimming || !this.props.layoutDoc.clipEnd ? ( - <Waveform - color={Colors.MEDIUM_BLUE} - height={this._waveHeight} - barWidth={0.1} - pos={this.props.duration} - duration={this.props.duration} - peaks={ - audioBuckets.length === AudioWaveform.NUMBER_OF_BUCKETS - ? audioBuckets - : undefined - } - progressColor={Colors.MEDIUM_BLUE} - /> - ) : ( - <Waveform - color={Colors.MEDIUM_BLUE} - height={this._waveHeight} - barWidth={0.1} - pos={this.props.duration} - duration={this.props.duration} - peaks={this.createTrimBuckets()} - progressColor={Colors.MEDIUM_BLUE} - /> - )} + <Waveform + color={Colors.MEDIUM_BLUE_ALT} + height={this.waveHeight} + barWidth={200 / this.audioBuckets.length} + pos={this.props.duration} + duration={this.props.duration} + peaks={this.audioBuckets} + progressColor={Colors.MEDIUM_BLUE_ALT} + /> </div> ); } -} +}
\ No newline at end of file diff --git a/src/client/views/ContextMenuItem.tsx b/src/client/views/ContextMenuItem.tsx index c3921d846..25d00f701 100644 --- a/src/client/views/ContextMenuItem.tsx +++ b/src/client/views/ContextMenuItem.tsx @@ -39,7 +39,7 @@ export class ContextMenuItem extends React.Component<ContextMenuProps & { select handleEvent = async (e: React.MouseEvent<HTMLDivElement>) => { if ("event" in this.props) { - this.props.closeMenu && this.props.closeMenu(); + this.props.closeMenu?.(); let batch: UndoManager.Batch | undefined; if (this.props.undoable !== false) { batch = UndoManager.StartBatch(`Context menu event: ${this.props.description}`); @@ -90,7 +90,7 @@ export class ContextMenuItem extends React.Component<ContextMenuProps & { select </span> ) : null} <div className="contextMenu-description"> - {this.props.description.replace(":","")} + {this.props.description.replace(":", "")} </div> </div> ); diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index d0b5bfe46..657d92b8a 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -439,8 +439,8 @@ export class DocumentDecorations extends React.Component<{ PanelWidth: number, P const canOpen = SelectionManager.Views().some(docView => !docView.props.Document._stayInCollection && !docView.props.Document.isGroup && !docView.props.Document.hideOpenButton); const canDelete = SelectionManager.Views().some(docView => { const collectionAcl = docView.props.ContainingCollectionView ? GetEffectiveAcl(docView.props.ContainingCollectionDoc?.[DataSym]) : AclEdit; - return (!docView.rootDoc._stayInCollection || docView.rootDoc.isInkMask) && - (collectionAcl === AclAdmin || collectionAcl === AclEdit || GetEffectiveAcl(docView.rootDoc) === AclAdmin); + //return (!docView.rootDoc._stayInCollection || docView.rootDoc.isInkMask) && + return (collectionAcl === AclAdmin || collectionAcl === AclEdit || GetEffectiveAcl(docView.rootDoc) === AclAdmin); }); const topBtn = (key: string, icon: string, pointerDown: undefined | ((e: React.PointerEvent) => void), click: undefined | ((e: any) => void), title: string) => ( <Tooltip key={key} title={<div className="dash-tooltip">{title}</div>} placement="top"> diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 24df3925b..f034e06ad 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -42,6 +42,7 @@ export class GestureOverlay extends Touchable { @observable private _menuY: number = -300; @observable private _pointerY?: number; @observable private _points: { X: number, Y: number }[] = []; + @observable private _timeStamps: number[] = []; @observable private _strokes: InkData[] = []; @observable private _palette?: JSX.Element; @observable private _clipboardDoc?: JSX.Element; @@ -60,6 +61,7 @@ export class GestureOverlay extends Touchable { private pointerIdentifier?: number; private _hands: Map<number, React.Touch[]> = new Map<number, React.Touch[]>(); private _holdTimer: NodeJS.Timeout | undefined; + private _strokeStartTime: number = 0; protected _multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer; @@ -424,6 +426,7 @@ export class GestureOverlay extends Touchable { this._strokes = []; this._points = []; + this._timeStamps = []; this._possibilities = []; document.removeEventListener("touchend", this.handleHandUp); } @@ -502,6 +505,8 @@ export class GestureOverlay extends Touchable { CurrentUserUtils.SelectedTool = InkTool.Write; } this._points.push({ X: e.clientX, Y: e.clientY }); + this._timeStamps.push(0); + this._strokeStartTime = new Date().getTime(); setupMoveUpEvents(this, e, this.onPointerMove, this.onPointerUp, emptyFunction); // if (CurrentUserUtils.SelectedTool === InkTool.Highlighter) SetActiveInkColor("rgba(245, 230, 95, 0.75)"); } @@ -510,6 +515,7 @@ export class GestureOverlay extends Touchable { @action onPointerMove = (e: PointerEvent) => { this._points.push({ X: e.clientX, Y: e.clientY }); + this._timeStamps.push(new Date().getTime() - this._strokeStartTime); if (this._points.length > 1) { const B = this.svgBounds; @@ -556,6 +562,7 @@ export class GestureOverlay extends Touchable { const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); //push first points to so interactionUtil knows pointer is up this._points.push({ X: this._points[0].X, Y: this._points[0].Y }); + this._timeStamps.push(this._timeStamps[0]); const initialPoint = this._points[0.]; const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height); @@ -567,6 +574,7 @@ export class GestureOverlay extends Touchable { case ToolglassTools.InkToText: this._strokes.push(new Array(...this._points)); this._points = []; + this._timeStamps = []; CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes).then((results) => { const wordResults = results.filter((r: any) => r.category === "line"); const possibilities: string[] = []; @@ -590,6 +598,7 @@ export class GestureOverlay extends Touchable { case ToolglassTools.IgnoreGesture: this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; + this._timeStamps = []; break; } } @@ -598,6 +607,7 @@ export class GestureOverlay extends Touchable { this.makePolygon(this.InkShape, false); this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; + this._timeStamps = []; if (!CollectionFreeFormViewChrome.Instance?._keepPrimitiveMode) { this.InkShape = ""; Doc.UserDoc().activeInkTool = InkTool.None; @@ -647,9 +657,11 @@ export class GestureOverlay extends Touchable { } this._points = []; + this._timeStamps = []; } } else { this._points = []; + this._timeStamps = []; } CollectionFreeFormViewChrome.Instance?.primCreated(); } @@ -698,6 +710,7 @@ export class GestureOverlay extends Touchable { } } this._points = []; + this._timeStamps = []; switch (shape) { //must push an extra point in the end so InteractionUtils knows pointer is up. //must be (points[0].X,points[0]-1) @@ -820,7 +833,8 @@ export class GestureOverlay extends Touchable { points: stroke ?? this._points, gesture: gesture as any, bounds: this.getBounds(stroke ?? this._points), - text: data + text: data, + times: this._timeStamps } } ) diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss new file mode 100644 index 000000000..bbb0a1afa --- /dev/null +++ b/src/client/views/InkTranscription.scss @@ -0,0 +1,5 @@ +.ink-transcription { + .error-msg { + display: none !important; + } +}
\ No newline at end of file diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx new file mode 100644 index 000000000..950f622dd --- /dev/null +++ b/src/client/views/InkTranscription.tsx @@ -0,0 +1,188 @@ +import { timeStamp } from 'console'; +import * as iink from 'iink-js'; +import { action, observable } from 'mobx'; +import * as React from 'react'; +import { Doc } from '../../fields/Doc'; +import { InkData, InkField } from "../../fields/InkField"; +import { Cast, NumCast, StrCast } from '../../fields/Types'; +import { Docs } from "../documents/Documents"; +import './InkTranscription.scss'; + +export class InkTranscription extends React.Component { + static Instance: InkTranscription; + + @observable _mathRegister: any; + @observable _mathRef: any; + @observable _textRegister: any; + @observable _textRef: any; + private addDocument?: (doc: Doc | Doc[]) => boolean; + private bounds: { x: number, y: number, width: number, height: number } = { x: 0, y: 0, width: 0, height: 0 }; + + constructor(props: Readonly<{}>) { + super(props); + + InkTranscription.Instance = this; + } + + componentWillUnmount() { + this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + this._mathRef.removeEventListener('changed', (e: any) => this.callExport(this._mathRef)); + this._textRef.removeEventListener('changed', (e: any) => this.callExport(this._textRef)); + } + + @action + setMathRef = (r: any) => { + if (!this._mathRegister) { + this._mathRegister = r ? iink.register(r, { + recognitionParams: { + type: 'MATH', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + math: { + mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._mathRef)); + + return this._mathRef = r; + } + + @action + setTextRef = (r: any) => { + if (!this._textRegister) { + this._textRegister = r ? iink.register(r, { + recognitionParams: { + type: 'TEXT', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + text: { + mimeTypes: ['text/plain'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._textRef)); + + return this._textRef = r; + } + + transcribeInk = (inkdocs: Doc[], math: boolean, bounds: { x: number, y: number, width: number, height: number }, addDocument?: (doc: Doc | Doc[]) => boolean) => { + const strokes: InkData[] = []; + inkdocs.filter(i => Cast(i.data, InkField)).forEach(i => { + // TODO: interpolate missing times stamps + const d = Cast(i.data, InkField, null); + const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); + const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); + strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time }))); + }); + + this.addDocument = addDocument; + this.bounds = bounds; + + const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) }; + // console.log(JSON.stringify(pointerData)); + const processGestures = false; + + if (math) { + this._mathRef.editor.pointerEvents(pointerData, processGestures); + } + else { + this._textRef.editor.pointerEvents(pointerData, processGestures); + } + } + + callExport(ref: any) { + if (ref.editor.canExport) { + ref.editor.export_(); + } + } + + inkJSON = (stroke: InkData) => { + return { + "pointerType": "PEN", + "pointerId": 1, + "x": stroke.map(point => point.X), + "y": stroke.map(point => point.Y), + "t": stroke.map(point => point.time), + "p": new Array(stroke.length).fill(1.0) + }; + } + + exportInk = (e: any, ref: any) => { + const exports = e.detail.exports; + if (exports) { + if (exports['application/x-latex']) { + const latex = exports['application/x-latex']; + console.log(latex); + + this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + else if (exports['text/plain']) { + const text = exports['text/plain']; + console.log(text); + this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + + ref.editor.clear(); + } + } + + render() { + return ( + <div className="ink-transcription"> + <div className='math-editor' + ref={this.setMathRef} + touch-action="none"> + </div> + <div className='text-editor' + ref={this.setTextRef} + touch-action="none"> + </div> + </div> + ) + } +}
\ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 8c0795881..84c1037dd 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -44,6 +44,7 @@ import { GestureOverlay } from './GestureOverlay'; import { DASHBOARD_SELECTOR_HEIGHT, LEFT_MENU_WIDTH } from './global/globalCssVariables.scss'; import { Colors } from './global/globalEnums'; import { KeyManager } from './GlobalKeyHandler'; +import { InkTranscription } from './InkTranscription'; import { LightboxView } from './LightboxView'; import { LinkMenu } from './linking/LinkMenu'; import "./MainView.scss"; @@ -184,7 +185,7 @@ export class MainView extends React.Component { fa.faArrowAltCircleDown, fa.faArrowAltCircleUp, fa.faArrowAltCircleLeft, fa.faArrowAltCircleRight, fa.faStopCircle, fa.faCheckCircle, fa.faGripVertical, fa.faSortUp, fa.faSortDown, fa.faTable, fa.faTh, fa.faThList, fa.faProjectDiagram, fa.faSignature, fa.faColumns, fa.faChevronCircleUp, fa.faUpload, fa.faBorderAll, fa.faBraille, fa.faChalkboard, fa.faPencilAlt, fa.faEyeSlash, fa.faSmile, fa.faIndent, fa.faOutdent, fa.faChartBar, fa.faBan, fa.faPhoneSlash, fa.faGripLines, - fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt]); + fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt, fa.faSquareRootAlt]); this.initAuthenticationRouters(); } @@ -636,6 +637,7 @@ export class MainView extends React.Component { <OverlayView /> <TimelineMenu /> <RichTextMenu /> + <InkTranscription /> {this.snapLines} <div className="mainView-webRef" ref={this.makeWebRef} /> <LightboxView PanelWidth={this._windowWidth} PanelHeight={this._windowHeight} maxBorder={[200, 50]} /> diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx index a2f23ee09..2782574c5 100644 --- a/src/client/views/StyleProvider.tsx +++ b/src/client/views/StyleProvider.tsx @@ -108,7 +108,7 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<DocumentViewProps if (!backColor) return undefined; return lightOrDark(backColor); case StyleProp.Hidden: return BoolCast(doc?._hidden); - case StyleProp.BorderRounding: return StrCast(doc?.[fieldKey + "borderRounding"], doc?._viewType === CollectionViewType.Pile ? "50%" : ""); + case StyleProp.BorderRounding: return StrCast(doc?.[fieldKey + "borderRounding"], StrCast(doc?.borderRounding, doc?._viewType === CollectionViewType.Pile ? "50%" : "")); case StyleProp.TitleHeight: return 15; case StyleProp.BorderPath: return comicStyle() && props?.renderDepth && doc?.type !== DocumentType.INK ? { path: wavyBorderPath(props?.PanelWidth?.() || 0, props?.PanelHeight?.() || 0), fill: wavyBorderPath(props?.PanelWidth?.() || 0, props?.PanelHeight?.() || 0, .08), width: 3 } : { path: undefined, width: 0 }; case StyleProp.JitterRotation: return comicStyle() ? random(-1, 1, NumCast(doc?.x), NumCast(doc?.y)) * ((props?.PanelWidth() || 0) > (props?.PanelHeight() || 0) ? 5 : 10) : 0; @@ -184,7 +184,7 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<DocumentViewProps if (doc?.type !== DocumentType.INK && layer === true) return "all"; return undefined; case StyleProp.Decorations: - if (props?.ContainingCollectionDoc?._viewType === CollectionViewType.Freeform) { + if (props?.ContainingCollectionDoc?._viewType === CollectionViewType.Freeform || doc?.x !== undefined || doc?.y !== undefined) { return doc && (isBackground() || selected) && (props?.renderDepth || 0) > 0 && ((doc.type === DocumentType.COL && doc._viewType !== CollectionViewType.Pile) || [DocumentType.RTF, DocumentType.IMG, DocumentType.INK].includes(doc.type as DocumentType)) ? <div className="styleProvider-lock" onClick={() => toggleBackground(doc)}> diff --git a/src/client/views/collections/CollectionStackedTimeline.scss b/src/client/views/collections/CollectionStackedTimeline.scss index 59c21210a..e8b6817b4 100644 --- a/src/client/views/collections/CollectionStackedTimeline.scss +++ b/src/client/views/collections/CollectionStackedTimeline.scss @@ -1,94 +1,110 @@ @import "../global/globalCssVariables.scss"; -.collectionStackedTimeline { - position: absolute; - width: 100%; - height: 100%; - z-index: 1000; - overflow: hidden; - top: 0px; - - .collectionStackedTimeline-trim-shade { - position: absolute; +.timeline-container { height: 100%; - background-color: $dark-gray; - opacity: 0.3; - } + overflow-x: auto; + overflow-y: hidden; + border: none; + background-color: $white; + border: 2px solid $dark-gray; + border-width: 0 2px 0 2px; +} - .collectionStackedTimeline-trim-controls { - height: 100%; +::-webkit-scrollbar { + height: 5px; +} + +.collectionStackedTimeline { position: absolute; - box-sizing: border-box; - border: 2px solid $medium-blue; - display: flex; - justify-content: space-between; - max-width: 100%; + background: $off-white; + z-index: 1000; + height: 100%; - .collectionStackedTimeline-trim-handle { - background-color: $medium-blue; - height: 100%; - width: 5px; - cursor: ew-resize; + .collectionStackedTimeline-trim-shade { + position: absolute; + height: 100%; + background-color: $dark-gray; + opacity: 0.3; + top: 0; } - } - - .collectionStackedTimeline-selector { - position: absolute; - width: 10px; - top: 2.5%; - height: 95%; - background: $light-blue; - border-radius: 3px; - opacity: 0.3; - z-index: 500; - border-style: solid; - border-color: $medium-blue; - border-width: 1px; - } - .collectionStackedTimeline-current { - width: 1px; - height: 100%; - background-color: $pink; - position: absolute; - top: 0px; - pointer-events: none; - } + .collectionStackedTimeline-trim-controls { + height: 100%; + position: absolute; + box-sizing: border-box; + border: 2px solid $medium-blue; + display: flex; + justify-content: space-between; + max-width: 100%; + top: 0; + left: 0; - .collectionStackedTimeline-marker-timeline { - position: absolute; - top: 2.5%; - height: 95%; - border-radius: 4px; - &:hover { - opacity: 1; + .collectionStackedTimeline-trim-handle { + background-color: $medium-blue; + height: 100%; + width: 5px; + cursor: ew-resize; + } } - .collectionStackedTimeline-left-resizer, - .collectionStackedTimeline-resizer { - background: $medium-gray; - position: absolute; - top: 0; - height: 100%; - width: 10px; - pointer-events: all; - cursor: ew-resize; - z-index: 100; + .collectionStackedTimeline-selector { + position: absolute; + width: 10px; + top: 2.5%; + height: 95%; + background: $light-blue; + border-radius: 3px; + opacity: 0.3; + z-index: 500; + border-style: solid; + border-color: $medium-blue; + border-width: 1px; } - .collectionStackedTimeline-resizer { - right: 0; + + .collectionStackedTimeline-current { + width: 1px; + height: 100%; + background-color: $pink; + position: absolute; + top: 0px; + pointer-events: none; } - .collectionStackedTimeline-left-resizer { - left: 0; + + .collectionStackedTimeline-marker-timeline { + position: absolute; + top: 2.5%; + height: 95%; + border-radius: 4px; + background: $light-gray; + &:hover { + opacity: 1; + } + + .collectionStackedTimeline-left-resizer, + .collectionStackedTimeline-resizer { + background: $medium-gray; + position: absolute; + top: 0; + height: 100%; + width: 10px; + pointer-events: all; + cursor: ew-resize; + z-index: 100; + } + .collectionStackedTimeline-resizer { + right: 0; + } + .collectionStackedTimeline-left-resizer { + left: 0; + } } - } - .collectionStackedTimeline-waveform { - position: absolute; - width: 100%; - height: 100%; - top: 0; - left: 0; - pointer-events: none; - } + .collectionStackedTimeline-waveform { + position: absolute; + width: 100%; + height: 100%; + top: 0; + left: 0; + pointer-events: none; + } } diff --git a/src/client/views/collections/CollectionStackedTimeline.tsx b/src/client/views/collections/CollectionStackedTimeline.tsx index e09e9aa35..e0b947211 100644 --- a/src/client/views/collections/CollectionStackedTimeline.tsx +++ b/src/client/views/collections/CollectionStackedTimeline.tsx @@ -4,8 +4,7 @@ import { computed, IReactionDisposer, observable, - reaction, - runInAction + reaction } from "mobx"; import { observer } from "mobx-react"; import { computedFn } from "mobx-utils"; @@ -20,9 +19,7 @@ import { formatTime, OmitKeys, returnFalse, - returnOne, - setupMoveUpEvents, - StopEvent + returnOne, returnTrue, setupMoveUpEvents, smoothScrollHorizontal, StopEvent } from "../../../Utils"; import { Docs } from "../../documents/Documents"; import { DocumentManager } from "../../util/DocumentManager"; @@ -32,7 +29,7 @@ import { ScriptingGlobals } from "../../util/ScriptingGlobals"; import { SelectionManager } from "../../util/SelectionManager"; import { SnappingManager } from "../../util/SnappingManager"; import { Transform } from "../../util/Transform"; -import { undoBatch } from "../../util/UndoManager"; +import { undoBatch, UndoManager } from "../../util/UndoManager"; import { AudioWaveform } from "../AudioWaveform"; import { CollectionSubView } from "../collections/CollectionSubView"; import { Colors } from "../global/globalEnums"; @@ -47,7 +44,6 @@ import { LabelBox } from "../nodes/LabelBox"; import "./CollectionStackedTimeline.scss"; export type CollectionStackedTimelineProps = { - duration: number; Play: () => void; Pause: () => void; playLink: (linkDoc: Doc) => void; @@ -58,61 +54,55 @@ export type CollectionStackedTimelineProps = { endTag: string; mediaPath: string; dictationKey: string; - trimming: boolean; - trimStart: number; - trimEnd: number; - trimDuration: number; - setStartTrim: (newStart: number) => void; - setEndTrim: (newEnd: number) => void; + rawDuration: number; + fieldKey: string; }; +// trimming state: shows full clip, current trim bounds, or not trimming +export enum TrimScope { + All = 2, + Clip = 1, + None = 0, +} + + @observer export class CollectionStackedTimeline extends CollectionSubView<CollectionStackedTimelineProps>() { - @observable static SelectingRegion: CollectionStackedTimeline | undefined = - undefined; + @observable static SelectingRegion: CollectionStackedTimeline | undefined = undefined; + @observable public static CurrentlyPlaying: Doc[]; + static RangeScript: ScriptField; static LabelScript: ScriptField; static RangePlayScript: ScriptField; static LabelPlayScript: ScriptField; - private _timeline: HTMLDivElement | null = null; + private _timeline: HTMLDivElement | null = null; // ref to actual timeline div + private _timelineWrapper: HTMLDivElement | null = null; // ref to timeline wrapper div for zooming and scrolling private _markerStart: number = 0; - @observable _markerEnd: number = 0; + @observable _markerEnd: number | undefined; + @observable _trimming: number = TrimScope.None; + @observable _trimStart: number = 0; // trim controls start pos + @observable _trimEnd: number = 0; // trim controls end pos - get minLength() { - const rect = this._timeline?.getBoundingClientRect(); - if (rect) { - return 0.05 * this.duration; - } - return 0; - } + @observable _zoomFactor: number = 1; - get trimStart() { - return this.props.trimStart; - } + @observable _scroll: number = 0; - get trimEnd() { - return this.props.trimEnd; - } + // ensures that clip doesn't get trimmed so small that controls cannot be adjusted anymore + get minTrimLength() { return Math.max(this._timeline?.getBoundingClientRect() ? 0.05 * this.clipDuration : 0, 0.5) } - get duration() { - return this.props.duration; - } + @computed get trimStart() { return this.IsTrimming !== TrimScope.None ? this._trimStart : this.clipStart; } + @computed get trimDuration() { return this.trimEnd - this.trimStart; } + @computed get trimEnd() { return this.IsTrimming !== TrimScope.None ? this._trimEnd : this.clipEnd; } + + @computed get clipStart() { return this.IsTrimming === TrimScope.All ? 0 : NumCast(this.layoutDoc.clipStart); } + @computed get clipDuration() { return this.clipEnd - this.clipStart; } + @computed get clipEnd() { return this.IsTrimming === TrimScope.All ? this.props.rawDuration : NumCast(this.layoutDoc.clipEnd, this.props.rawDuration); } + + @computed get currentTime() { return NumCast(this.layoutDoc._currentTimecode); } + + @computed get zoomFactor() { return this._zoomFactor } - @computed get currentTime() { - return NumCast(this.layoutDoc._currentTimecode); - } - @computed get selectionContainer() { - return CollectionStackedTimeline.SelectingRegion !== this ? null : ( - <div - className="collectionStackedTimeline-selector" - style={{ - left: `${((Math.min(this._markerStart, this._markerEnd) - this.trimStart) / this.props.trimDuration) * 100}%`, - width: `${(Math.abs(this._markerStart - this._markerEnd) / this.props.trimDuration) * 100}%`, - }} - /> - ); - } constructor(props: any) { super(props); @@ -136,60 +126,100 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack componentDidMount() { document.addEventListener("keydown", this.keyEvents, true); } + + @action componentWillUnmount() { document.removeEventListener("keydown", this.keyEvents, true); if (CollectionStackedTimeline.SelectingRegion === this) { - runInAction( - () => (CollectionStackedTimeline.SelectingRegion = undefined) - ); + CollectionStackedTimeline.SelectingRegion = undefined; } } - anchorStart = (anchor: Doc) => - NumCast(anchor._timecodeToShow, NumCast(anchor[this.props.startTag])) - anchorEnd = (anchor: Doc, val: any = null) => { - const endVal = NumCast(anchor[this.props.endTag], val); - return NumCast( - anchor._timecodeToHide, - endVal === undefined ? null : endVal - ); + + public get IsTrimming() { return this._trimming; } + + @action + public StartTrimming(scope: TrimScope) { + this._trimStart = this.clipStart; + this._trimEnd = this.clipEnd; + this._trimming = scope; + } + @action + public StopTrimming() { + this.layoutDoc.clipStart = this.trimStart; + this.layoutDoc.clipEnd = this.trimEnd; + this._trimming = TrimScope.None; + } + + @action + public setZoom(zoom: number) { + this._zoomFactor = zoom; } + + + anchorStart = (anchor: Doc) => NumCast(anchor._timecodeToShow, NumCast(anchor[this.props.startTag])); + anchorEnd = (anchor: Doc, val: any = null) => NumCast(anchor._timecodeToHide, NumCast(anchor[this.props.endTag], val) ?? null); + + + // converts screen pixel offset to time toTimeline = (screen_delta: number, width: number) => { return Math.max( - this.trimStart, - Math.min(this.trimEnd, (screen_delta / width) * this.props.trimDuration + this.trimStart)); + this.clipStart, + Math.min(this.clipEnd, (screen_delta / width) * this.clipDuration + this.clipStart)); } + rangeClickScript = () => CollectionStackedTimeline.RangeScript; rangePlayScript = () => CollectionStackedTimeline.RangePlayScript; - // for creating key anchors with key events + + // handles key events for for creating key anchors, scrubbing, exiting trim @action keyEvents = (e: KeyboardEvent) => { if ( !(e.target instanceof HTMLInputElement) && this.props.isSelected(true) ) { + // if shift pressed scrub 1 second otherwise 1/10th + const jump = e.shiftKey ? 1 : 0.1; + e.stopPropagation(); switch (e.key) { case " ": if (!CollectionStackedTimeline.SelectingRegion) { this._markerStart = this._markerEnd = this.currentTime; CollectionStackedTimeline.SelectingRegion = this; } else { + this._markerEnd = this.currentTime; CollectionStackedTimeline.createAnchor( this.rootDoc, this.dataDoc, this.props.fieldKey, this.props.startTag, this.props.endTag, - this.currentTime + this._markerStart, + this._markerEnd ); + this._markerEnd = undefined; CollectionStackedTimeline.SelectingRegion = undefined; } + break; + case "Escape": + // abandons current trim + this._trimStart = this.clipStart; + this._trimStart = this.clipEnd; + this._trimming = TrimScope.None; + break; + case "ArrowLeft": + this.props.setTime(Math.min(Math.max(this.clipStart, this.currentTime - jump), this.clipEnd)); + break; + case "ArrowRight": + this.props.setTime(Math.min(Math.max(this.clipStart, this.currentTime + jump), this.clipEnd)); + break; } } } + getLinkData(l: Doc) { let la1 = l.anchor1 as Doc; let la2 = l.anchor2 as Doc; @@ -204,28 +234,25 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack return { la1, la2, linkTime }; } - // starting the drag event for anchor resizing + + // handles dragging selection to create markers @action onPointerDownTimeline = (e: React.PointerEvent): void => { const rect = this._timeline?.getBoundingClientRect(); const clientX = e.clientX; + const diff = rect ? clientX - rect?.x : null; + const shiftKey = e.shiftKey; if (rect && this.props.isContentActive()) { const wasPlaying = this.props.playing(); if (wasPlaying) this.props.Pause(); - const wasSelecting = CollectionStackedTimeline.SelectingRegion === this; + var wasSelecting = this._markerEnd !== undefined; setupMoveUpEvents( this, e, action((e) => { - if ( - !wasSelecting && - CollectionStackedTimeline.SelectingRegion !== this - ) { - this._markerStart = this._markerEnd = this.toTimeline( - clientX - rect.x, - rect.width - ); - CollectionStackedTimeline.SelectingRegion = this; + if (!wasSelecting) { + this._markerStart = this._markerEnd = this.toTimeline(clientX - rect.x, rect.width); + wasSelecting = true; } this._markerEnd = this.toTimeline(e.clientX - rect.x, rect.width); return false; @@ -239,9 +266,8 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack } if ( !isClick && - CollectionStackedTimeline.SelectingRegion === this && Math.abs(movement[0]) > 15 && - !this.props.trimming + !this.IsTrimming ) { const anchor = CollectionStackedTimeline.createAnchor( this.rootDoc, @@ -255,11 +281,18 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack setTimeout(() => DocumentManager.Instance.getDocumentView(anchor)?.select(false)); } (!isClick || !wasSelecting) && - (CollectionStackedTimeline.SelectingRegion = undefined); + (this._markerEnd = undefined); }), (e, doubleTap) => { - this.props.select(false); - e.shiftKey && + if (e.button !== 2) { + this.props.select(false); + !wasPlaying && doubleTap && this.props.Play(); + } + }, + this.props.isSelected(true) || this.props.isContentActive(), + undefined, + () => { + if (shiftKey) { CollectionStackedTimeline.createAnchor( this.rootDoc, this.dataDoc, @@ -268,23 +301,17 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack this.props.endTag, this.currentTime ); - !wasPlaying && doubleTap && this.props.Play(); - }, - this.props.isSelected(true) || this.props.isContentActive(), - undefined, - () => { - !wasPlaying && - (this.props.trimming && this.duration ? - this.props.setTime(((clientX - rect.x) / rect.width) * this.duration) - : - this.props.setTime(((clientX - rect.x) / rect.width) * this.props.trimDuration + this.trimStart) - ); + } else { + !wasPlaying && this.props.setTime(this.toTimeline(clientX - rect.x, rect.width)); + } } ); } } + + // for dragging trim start handle @action trimLeft = (e: React.PointerEvent): void => { const rect = this._timeline?.getBoundingClientRect(); @@ -294,25 +321,24 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack e, action((e, [], []) => { if (rect && this.props.isContentActive()) { - this.props.setStartTrim(Math.min( + this._trimStart = Math.min( Math.max( - this.trimStart + (e.movementX / rect.width) * this.duration, - 0 + this.trimStart + (e.movementX / rect.width) * this.clipDuration, + this.clipStart ), - this.trimEnd - this.minLength - )); + this.trimEnd - this.minTrimLength + ); } return false; }), emptyFunction, action((e, doubleTap) => { - if (doubleTap) { - this.props.setStartTrim(0); - } + doubleTap && (this._trimStart = this.clipStart); }) ); } + // for dragging trim end handle @action trimRight = (e: React.PointerEvent): void => { const rect = this._timeline?.getBoundingClientRect(); @@ -322,32 +348,64 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack e, action((e, [], []) => { if (rect && this.props.isContentActive()) { - this.props.setEndTrim(Math.max( + this._trimEnd = Math.max( Math.min( - this.trimEnd + (e.movementX / rect.width) * this.duration, - this.duration + this.trimEnd + (e.movementX / rect.width) * this.clipDuration, + this.clipEnd ), - this.trimStart + this.minLength - )); + this.trimStart + this.minTrimLength + ); } return false; }), emptyFunction, action((e, doubleTap) => { - if (doubleTap) { - this.props.setEndTrim(this.duration); - } + doubleTap && (this._trimEnd = this.clipEnd); }) ); } + + // for rendering scrolling when timeline zoomed + @action + setScroll = (e: React.UIEvent) => { + e.stopPropagation(); + this._scroll = this._timelineWrapper!.scrollLeft; + } + + // smooth scrolls to time like when following links overflowed due to zoom + @action + scrollToTime = (time: number) => { + if (this._timelineWrapper) { + if (time > this.toTimeline(this._scroll + this.props.PanelWidth(), this.timelineContentWidth)) { + this._scroll = Math.min(this._scroll + this.props.PanelWidth(), this.timelineContentWidth - this.props.PanelWidth()); + smoothScrollHorizontal(200, this._timelineWrapper, this._scroll); + } + else if (time < this.toTimeline(this._scroll, this.timelineContentWidth)) { + this._scroll = time / this.timelineContentWidth * this.clipDuration; + smoothScrollHorizontal(200, this._timelineWrapper, this._scroll); + } + } + } + + + // handles dragging and dropping markers in timeline @action internalDocDrop(e: Event, de: DragManager.DropEvent, docDragData: DragManager.DocumentDragData, xp: number) { if (!de.embedKey && this.props.layerProvider?.(this.props.Document) !== false && this.props.Document._isGroup) return false; if (!super.onInternalDrop(e, de)) return false; - // determine x coordinate of drop and assign it to the documents being dragged --- see internalDocDrop of collectionFreeFormView.tsx for how it's done when dropping onto a 2D freeform view + const localPt = this.props.ScreenToLocalTransform().transformPoint(de.x, de.y); + const x = localPt[0] - docDragData.offset[0]; + const timelinePt = this.toTimeline(x + this._scroll, this.timelineContentWidth); + docDragData.droppedDocuments.forEach(drop => { + const anchorEnd = this.anchorEnd(drop); + if (anchorEnd !== undefined) { + Doc.SetInPlace(drop, drop._timecodeToHide === undefined ? this.props.endTag : "timecodeToHide", timelinePt + anchorEnd - this.anchorStart(drop), false); + } + Doc.SetInPlace(drop, drop._timecodeToShow === undefined ? this.props.startTag : "timecodeToShow", timelinePt, false); + }); return true; } @@ -357,6 +415,8 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack return false; } + + // creates marker on timeline @undoBatch @action static createAnchor( @@ -379,10 +439,11 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack hideLinkButton: true, annotationOn: rootDoc, _timelineLabel: true, + borderRounding: anchorEndTime === undefined ? "100%" : undefined }); Doc.GetProto(anchor)[startTag] = anchorStartTime; Doc.GetProto(anchor)[endTag] = anchorEndTime; - if (Cast(dataDoc[fieldKey], listSpec(Doc), null) !== undefined) { + if (Cast(dataDoc[fieldKey], listSpec(Doc), null)) { Cast(dataDoc[fieldKey], listSpec(Doc), []).push(anchor); } else { dataDoc[fieldKey] = new List<Doc>([anchor]); @@ -390,13 +451,17 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack return anchor; } + @action playOnClick = (anchorDoc: Doc, clientX: number) => { const seekTimeInSeconds = this.anchorStart(anchorDoc) - 0.25; const endTime = this.anchorEnd(anchorDoc); if (this.layoutDoc.autoPlayAnchors) { if (this.props.playing()) this.props.Pause(); - else this.props.playFrom(seekTimeInSeconds, endTime); + else { + this.props.playFrom(seekTimeInSeconds, endTime); + this.scrollToTime(seekTimeInSeconds); + } } else { if ( seekTimeInSeconds < NumCast(this.layoutDoc._currentTimecode) && @@ -409,6 +474,7 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack } } else { this.props.playFrom(seekTimeInSeconds, endTime); + this.scrollToTime(seekTimeInSeconds); } } return { select: true }; @@ -448,11 +514,11 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack m: Doc, placed: { anchorStartTime: number; anchorEndTime: number; level: number }[] ) => { - const timelineContentWidth = this.props.PanelWidth(); + const timelineContentWidth = this.timelineContentWidth; const x1 = this.anchorStart(m); const x2 = this.anchorEnd( m, - x1 + (10 / timelineContentWidth) * this.duration + x1 + (10 / timelineContentWidth) * this.clipDuration ); let max = 0; const overlappedLevels = new Set( @@ -477,15 +543,20 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack return level; } + dictationHeightPercent = 50; - dictationHeight = () => - (this.props.PanelHeight() * (100 - this.dictationHeightPercent)) / 100 - timelineContentHeight = () => - (this.props.PanelHeight() * this.dictationHeightPercent) / 100 - dictationScreenToLocalTransform = () => - this.props - .ScreenToLocalTransform() - .translate(0, -this.timelineContentHeight()) + dictationHeight = () => (this.props.PanelHeight() * (100 - this.dictationHeightPercent)) / 100; + + @computed get timelineContentHeight() { return this.props.PanelHeight() * this.dictationHeightPercent / 100; } + @computed get timelineContentWidth() { return this.props.PanelWidth() * this.zoomFactor - 4 }; // subtract size of container border + + dictationScreenToLocalTransform = () => this.props.ScreenToLocalTransform().translate(0, -this.timelineContentHeight); + + isContentActive = () => this.props.isSelected() || this.props.isContentActive(); + + currentTimecode = () => this.currentTime; + + @computed get renderDictation() { const dictation = Cast(this.dataDoc[this.props.dictationKey], Doc, null); return !dictation ? null : ( @@ -493,7 +564,7 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack style={{ position: "absolute", height: "100%", - top: this.timelineContentHeight(), + top: this.timelineContentHeight, background: Colors.LIGHT_BLUE, }} > @@ -522,23 +593,22 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack </div> ); } - @computed get renderAudioWaveform() { - return !this.props.mediaPath ? null : ( - <div className="collectionStackedTimeline-waveform"> - <AudioWaveform - duration={this.duration} - mediaPath={this.props.mediaPath} - layoutDoc={this.layoutDoc} - PanelHeight={this.timelineContentHeight} - trimming={this.props.trimming} - /> - </div> + + // renders selection region on timeline + @computed get selectionContainer() { + const markerEnd = CollectionStackedTimeline.SelectingRegion === this ? this.currentTime : this._markerEnd; + return markerEnd === undefined ? null : ( + <div + className="collectionStackedTimeline-selector" + style={{ + left: `${((Math.min(this._markerStart, markerEnd) - this.trimStart) / this.trimDuration) * 100}%`, + width: `${(Math.abs(this._markerStart - markerEnd) / this.trimDuration) * 100}%`, + }} + /> ); } - currentTimecode = () => this.currentTime; render() { - const timelineContentWidth = this.props.PanelWidth(); const overlaps: { anchorStartTime: number; anchorEndTime: number; @@ -549,117 +619,133 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack anchor, })); const maxLevel = overlaps.reduce((m, o) => Math.max(m, o.level), 0) + 2; - const isActive = - this.props.isContentActive() || this.props.isSelected(false); return (<div ref={this.createDashEventsTarget} style={{ pointerEvents: SnappingManager.GetIsDragging() ? "all" : undefined }}> - <div - className="collectionStackedTimeline" - ref={(timeline: HTMLDivElement | null) => (this._timeline = timeline)} - onClick={(e) => isActive && StopEvent(e)} - onPointerDown={(e) => isActive && this.onPointerDownTimeline(e)} - > - {drawAnchors.map((d) => { - - const start = this.anchorStart(d.anchor); - const end = this.anchorEnd( - d.anchor, - start + (10 / timelineContentWidth) * this.duration - ); - const left = this.props.trimming ? - (start / this.duration) * timelineContentWidth - : (start - this.trimStart) / this.props.trimDuration * timelineContentWidth; - const top = (d.level / maxLevel) * this.timelineContentHeight(); - const timespan = end - start; - const width = (timespan / this.props.trimDuration) * timelineContentWidth; - const height = this.timelineContentHeight() / maxLevel; - return this.props.Document.hideAnchors ? null : ( - <div - className={"collectionStackedTimeline-marker-timeline"} - key={d.anchor[Id]} - style={{ - left, - top, - width: `${width}px`, - height: `${height}px`, - }} - onClick={(e) => { - this.props.playFrom(start, this.anchorEnd(d.anchor)); - e.stopPropagation(); - }} - > - <StackedTimelineAnchor - {...this.props} - mark={d.anchor} - rangeClickScript={this.rangeClickScript} - rangePlayScript={this.rangePlayScript} - left={left} - top={top} - width={width} - height={height} - toTimeline={this.toTimeline} - layoutDoc={this.layoutDoc} - currentTimecode={this.currentTimecode} - _timeline={this._timeline} - stackedTimeline={this} - trimStart={this.trimStart} - trimEnd={this.trimEnd} - /> - </div> - ); - })} - {!this.props.trimming && this.selectionContainer} - {this.renderAudioWaveform} - {this.renderDictation} - + <div className="timeline-container" + style={{ width: this.props.PanelWidth() }} + onWheel={e => e.stopPropagation()} + onScroll={this.setScroll} + ref={wrapper => this._timelineWrapper = wrapper}> <div - className="collectionStackedTimeline-current" - style={{ - left: this.props.trimming - ? `${(this.currentTime / this.duration) * 100}%` - : `${(this.currentTime - this.trimStart) / (this.trimEnd - this.trimStart) * 100}%`, - }} - /> - - {this.props.trimming && ( - <> - <div - className="collectionStackedTimeline-trim-shade" - style={{ width: `${(this.trimStart / this.duration) * 100}%` }} - ></div> - - <div - className="collectionStackedTimeline-trim-controls" - style={{ - left: `${(this.trimStart / this.duration) * 100}%`, - width: `${((this.trimEnd - this.trimStart) / this.duration) * 100 - }%`, - }} - > + className="collectionStackedTimeline" + ref={(timeline: HTMLDivElement | null) => (this._timeline = timeline)} + onClick={(e) => this.isContentActive() && StopEvent(e)} + onPointerDown={(e) => this.isContentActive() && this.onPointerDownTimeline(e)} + style={{ width: this.timelineContentWidth }}> + + {drawAnchors.map((d) => { + const start = this.anchorStart(d.anchor); + const end = this.anchorEnd( + d.anchor, + start + (10 / this.timelineContentWidth) * this.clipDuration + ); + if (end < this.clipStart || start > this.clipEnd) return (null); + const left = Math.max((start - this.clipStart) / this.clipDuration * this.timelineContentWidth, 0); + const top = (d.level / maxLevel) * this.props.PanelHeight(); + const timespan = Math.max(0, Math.min(end - this.clipStart, this.clipEnd)) - Math.max(0, start - this.clipStart); + const width = (timespan / this.clipDuration) * this.timelineContentWidth; + const height = this.props.PanelHeight() / maxLevel; + return this.props.Document.hideAnchors ? null : ( <div - className="collectionStackedTimeline-trim-handle" - onPointerDown={this.trimLeft} - ></div> + className={"collectionStackedTimeline-marker-timeline"} + key={d.anchor[Id]} + style={{ + left, + top, + width: `${width}px`, + height: `${height}px`, + }} + onClick={(e) => { + this.props.playFrom(start, this.anchorEnd(d.anchor)); + e.stopPropagation(); + }} + > + <StackedTimelineAnchor + {...this.props} + mark={d.anchor} + rangeClickScript={this.rangeClickScript} + rangePlayScript={this.rangePlayScript} + left={left - this._scroll} + top={top} + width={width} + height={height} + toTimeline={this.toTimeline} + layoutDoc={this.layoutDoc} + // isDocumentActive={this.props.childDocumentsActive ? this.props.isDocumentActive : this.isContentActive} + currentTimecode={this.currentTimecode} + _timeline={this._timeline} + stackedTimeline={this} + trimStart={this.trimStart} + trimEnd={this.trimEnd} + /> + </div> + ); + })} + {!this.IsTrimming && this.selectionContainer} + <AudioWaveform + rawDuration={this.props.rawDuration} + duration={this.clipDuration} + mediaPath={this.props.mediaPath} + layoutDoc={this.layoutDoc} + clipStart={this.clipStart} + clipEnd={this.clipEnd} + zoomFactor={this.zoomFactor} + PanelHeight={this.timelineContentHeight} + PanelWidth={this.timelineContentWidth} + /> + {this.renderDictation} + + <div + className="collectionStackedTimeline-current" + style={{ + left: `${((this.currentTime - this.clipStart) / this.clipDuration) * 100}%`, + }} + /> + + {this.IsTrimming !== TrimScope.None && ( + <> <div - className="collectionStackedTimeline-trim-handle" - onPointerDown={this.trimRight} + className="collectionStackedTimeline-trim-shade" + style={{ width: `${((this.trimStart - this.clipStart) / this.clipDuration) * 100}%` }} ></div> - </div> - <div - className="collectionStackedTimeline-trim-shade" - style={{ - left: `${(this.trimEnd / this.duration) * 100}%`, - width: `${((this.duration - this.trimEnd) / this.duration) * 100 - }%`, - }} - ></div> - </> - )} + <div + className="collectionStackedTimeline-trim-controls" + style={{ + left: `${((this.trimStart - this.clipStart) / this.clipDuration) * 100}%`, + width: `${((this.trimEnd - this.trimStart) / this.clipDuration) * 100}%`, + }} + > + <div + className="collectionStackedTimeline-trim-handle" + onPointerDown={this.trimLeft} + ></div> + <div + className="collectionStackedTimeline-trim-handle" + onPointerDown={this.trimRight} + ></div> + </div> + + <div + className="collectionStackedTimeline-trim-shade" + style={{ + left: `${((this.trimEnd - this.clipStart) / this.clipDuration) * 100}%`, + width: `${((this.clipEnd - this.trimEnd) / this.clipDuration) * 100}%`, + }} + ></div> + </> + )} + </div> </div> - </div>); + </div >); } } + +/** + * StackedTimelineAnchor + * creates the anchors to display markers, links, and embedded documents on timeline + */ + interface StackedTimelineAnchorProps { mark: Doc; rangeClickScript: () => ScriptField; @@ -675,6 +761,7 @@ interface StackedTimelineAnchorProps { endTag: string; renderDepth: number; layoutDoc: Doc; + isDocumentActive?: () => boolean; ScreenToLocalTransform: () => Transform; _timeline: HTMLDivElement | null; focus: DocFocusFunc; @@ -684,14 +771,26 @@ interface StackedTimelineAnchorProps { trimStart: number; trimEnd: number; } + + @observer class StackedTimelineAnchor extends React.Component<StackedTimelineAnchorProps> { _lastTimecode: number; _disposer: IReactionDisposer | undefined; + constructor(props: any) { super(props); this._lastTimecode = this.props.currentTimecode(); } + + // updates marker document title to reflect correct timecodes + computeTitle = () => { + const start = Math.max(NumCast(this.props.mark[this.props.startTag]), this.props.trimStart) - this.props.trimStart; + const end = Math.min(NumCast(this.props.mark[this.props.endTag]), this.props.trimEnd) - this.props.trimStart; + return `#${formatTime(start)}-${formatTime(end)}`; + } + + componentDidMount() { this._disposer = reaction( () => this.props.currentTimecode(), @@ -712,6 +811,7 @@ class StackedTimelineAnchor extends React.Component<StackedTimelineAnchorProps> // bcz: when should links be followed? we don't want to move away from the video to follow a link but we can open it in a sidebar/etc. But we don't know that upfront. // for now, we won't follow any links when the lightbox is oepn to avoid "losing" the video. /*(isDictation || !Doc.AreProtosEqual(LightboxView.LightboxDoc, this.props.layoutDoc))*/ + !this.props.layoutDoc.dontAutoFollowLinks && DocListCast(this.props.mark.links).length && time > NumCast(this.props.mark[this.props.startTag]) && time < NumCast(this.props.mark[this.props.endTag]) && @@ -729,9 +829,12 @@ class StackedTimelineAnchor extends React.Component<StackedTimelineAnchorProps> } ); } + componentWillUnmount() { this._disposer?.(); } + + // starting the drag event for anchor resizing onAnchorDown = (e: React.PointerEvent, anchor: Doc, left: boolean): void => { this.props._timeline?.setPointerCapture(e.pointerId); @@ -739,52 +842,59 @@ class StackedTimelineAnchor extends React.Component<StackedTimelineAnchorProps> const rect = (e.target as any).getBoundingClientRect(); return this.props.toTimeline(e.clientX - rect.x, rect.width); }; - const changeAnchor = (anchor: Doc, left: boolean, time: number) => { - const timelineOnly = - Cast(anchor[this.props.startTag], "number", null) !== undefined; + const changeAnchor = (anchor: Doc, left: boolean, time: number | undefined) => { + const timelineOnly = Cast(anchor[this.props.startTag], "number", null) !== undefined; if (timelineOnly) { + if (!left && time !== undefined && time <= NumCast(anchor[this.props.startTag])) time = undefined; Doc.SetInPlace( anchor, left ? this.props.startTag : this.props.endTag, time, true ); + if (!left) Doc.SetInPlace(anchor, "borderRounding", time !== undefined ? undefined : "100%", true); } else { - left - ? (anchor._timecodeToShow = time) - : (anchor._timecodeToHide = time); + anchor[left ? "_timecodeToShow" : "_timecodeToHide"] = time; } return false; }; + var undo: UndoManager.Batch | undefined; + setupMoveUpEvents( this, e, - (e) => changeAnchor(anchor, left, newTime(e)), + (e) => { + if (!undo) undo = UndoManager.StartBatch("drag anchor"); + this.props.setTime(newTime(e)); + return changeAnchor(anchor, left, newTime(e)); + }, (e) => { this.props.setTime(newTime(e)); this.props._timeline?.releasePointerCapture(e.pointerId); + undo?.end(); }, emptyFunction ); } - @action - computeTitle = () => { - const start = Math.max(NumCast(this.props.mark[this.props.startTag]), this.props.trimStart) - this.props.trimStart; - const end = Math.min(NumCast(this.props.mark[this.props.endTag]), this.props.trimEnd) - this.props.trimStart; - return `#${formatTime(start)}-${formatTime(end)}`; + + // context menu + contextMenuItems = () => { + const resetTitle = { script: ScriptField.MakeFunction(`self.title = "#" + formatToTime(self["${this.props.startTag}"]) + "-" + formatToTime(self["${this.props.endTag}"])`)!, icon: "folder-plus", label: "Reset Title" }; + return [resetTitle]; } + + // renders anchor LabelBox renderInner = computedFn(function ( this: StackedTimelineAnchor, mark: Doc, script: undefined | (() => ScriptField), doublescript: undefined | (() => ScriptField), - x: number, - y: number, - width: number, - height: number + screenXf: () => Transform, + width: () => number, + height: () => number ) { const anchor = observable({ view: undefined as any }); const focusFunc = ( @@ -803,42 +913,43 @@ class StackedTimelineAnchor extends React.Component<StackedTimelineAnchorProps> <DocumentView key="view" {...OmitKeys(this.props, ["NativeWidth", "NativeHeight"]).omit} - ref={action((r: DocumentView | null) => (anchor.view = r))} + ref={action((r: DocumentView | null) => anchor.view = r)} Document={mark} DataDoc={undefined} renderDepth={this.props.renderDepth + 1} LayoutTemplate={undefined} LayoutTemplateString={LabelBox.LayoutStringWithTitle(LabelBox, "data", this.computeTitle())} - isDocumentActive={returnFalse} - PanelWidth={() => width} - PanelHeight={() => height} - ScreenToLocalTransform={() => - this.props.ScreenToLocalTransform().translate(-x, -y) - } + isDocumentActive={this.props.isDocumentActive} + PanelWidth={width} + PanelHeight={height} + fitWidth={returnTrue} + ScreenToLocalTransform={screenXf} focus={focusFunc} rootSelected={returnFalse} onClick={script} - onDoubleClick={ - this.props.layoutDoc.autoPlayAnchors ? undefined : doublescript - } + onDoubleClick={this.props.layoutDoc.autoPlayAnchors ? undefined : doublescript} ignoreAutoHeight={false} hideResizeHandles={true} bringToFront={emptyFunction} + contextMenuItems={this.contextMenuItems} scriptContext={this.props.stackedTimeline} /> ), }; }); + anchorScreenToLocalXf = () => this.props.ScreenToLocalTransform().translate(-this.props.left, -this.props.top); + width = () => this.props.width; + height = () => this.props.height; + render() { const inner = this.renderInner( this.props.mark, this.props.rangeClickScript, this.props.rangePlayScript, - this.props.left, - this.props.top, - this.props.width, - this.props.height + this.anchorScreenToLocalXf, + this.width, + this.height ); return ( <> diff --git a/src/client/views/collections/TabDocView.tsx b/src/client/views/collections/TabDocView.tsx index 7e57d0e89..d52746d11 100644 --- a/src/client/views/collections/TabDocView.tsx +++ b/src/client/views/collections/TabDocView.tsx @@ -220,8 +220,8 @@ export class TabDocView extends React.Component<TabDocViewProps> { if (!pinProps?.audioRange && duration !== undefined) { pinDoc.mediaStart = "manual"; pinDoc.mediaStop = "manual"; - pinDoc.presStartTime = 0; - pinDoc.presEndTime = duration; + pinDoc.presStartTime = NumCast(doc.clipStart); + pinDoc.presEndTime = NumCast(doc.clipEnd, duration); } //save position if (pinProps?.setPosition || pinDoc.isInkMask) { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index f927c7934..3c0cc17ca 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -50,6 +50,7 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); +import { InkTranscription } from "../../InkTranscription"; export const panZoomSchema = createSchema({ _panX: "number", @@ -516,7 +517,9 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection case GestureUtils.Gestures.Stroke: const points = ge.points; const B = this.getTransform().transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height); - const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), points, + const times = ge.times; + const strokes: PointData[] = !times || times.length == 0 ? points : points.map((pt, i) => { return { X: pt.X, Y: pt.Y, time: times[i] } }); + const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes, { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() }); if (CurrentUserUtils.SelectedTool === InkTool.Write) { this.unprocessedDocs.push(inkDoc); @@ -1505,6 +1508,11 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection appearanceItems.push({ description: `${this.fitToContent ? "Make Zoomable" : "Scale to Window"}`, event: () => this.Document._fitToBox = !this.fitToContent, icon: !this.fitToContent ? "expand-arrows-alt" : "compress-arrows-alt" }); this.props.ContainingCollectionView && appearanceItems.push({ description: "Ungroup collection", event: this.promoteCollection, icon: "table" }); + + this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to text", event: () => this.transcribeStrokes(false), icon: "font" }); + + this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to math", event: () => this.transcribeStrokes(true), icon: "square-root-alt" }); + !Doc.UserDoc().noviceMode ? appearanceItems.push({ description: "Arrange contents in grid", event: this.layoutDocsInGrid, icon: "table" }) : null; !appearance && ContextMenu.Instance.addItem({ description: "Appearance...", subitems: appearanceItems, icon: "eye" }); @@ -1564,6 +1572,15 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection input.click(); } + @undoBatch + @action + transcribeStrokes = (math: boolean) => { + if (this.props.Document._isGroup) { + const inkdocs = this.childDocs.filter(s => s.type === DocumentType.INK); + InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.addDocument); + } + } + @action setupDragLines = (snapToDraggedDoc: boolean = false) => { const activeDocs = this.getActiveDocuments(); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index 1f59f9732..8a8b528f6 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -14,7 +14,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean) => void = unimplementedFunction; public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; - public inkToText: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; @@ -64,16 +63,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { </button> </Tooltip>, ]; - if (false && !SelectionManager.Views().some(v => v.props.Document.type !== DocumentType.INK)) { - buttons.push( - <Tooltip key="inkToText" title={<div className="dash-tooltip">Change to Text</div>} placement="bottom"> - <button - className="antimodeMenu-button" - onPointerDown={this.inkToText}> - <FontAwesomeIcon icon="font" size="lg" /> - </button> - </Tooltip>); - } return this.getElement(buttons); } }
\ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 984e22d95..f762c6619 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,14 +1,15 @@ import { action, computed, observable } from "mobx"; import { observer } from "mobx-react"; -import { AclAugment, AclAdmin, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc"; +import { AclAdmin, AclAugment, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc"; import { Id } from "../../../../fields/FieldSymbols"; import { InkData, InkField, InkTool } from "../../../../fields/InkField"; import { List } from "../../../../fields/List"; import { RichTextField } from "../../../../fields/RichTextField"; import { SchemaHeaderField } from "../../../../fields/SchemaHeaderField"; import { Cast, FieldValue, NumCast, StrCast } from "../../../../fields/Types"; +import { ImageField } from "../../../../fields/URLField"; import { GetEffectiveAcl } from "../../../../fields/util"; -import { Utils, intersectRect, returnFalse } from "../../../../Utils"; +import { intersectRect, returnFalse, Utils } from "../../../../Utils"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; import { Docs, DocumentOptions, DocUtils } from "../../../documents/Documents"; import { DocumentType } from "../../../documents/DocumentTypes"; @@ -21,18 +22,17 @@ import { ContextMenu } from "../../ContextMenu"; import { FormattedTextBox } from "../../nodes/formattedText/FormattedTextBox"; import { PresBox } from "../../nodes/trails/PresBox"; import { PresMovement } from "../../nodes/trails/PresEnums"; +import { VideoBox } from "../../nodes/VideoBox"; +import { pasteImageBitmap } from "../../nodes/WebBoxRenderer"; import { PreviewCursor } from "../../PreviewCursor"; +import { StyleLayers } from "../../StyleProvider"; import { CollectionDockingView } from "../CollectionDockingView"; import { SubCollectionViewProps } from "../CollectionSubView"; import { CollectionView } from "../CollectionView"; +import { TreeView } from "../TreeView"; import { MarqueeOptionsMenu } from "./MarqueeOptionsMenu"; import "./MarqueeView.scss"; import React = require("react"); -import { StyleLayers } from "../../StyleProvider"; -import { TreeView } from "../TreeView"; -import { VideoBox } from "../../nodes/VideoBox"; -import { ImageField, WebField } from "../../../../fields/URLField"; -import { pasteImageBitmap } from "../../nodes/WebBoxRenderer"; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -267,7 +267,7 @@ export class MarqueeView extends React.Component<SubCollectionViewProps & Marque MarqueeOptionsMenu.Instance.createCollection = this.collection; MarqueeOptionsMenu.Instance.delete = this.delete; MarqueeOptionsMenu.Instance.summarize = this.summary; - MarqueeOptionsMenu.Instance.inkToText = this.syntaxHighlight; + // MarqueeOptionsMenu.Instance.inkToText = this.syntaxHighlight; MarqueeOptionsMenu.Instance.showMarquee = this.showMarquee; MarqueeOptionsMenu.Instance.hideMarquee = this.hideMarquee; MarqueeOptionsMenu.Instance.jumpTo(e.clientX, e.clientY); diff --git a/src/client/views/collections/collectionLinear/CollectionLinearView.scss b/src/client/views/collections/collectionLinear/CollectionLinearView.scss index 968048e39..e8df192cf 100644 --- a/src/client/views/collections/collectionLinear/CollectionLinearView.scss +++ b/src/client/views/collections/collectionLinear/CollectionLinearView.scss @@ -17,7 +17,7 @@ background-color: $medium-blue-alt; } - >input:not(:checked)~&.true { + > input:not(:checked) ~ &.true { background-color: transparent; } @@ -31,7 +31,7 @@ overflow: visible !important; } - >span { + > span { background: $dark-gray; color: $white; border-radius: 18px; @@ -39,6 +39,10 @@ cursor: pointer; } + .audio-title:hover { + text-decoration: underline; + } + .bottomPopup-background { background: $medium-blue; display: flex; @@ -58,6 +62,7 @@ padding-right: 20px; vertical-align: middle; font-size: 12.5px; + pointer-events: all; } .bottomPopup-descriptions { @@ -86,7 +91,7 @@ color: black; } - >label { + > label { pointer-events: all; cursor: pointer; background-color: $medium-blue; @@ -104,20 +109,20 @@ justify-content: center; transition: 0.2s; - &:hover{ + &:hover { filter: brightness(0.85); } } - >input { + > input { display: none; } - >input:not(:checked)~.collectionLinearView-content { + > input:not(:checked) ~ .collectionLinearView-content { display: none; } - >input:checked~label { + > input:checked ~ label { transform: rotate(45deg); transition: transform 0.5s; cursor: pointer; @@ -151,4 +156,4 @@ } } } -}
\ No newline at end of file +} diff --git a/src/client/views/collections/collectionLinear/CollectionLinearView.tsx b/src/client/views/collections/collectionLinear/CollectionLinearView.tsx index 44762dbe3..160134b60 100644 --- a/src/client/views/collections/collectionLinear/CollectionLinearView.tsx +++ b/src/client/views/collections/collectionLinear/CollectionLinearView.tsx @@ -7,13 +7,17 @@ import { Doc, HeightSym, Opt, WidthSym } from '../../../../fields/Doc'; import { Id } from '../../../../fields/FieldSymbols'; import { BoolCast, NumCast, ScriptCast, StrCast } from '../../../../fields/Types'; import { emptyFunction, returnEmptyDoclist, returnTrue, Utils } from '../../../../Utils'; +import { DocUtils } from '../../../documents/Documents'; +import { DocumentManager } from "../../../util/DocumentManager"; import { DragManager } from '../../../util/DragManager'; import { Transform } from '../../../util/Transform'; import { Colors, Shadows } from '../../global/globalEnums'; +import { AudioBox } from '../../nodes/AudioBox'; import { DocumentLinksButton } from '../../nodes/DocumentLinksButton'; import { DocumentView } from '../../nodes/DocumentView'; import { LinkDescriptionPopup } from '../../nodes/LinkDescriptionPopup'; import { StyleProp } from '../../StyleProvider'; +import { CollectionStackedTimeline } from '../CollectionStackedTimeline'; import { CollectionSubView } from '../CollectionSubView'; import { CollectionViewType } from '../CollectionView'; import "./CollectionLinearView.scss"; @@ -117,14 +121,14 @@ export class CollectionLinearView extends CollectionSubView() { } - getDisplayDoc = (doc: Doc) => { + getDisplayDoc = (doc: Doc, preview: boolean = false) => { const nested = doc._viewType === CollectionViewType.Linear; const hidden = doc.hidden === true; let dref: Opt<HTMLDivElement>; const docXf = () => this.getTransform(dref); // const scalable = pair.layout.onClick || pair.layout.onDragStart; - return hidden ? (null) : <div className={`collectionLinearView-docBtn`} key={doc[Id]} ref={r => dref = r || undefined} + return hidden ? (null) : <div className={preview ? "preview" : `collectionLinearView-docBtn`} key={doc[Id]} ref={r => dref = r || undefined} style={{ pointerEvents: "all", width: nested ? undefined : NumCast(doc._width), @@ -158,7 +162,8 @@ export class CollectionLinearView extends CollectionSubView() { docRangeFilters={this.props.docRangeFilters} searchFilterDocs={this.props.searchFilterDocs} ContainingCollectionView={undefined} - ContainingCollectionDoc={undefined} /> + ContainingCollectionDoc={undefined} + hideResizeHandles={true} /> </div>; } @@ -222,6 +227,16 @@ export class CollectionLinearView extends CollectionSubView() { </Tooltip> </span> : null} + {CollectionStackedTimeline.CurrentlyPlaying && CollectionStackedTimeline.CurrentlyPlaying.length != 0 && StrCast(this.layoutDoc.title) === "docked buttons" ? <span className="bottomPopup-background"> + <span className="bottomPopup-text"> + Currently playing: {CollectionStackedTimeline.CurrentlyPlaying.map((clip, i) => + <span className="audio-title" onPointerDown={() => { + DocumentManager.Instance.jumpToDocument(clip, true); + }}>{clip.title + (i == CollectionStackedTimeline.CurrentlyPlaying.length - 1 ? "" : ",")} </span> + )} + </span> + </span> : null} + </div> </div>; } diff --git a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx index 8b73351d5..a93762ea4 100644 --- a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx +++ b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx @@ -1,9 +1,10 @@ import React = require("react"); import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, observable, untracked, trace } from "mobx"; +import { action, computed, observable, untracked } from "mobx"; import { observer } from "mobx-react"; import Measure from "react-measure"; import { Resize } from "react-table"; +import "react-table/react-table.css"; import { Doc, Opt } from "../../../../fields/Doc"; import { List } from "../../../../fields/List"; import { listSpec } from "../../../../fields/Schema"; @@ -16,7 +17,6 @@ import { SelectionManager } from "../../../util/SelectionManager"; import { SnappingManager } from "../../../util/SnappingManager"; import { Transform } from "../../../util/Transform"; import { undoBatch } from "../../../util/UndoManager"; -import '../../../views/DocumentDecorations.scss'; import { ContextMenu } from "../../ContextMenu"; import { ContextMenuProps } from "../../ContextMenuItem"; import { COLLECTION_BORDER_WIDTH, SCHEMA_DIVIDER_WIDTH } from '../../global/globalCssVariables.scss'; diff --git a/src/client/views/nodes/AudioBox.scss b/src/client/views/nodes/AudioBox.scss index a6494e540..d40537776 100644 --- a/src/client/views/nodes/AudioBox.scss +++ b/src/client/views/nodes/AudioBox.scss @@ -1,222 +1,214 @@ @import "../global/globalCssVariables.scss"; - -.audiobox-container, -.audiobox-container-interactive { +.audiobox-container { width: 100%; height: 100%; position: inherit; display: flex; position: relative; cursor: default; +} + +.audiobox-recorder { + display: flex; + flex-direction: row; + overflow: hidden; + width: 100%; + height: 100%; + cursor: pointer; - .audiobox-buttons { + .audiobox-dictation { + width: 40px; + background: $medium-gray; + color: $dark-gray; + display: flex; + justify-content: center; + align-items: center; + + &:hover { + color: $black; + } + } + + .audiobox-start-record { + color: $white; + background: $dark-gray; display: flex; + align-items: center; + justify-content: center; + font-size: $body-text; width: 100%; + height: 100%; + gap: 5px; + + &:hover { + background: $black; + } + } + + .recording-controls { + display: flex; + flex-direction: row; align-items: center; + justify-content: center; + gap: 5px; + width: 100%; height: 100%; + background: $dark-gray; + color: white; - .audiobox-dictation { - position: relative; + .record-timecode { + font-size: $large-header; + } + + .record-button { + cursor: pointer; width: 30px; - height: 100%; + height: 30px; + border-radius: 50%; + background: $dark-gray; + display: flex; align-items: center; - display: inherit; - background: $medium-gray; - left: 0px; - color: $dark-gray; + justify-content: center; + + svg { + width: 15px; + } &:hover { - color: $black; - cursor: pointer; + background: $black; } } } +} - .audiobox-control, - .audiobox-control-interactive { - top: 0; - max-height: 32px; - width: 100%; - display: inline-block; - pointer-events: none; - } - - .audiobox-control-interactive { - pointer-events: all; - } +.audiobox-file { + overflow: hidden; + display: flex; + flex-direction: column; + align-items: center; + background: $dark-gray; + width: 100%; + height: 100%; + color: $white; - .audiobox-record-interactive, - .audiobox-record { - pointer-events: all; + .audiobox-button { + margin: 2.5px; cursor: pointer; - width: 100%; - height: 100%; - position: relative; + width: 25px; + height: 25px; + border-radius: 50%; + background: $dark-gray; display: flex; - flex-direction: row; align-items: center; justify-content: center; - gap: 10px; - color: white; - font-weight: bold; + + svg { + width: 15px; + } + + &:hover { + background: $black; + } + } + + svg { + width: 10px; + } + + input[type="range"] { + width: 50px; + -webkit-appearance: none; + background: none; + margin: 5px; } - .audiobox-record { - pointer-events: none; + input[type="range"]:focus { + outline: none; } - .recording { - margin-top: auto; - margin-bottom: auto; + input[type="range"]::-webkit-slider-runnable-track { width: 100%; - height: 100%; - position: relative; - padding-right: 5px; + height: 6px; + cursor: pointer; + box-shadow: 0; + background: $light-gray; + border-radius: 3px; + } + + input[type="range"]::-webkit-slider-thumb { + box-shadow: 0; + border: 0; + height: 10px; + width: 10px; + border-radius: 10px; + background: $medium-blue; + cursor: pointer; + -webkit-appearance: none; + margin-top: -2px; + } + + .audiobox-controls { display: flex; flex-direction: row; - justify-content: center; + justify-content: space-between; align-items: center; - gap: 7px; - background-color: $medium-blue; - padding: 10px; + width: 100%; + height: 30px; - .time { - position: relative; - height: 100%; - width: 100%; - font-size: 16px; - text-align: center; + .controls-left { display: flex; - justify-content: center; - align-items: center; - font-weight: bold; + flex-direction: row; } - .buttons { - cursor: pointer; - position: relative; - margin-top: auto; - margin-bottom: auto; - width: 25px; - width: 25px; - padding: 5px; - color: $dark-gray; + .controls-right { + display: flex; + flex-direction: row; - &:hover { - color: $black; + .audiobox-button { + width: 15px; + height: 15px; + margin: 0; + + svg { + width: 10px; + } } } } - .audiobox-controls { + .audiobox-playback { width: 100%; height: 100%; - position: relative; - display: flex; - background: $dark-gray; + background: $white; - .audiobox-dictation { + .audiobox-timeline { + height: calc(100% - 50px); + width: 100%; + background: $white; position: absolute; - width: 40px; - height: 100%; - align-items: center; - display: inherit; - background: $medium-gray; - left: 0px; } - .audiobox-player { - margin-top: auto; - margin-bottom: auto; + .audiobox-timeline > div { width: 100%; - position: relative; - padding-right: 5px; - display: flex; - flex-direction: column; - justify-content: center; - - .audiobox-buttons { - position: relative; - margin-top: auto; - margin-bottom: auto; - width: 30px; - height: 30px; - border-radius: 50%; - background-color: $dark-gray; - color: $white; - display: flex; - align-items: center; - justify-content: center; - left: 5px; - - &:hover { - background-color: $black; - } - - svg { - width: 100%; - position: absolute; - border-width: "thin"; - border-color: "white"; - } - } - - .audiobox-dictation { - position: relative; - margin-top: auto; - margin-bottom: auto; - width: 25px; - align-items: center; - display: inherit; - background: $medium-gray; - } - - .audiobox-timeline { - position: absolute; - width: 100%; - z-index: 1000; - overflow: hidden; - border-right: 5px solid black; - } - - .audioBox-total-time, - .audioBox-current-time { - position: absolute; - font-size: $small-text; - top: 100%; - color: $white; - } - - .audioBox-current-time { - left: 42px; - } - - .audioBox-total-time { - right: 2px; - } + height: 100%; } } -} -@media only screen and (max-device-width: 480px) { - .audiobox-dictation { - font-size: 5em; + .audiobox-timecodes { display: flex; - width: 100; - justify-content: center; - flex-direction: column; + flex-direction: row; + justify-content: space-between; align-items: center; - } - - .audiobox-container .audiobox-record, - .audiobox-container-interactive .audiobox-record { - font-size: 3em; - } + width: 100%; + height: 20px; + padding: 3px; + font-size: $small-text; - .audiobox-container .audiobox-controls .audiobox-player .audiobox-buttons, - .audiobox-container .audiobox-controls .audiobox-player .audiobox-dictation, - .audiobox-container-interactive .audiobox-controls .audiobox-player .audiobox-buttons { - width: 70px; + .bottom-controls-middle { + display: flex; + flex-direction: row; + align-items: center; + } } -}
\ No newline at end of file +} diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx index 93377f1dc..be18cc1de 100644 --- a/src/client/views/nodes/AudioBox.tsx +++ b/src/client/views/nodes/AudioBox.tsx @@ -1,162 +1,105 @@ import React = require("react"); import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { - action, - computed, - IReactionDisposer, - observable, - reaction, - runInAction -} from "mobx"; +import { action, computed, IReactionDisposer, observable, runInAction } from "mobx"; import { observer } from "mobx-react"; import { DateField } from "../../../fields/DateField"; -import { Doc, DocListCast, Opt } from "../../../fields/Doc"; +import { Doc, DocListCast } from "../../../fields/Doc"; +import { documentSchema } from "../../../fields/documentSchemas"; +import { makeInterface } from "../../../fields/Schema"; import { ComputedField } from "../../../fields/ScriptField"; -import { Cast, NumCast } from "../../../fields/Types"; +import { Cast, DateCast, NumCast } from "../../../fields/Types"; import { AudioField, nullAudio } from "../../../fields/URLField"; -import { emptyFunction, formatTime } from "../../../Utils"; +import { emptyFunction, formatTime, OmitKeys, returnFalse, setupMoveUpEvents } from "../../../Utils"; import { DocUtils } from "../../documents/Documents"; import { Networking } from "../../Network"; import { CurrentUserUtils } from "../../util/CurrentUserUtils"; -import { SnappingManager } from "../../util/SnappingManager"; -import { CollectionStackedTimeline } from "../collections/CollectionStackedTimeline"; +import { DragManager } from "../../util/DragManager"; +import { undoBatch } from "../../util/UndoManager"; +import { CollectionStackedTimeline, TrimScope } from "../collections/CollectionStackedTimeline"; import { ContextMenu } from "../ContextMenu"; import { ContextMenuProps } from "../ContextMenuItem"; -import { - ViewBoxAnnotatableComponent, - ViewBoxAnnotatableProps -} from "../DocComponent"; -import { Colors } from "../global/globalEnums"; +import { ViewBoxAnnotatableComponent, ViewBoxAnnotatableProps } from "../DocComponent"; import "./AudioBox.scss"; import { FieldView, FieldViewProps } from "./FieldView"; -import { LinkDocPreview } from "./LinkDocPreview"; + +/** + * AudioBox + * Main component: AudioBox.tsx + * Supporting Components: CollectionStackedTimeline, AudioWaveform + * + * AudioBox is a node that supports the recording and playback of audio files in Dash. + * When an audio file is importeed into Dash, it is immediately rendered as an AudioBox document. + * When a blank AudioBox node is created in Dash, audio recording controls are displayed and the user can start a recording which can be paused or stopped, and can use dictation to create a text transcript. + * Recording is done using the MediaDevices API to access the user's device microphone (see recordAudioAnnotation below) + * CollectionStackedTimeline handles AudioBox and VideoBox shared behavior, but AudioBox handles playing, pausing, etc because it contains <audio> element + * User can trim audio: nondestructive, just sets new bounds for playback and rendering timelin + */ + + +// used as a wrapper class for MediaStream from MediaDevices API declare class MediaRecorder { constructor(e: any); // whatever MediaRecorder has } + +enum media_state { + PendingRecording = "pendingRecording", + Recording = "recording", + Paused = "paused", + Playing = "playing" +} + + @observer export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps & FieldViewProps>() { - public static LayoutString(fieldKey: string) { - return FieldView.LayoutString(AudioBox, fieldKey); - } + + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(AudioBox, fieldKey); } public static Enabled = false; - static playheadWidth = 40; // width of playhead - static heightPercent = 75; // height of timeline in percent of height of audioBox. - static Instance: AudioBox; + static topControlsHeight = 30; // height of upper controls above timeline + static bottomControlsHeight = 20; // height of lower controls below timeline + + _dropDisposer?: DragManager.DragDropDisposer; _disposers: { [name: string]: IReactionDisposer } = {}; - _ele: HTMLAudioElement | null = null; - _stackedTimeline = React.createRef<CollectionStackedTimeline>(); - _recorder: any; + _ele: HTMLAudioElement | null = null; // <audio> ref + _recorder: any; // MediaRecorder _recordStart = 0; - _pauseStart = 0; + _pauseStart = 0; // time when recording is paused (used to keep track of recording timecodes) _pauseEnd = 0; _pausedTime = 0; - _stream: MediaStream | undefined; - _start: number = 0; - _play: any = null; - _ended: boolean = false; - - @observable static _scrubTime = 0; - @observable _markerEnd: number = 0; - @observable _position: number = 0; - @observable _waveHeight: Opt<number> = NumCast(this.layoutDoc._height); - @observable _paused: boolean = false; - @observable _trimming: boolean = false; - @observable _trimStart: number = NumCast(this.layoutDoc.clipStart) ? NumCast(this.layoutDoc.clipStart) : 0; - @observable _trimEnd: number = NumCast(this.layoutDoc.clipEnd) ? NumCast(this.layoutDoc.clipEnd) - : this.duration; - - @computed get mediaState(): - | undefined - | "pendingRecording" - | "recording" - | "paused" - | "playing" { - return this.dataDoc.mediaState as - | undefined - | "pendingRecording" - | "recording" - | "paused" - | "playing"; - } - set mediaState(value) { - this.dataDoc.mediaState = value; - } - public static SetScrubTime = action((timeInMillisFrom1970: number) => { - AudioBox._scrubTime = 0; - AudioBox._scrubTime = timeInMillisFrom1970; - }); - @computed get recordingStart() { - return Cast( - this.dataDoc[this.props.fieldKey + "-recordingStart"], - DateField - )?.date.getTime(); - } - @computed get duration() { - return NumCast(this.dataDoc[`${this.fieldKey}-duration`]); - } - @computed get trimDuration() { - return this._trimming && this._trimEnd ? this.duration : this._trimEnd - this._trimStart; - } - @computed get anchorDocs() { - return DocListCast(this.dataDoc[this.annotationKey]); - } - @computed get links() { - return DocListCast(this.dataDoc.links); - } - @computed get pauseTime() { - return this._pauseEnd - this._pauseStart; - } // total time paused to update the correct time - @computed get heightPercent() { - return AudioBox.heightPercent; - } - - constructor(props: Readonly<ViewBoxAnnotatableProps & FieldViewProps>) { - super(props); - AudioBox.Instance = this; - - if (this.duration === undefined) { - runInAction( - () => - (this.Document[this.fieldKey + "-duration"] = this.Document.duration) - ); - } + _stream: MediaStream | undefined; // passed to MediaRecorder, records device input audio + _play: any = null; // timeout for playback + + @observable _stackedTimeline: any; // CollectionStackedTimeline ref + @observable _finished: boolean = false; // has playback reached end of clip + @observable _volume: number = 1; + @observable _muted: boolean = false; + @observable _paused: boolean = false; // is recording paused + // @observable rawDuration: number = 0; // computed from the length of the audio element when loaded + @computed get recordingStart() { return DateCast(this.dataDoc[this.fieldKey + "-recordingStart"])?.date.getTime(); } + @computed get rawDuration() { return NumCast(this.dataDoc[`${this.fieldKey}-duration`]); } // bcz: shouldn't be needed since it's computed from audio element + // mehek: not 100% sure but i think due to the order in which things are loaded this is necessary ^^ + // if you get rid of it and set the value to 0 the timeline and waveform will set their bounds incorrectly + + @computed get miniPlayer() { return this.props.PanelHeight() < 50 } // used to collapse timeline when node is shrunk + @computed get links() { return DocListCast(this.dataDoc.links); } + @computed get pauseTime() { return this._pauseEnd - this._pauseStart; } // total time paused to update the correct recording time + @computed get mediaState() { return this.layoutDoc.mediaState as media_state; } + @computed get path() { // returns the path of the audio file + const path = Cast(this.props.Document[this.fieldKey], AudioField, null)?.url.href || ""; + return path === nullAudio ? "" : path; } + set mediaState(value) { this.layoutDoc.mediaState = value; } - getLinkData(l: Doc) { - let la1 = l.anchor1 as Doc; - let la2 = l.anchor2 as Doc; - const linkTime = - this._stackedTimeline.current?.anchorStart(la2) || - this._stackedTimeline.current?.anchorStart(la1) || - 0; - if (Doc.AreProtosEqual(la1, this.dataDoc)) { - la1 = l.anchor2 as Doc; - la2 = l.anchor1 as Doc; - } - return { la1, la2, linkTime }; - } + @computed get timeline() { return this._stackedTimeline; } // returns CollectionStackedTimeline ref - getAnchor = () => { - return ( - CollectionStackedTimeline.createAnchor( - this.rootDoc, - this.dataDoc, - this.annotationKey, - "_timecodeToShow" /* audioStart */, - "_timecodeToHide" /* audioEnd */, - this._ele?.currentTime || - Cast(this.props.Document._currentTimecode, "number", null) || - (this.mediaState === "recording" - ? (Date.now() - (this.recordingStart || 0)) / 1000 - : undefined) - ) || this.rootDoc - ); - } componentWillUnmount() { + this.removeCurrentlyPlaying(); + this._dropDisposer?.(); Object.values(this._disposers).forEach((disposer) => disposer?.()); + + // removes doc from active recordings if recording when closed const ind = DocUtils.ActiveRecordings.indexOf(this); ind !== -1 && DocUtils.ActiveRecordings.splice(ind, 1); } @@ -165,128 +108,124 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp componentDidMount() { this.props.setContentView?.(this); // this tells the DocumentView that this AudioBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. - this.mediaState = this.path ? "paused" : undefined; - - this.layoutDoc.clipStart = this.layoutDoc.clipStart ? this.layoutDoc.clipStart : 0; - this.layoutDoc.clipEnd = this.layoutDoc.clipEnd ? this.layoutDoc.clipEnd : this.duration ? this.duration : undefined; - - this.path && this.setAnchorTime(NumCast(this.layoutDoc.clipStart)); - this.path && this.timecodeChanged(); - - this._disposers.triggerAudio = reaction( - () => - !LinkDocPreview.LinkInfo && this.props.renderDepth !== -1 - ? NumCast(this.Document._triggerAudio, null) - : undefined, - (start) => - start !== undefined && - setTimeout(() => { - this.playFrom(start); - setTimeout(() => { - this.Document._currentTimecode = start; - this.Document._triggerAudio = undefined; - }, 10); - }), // wait for mainCont and try again to play - { fireImmediately: true } - ); + if (this.path) { + this.mediaState = media_state.Paused; + this.setPlayheadTime(NumCast(this.layoutDoc.clipStart)); + } else { + this.mediaState = undefined as any as media_state; + } + } - this._disposers.audioStop = reaction( - () => - this.props.renderDepth !== -1 && !LinkDocPreview.LinkInfo - ? Cast(this.Document._audioStop, "number", null) - : undefined, - (audioStop) => - audioStop !== undefined && - setTimeout(() => { - this.Pause(); - setTimeout(() => (this.Document._audioStop = undefined), 10); - }), // wait for mainCont and try again to play - { fireImmediately: true } - ); + + getLinkData(l: Doc) { + let la1 = l.anchor1 as Doc; + let la2 = l.anchor2 as Doc; + const linkTime = + this.timeline?.anchorStart(la2) || + this.timeline?.anchorStart(la1) || + 0; + if (Doc.AreProtosEqual(la1, this.dataDoc)) { + la1 = l.anchor2 as Doc; + la2 = l.anchor1 as Doc; + } + return { la1, la2, linkTime }; } - // for updating the timecode + getAnchor = () => { + return CollectionStackedTimeline.createAnchor( + this.rootDoc, + this.dataDoc, + this.annotationKey, + "_timecodeToShow" /* audioStart */, + "_timecodeToHide" /* audioEnd */, + this._ele?.currentTime || + Cast(this.props.Document._currentTimecode, "number", null) || + (this.mediaState === media_state.Recording + ? (Date.now() - (this.recordingStart || 0)) / 1000 + : undefined) + ) || this.rootDoc; + } + + + // updates timecode and shows it in timeline, follows links at time @action timecodeChanged = () => { - const htmlEle = this._ele; - if (this.mediaState !== "recording" && htmlEle) { - htmlEle.duration && - htmlEle.duration !== Infinity && - runInAction( - () => (this.dataDoc[this.fieldKey + "-duration"] = htmlEle.duration) - ); - this.layoutDoc.clipEnd = this.layoutDoc.clipEnd ? Math.min(this.duration, NumCast(this.layoutDoc.clipEnd)) : this.duration; - this._trimEnd = this._trimEnd ? Math.min(this.duration, this._trimEnd) : this.duration; + if (this.mediaState !== media_state.Recording && this._ele) { this.links - .map((l) => this.getLinkData(l)) + .map(l => this.getLinkData(l)) .forEach(({ la1, la2, linkTime }) => { - if ( - linkTime > NumCast(this.layoutDoc._currentTimecode) && - linkTime < htmlEle.currentTime - ) { + if (linkTime > NumCast(this.layoutDoc._currentTimecode) && + linkTime < this._ele!.currentTime) { Doc.linkFollowHighlight(la1); } }); - this.layoutDoc._currentTimecode = htmlEle.currentTime; - + this.layoutDoc._currentTimecode = this._ele.currentTime; + this.timeline?.scrollToTime(NumCast(this.layoutDoc._currentTimecode)); } } - // pause play back - Pause = action(() => { - this._ele!.pause(); - this.mediaState = "paused"; - }); - - // play audio for documents created during recording - playFromTime = (absoluteTime: number) => { - this.recordingStart && - this.playFrom((absoluteTime - this.recordingStart) / 1000); - } - - // play back the audio from time + // play back the audio from seekTimeInSeconds, fullPlay tells whether clip is being played to end vs link range @action - playFrom = (seekTimeInSeconds: number, endTime: number = this._trimEnd, fullPlay: boolean = false) => { - clearTimeout(this._play); - if (Number.isNaN(this._ele?.duration)) { + playFrom = (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => { + clearTimeout(this._play); // abort any previous clip ending + if (Number.isNaN(this._ele?.duration)) { // audio element isn't loaded yet... wait 1/2 second and try again setTimeout(() => this.playFrom(seekTimeInSeconds, endTime), 500); - } else if (this._ele && AudioBox.Enabled) { - if (seekTimeInSeconds < 0) { - if (seekTimeInSeconds > -1) { - setTimeout(() => this.playFrom(0), -seekTimeInSeconds * 1000); - } else { - this.Pause(); - } - } else if (this._trimStart <= endTime && seekTimeInSeconds <= this._trimEnd) { - const start = Math.max(this._trimStart, seekTimeInSeconds); - const end = Math.min(this._trimEnd, endTime); + } + else if (this.timeline && this._ele && AudioBox.Enabled) { + // trimBounds override requested playback bounds + const end = Math.min(this.timeline.trimEnd, endTime ?? this.timeline.trimEnd); + const start = Math.max(this.timeline.trimStart, seekTimeInSeconds); + // checks if times are within clip range + if (seekTimeInSeconds >= 0 && this.timeline.trimStart <= end && seekTimeInSeconds <= this.timeline.trimEnd) { this._ele.currentTime = start; this._ele.play(); - runInAction(() => (this.mediaState = "playing")); - if (endTime !== this.duration) { - this._play = setTimeout( - () => { - this._ended = fullPlay ? true : this._ended; - this.Pause(); - }, - (end - start) * 1000 - ); // use setTimeout to play a specific duration - } + this.mediaState = media_state.Playing; + this.addCurrentlyPlaying(); + this._play = setTimeout( + () => { + // need to keep track of if end of clip is reached so on next play, clip restarts + if (fullPlay) this._finished = true; + // removes from currently playing if playback has reached end of range marker + else this.removeCurrentlyPlaying(); + this.Pause(); + }, + (end - start) * 1000); } else { this.Pause(); } } } + + // removes from currently playing display + @action + removeCurrentlyPlaying = () => { + if (CollectionStackedTimeline.CurrentlyPlaying) { + const index = CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc.doc as Doc); + index !== -1 && CollectionStackedTimeline.CurrentlyPlaying.splice(index, 1); + } + } + + // adds doc to currently playing display + @action + addCurrentlyPlaying = () => { + if (!CollectionStackedTimeline.CurrentlyPlaying) { + CollectionStackedTimeline.CurrentlyPlaying = []; + } + if (CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc.doc as Doc) == -1) { + CollectionStackedTimeline.CurrentlyPlaying.push(this.layoutDoc.doc as Doc); + } + } + + // update the recording time updateRecordTime = () => { - if (this.mediaState === "recording") { + if (this.mediaState === media_state.Recording) { setTimeout(this.updateRecordTime, 30); if (this._paused) { this._pausedTime += (new Date().getTime() - this._recordStart) / 1000; } else { - this.layoutDoc._currentTimecode = - (new Date().getTime() - this._recordStart - this.pauseTime) / 1000; + this.layoutDoc._currentTimecode = (new Date().getTime() - this._recordStart - this.pauseTime) / 1000; } } } @@ -295,49 +234,59 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp recordAudioAnnotation = async () => { this._stream = await navigator.mediaDevices.getUserMedia({ audio: true }); this._recorder = new MediaRecorder(this._stream); - this.dataDoc[this.props.fieldKey + "-recordingStart"] = new DateField( - new Date() - ); + this.dataDoc[this.fieldKey + "-recordingStart"] = new DateField(); DocUtils.ActiveRecordings.push(this); this._recorder.ondataavailable = async (e: any) => { console.log("Data available", e); const [{ result }] = await Networking.UploadFilesToServer(e.data); console.log("Data result", result); if (!(result instanceof Error)) { - this.props.Document[this.props.fieldKey] = new AudioField(result.accessPaths.agnostic.client); + this.props.Document[this.fieldKey] = new AudioField(result.accessPaths.agnostic.client); } }; this._recordStart = new Date().getTime(); - runInAction(() => (this.mediaState = "recording")); - setTimeout(this.updateRecordTime, 0); + runInAction(() => this.mediaState = media_state.Recording); + setTimeout(this.updateRecordTime); this._recorder.start(); - setTimeout(() => this._recorder && this.stopRecording(), 60 * 60 * 1000); // stop after an hour + setTimeout(this.stopRecording, 60 * 60 * 1000); // stop after an hour + } + + // stops recording + @action + stopRecording = () => { + if (this._recorder) { + this._recorder.stop(); + this._recorder = undefined; + this.dataDoc[this.fieldKey + "-duration"] = (new Date().getTime() - this._recordStart - this.pauseTime) / 1000; + this.mediaState = media_state.Paused; + this._stream?.getAudioTracks()[0].stop(); + const ind = DocUtils.ActiveRecordings.indexOf(this); + ind !== -1 && DocUtils.ActiveRecordings.splice(ind, 1); + } } + // context menu specificContextMenu = (e: React.MouseEvent): void => { const funcs: ContextMenuProps[] = []; funcs.push({ - description: - (this.layoutDoc.hideAnchors ? "Don't hide" : "Hide") + " anchors", - event: () => (this.layoutDoc.hideAnchors = !this.layoutDoc.hideAnchors), + description: (this.layoutDoc.hideAnchors ? "Don't hide" : "Hide") + " anchors", + event: e => this.layoutDoc.hideAnchors = !this.layoutDoc.hideAnchors, icon: "expand-arrows-alt", }); funcs.push({ - description: - (this.layoutDoc.dontAutoPlayFollowedLinks ? "" : "Don't") + - " play when link is selected", - event: () => - (this.layoutDoc.dontAutoPlayFollowedLinks = - !this.layoutDoc.dontAutoPlayFollowedLinks), + description: (this.layoutDoc.dontAutoFollowLinks ? "" : "Don't") + " follow links when encountered", + event: e => this.layoutDoc.dontAutoFollowLinks = !this.layoutDoc.dontAutoFollowLinks, icon: "expand-arrows-alt", }); funcs.push({ - description: - (this.layoutDoc.autoPlayAnchors ? "Don't auto play" : "Auto play") + - " anchors onClick", - event: () => - (this.layoutDoc.autoPlayAnchors = !this.layoutDoc.autoPlayAnchors), + description: (this.layoutDoc.dontAutoPlayFollowedLinks ? "" : "Don't") + " play when link is selected", + event: e => this.layoutDoc.dontAutoPlayFollowedLinks = !this.layoutDoc.dontAutoPlayFollowedLinks, + icon: "expand-arrows-alt", + }); + funcs.push({ + description: (this.layoutDoc.autoPlayAnchors ? "Don't auto" : "Auto") + " play anchors onClick", + event: e => this.layoutDoc.autoPlayAnchors = !this.layoutDoc.autoPlayAnchors, icon: "expand-arrows-alt", }); ContextMenu.Instance?.addItem({ @@ -347,23 +296,9 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp }); } - // stops the recording - stopRecording = action(() => { - this._recorder.stop(); - this._recorder = undefined; - this.dataDoc[this.fieldKey + "-duration"] = - (new Date().getTime() - this._recordStart - this.pauseTime) / 1000; - this.mediaState = "paused"; - this._trimEnd = this.duration; - this.layoutDoc.clipStart = 0; - this.layoutDoc.clipEnd = this.duration; - this._stream?.getAudioTracks()[0].stop(); - const ind = DocUtils.ActiveRecordings.indexOf(this); - ind !== -1 && DocUtils.ActiveRecordings.splice(ind, 1); - }); // button for starting and stopping the recording - recordClick = (e: React.MouseEvent) => { + Record = (e: React.MouseEvent) => { if (e.button === 0 && !e.ctrlKey) { this._recorder ? this.stopRecording() : this.recordAudioAnnotation(); e.stopPropagation(); @@ -372,33 +307,51 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp // for play button Play = (e?: any) => { - let start; - if (this._ended || this._ele!.currentTime === this.duration) { - start = this._trimStart; - this._ended = false; - } - else { - start = this._ele!.currentTime; + e?.stopPropagation?.(); + + if (this.timeline && this._ele) { + const eleTime = this._ele.currentTime; + + // if curr timecode outside of trim bounds, set it to start + let start = eleTime >= this.timeline.trimEnd || eleTime <= this.timeline.trimStart ? this.timeline.trimStart : eleTime; + + // restarts clip if reached end on last play + if (this._finished) { + this._finished = false; + start = this.timeline.trimStart; + } + + this.playFrom(start, this.timeline.trimEnd, true); } + } - this.playFrom(start, this._trimEnd, true); - e?.stopPropagation?.(); + // pause play back + @action + Pause = () => { + if (this._ele) { + this._ele.pause(); + this.mediaState = media_state.Paused; + + // if paused in the middle of playback, prevents restart on next play + if (!this._finished) clearTimeout(this._play); + this.removeCurrentlyPlaying(); + } } - // creates a text document for dictation + // for dictation button, creates a text document for dictation onFile = (e: any) => { const newDoc = CurrentUserUtils.GetNewTextDoc( "", - NumCast(this.props.Document.x), - NumCast(this.props.Document.y) + - NumCast(this.props.Document._height) + + NumCast(this.rootDoc.x), + NumCast(this.rootDoc.y) + + NumCast(this.layoutDoc._height) + 10, - NumCast(this.props.Document._width), - 2 * NumCast(this.props.Document._height) + NumCast(this.layoutDoc._width), + 2 * NumCast(this.layoutDoc._height) ); Doc.GetProto(newDoc).recordingSource = this.dataDoc; Doc.GetProto(newDoc).recordingStart = ComputedField.MakeFunction( - `self.recordingSource["${this.props.fieldKey}-recordingStart"]` + `self.recordingSource["${this.fieldKey}-recordingStart"]` ); Doc.GetProto(newDoc).mediaState = ComputedField.MakeFunction( "self.recordingSource.mediaState" @@ -407,27 +360,14 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp e.stopPropagation(); } - // ref for updating time + + // sets <audio> ref for updating time setRef = (e: HTMLAudioElement | null) => { e?.addEventListener("timeupdate", this.timecodeChanged); - e?.addEventListener("ended", this.Pause); + e?.addEventListener("ended", () => { this._finished = true; this.Pause() }); this._ele = e; } - // returns the path of the audio file - @computed get path() { - const field = Cast(this.props.Document[this.props.fieldKey], AudioField); - const path = field instanceof AudioField ? field.url.href : ""; - return path === nullAudio ? "" : path; - } - - // returns the html audio element - @computed get audio() { - return <audio ref={this.setRef} className={`audiobox-control${this.props.isContentActive() ? "-interactive" : ""}`}> - <source src={this.path} type="audio/mpeg" /> - Not supported. - </audio>; - } // pause the time during recording phase @action @@ -447,97 +387,224 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp e.stopPropagation(); } - playing = () => this.mediaState === "playing"; + + // plays link playLink = (link: Doc) => { - const stack = this._stackedTimeline.current; if (link.annotationOn === this.rootDoc) { if (!this.layoutDoc.dontAutoPlayFollowedLinks) { - this.playFrom(stack?.anchorStart(link) || 0, stack?.anchorEnd(link)); + this.playFrom(this.timeline?.anchorStart(link) || 0, this.timeline?.anchorEnd(link)); } else { - this._ele!.currentTime = this.layoutDoc._currentTimecode = - stack?.anchorStart(link) || 0; + this._ele!.currentTime = this.layoutDoc._currentTimecode = this.timeline?.anchorStart(link) || 0; } } else { this.links .filter((l) => l.anchor1 === link || l.anchor2 === link) .forEach((l) => { const { la1, la2 } = this.getLinkData(l); - const startTime = stack?.anchorStart(la1) || stack?.anchorStart(la2); - const endTime = stack?.anchorEnd(la1) || stack?.anchorEnd(la2); + const startTime = this.timeline?.anchorStart(la1) || this.timeline?.anchorStart(la2); + const endTime = this.timeline?.anchorEnd(la1) || this.timeline?.anchorEnd(la2); if (startTime !== undefined) { if (!this.layoutDoc.dontAutoPlayFollowedLinks) { - endTime - ? this.playFrom(startTime, endTime) - : this.playFrom(startTime); + this.playFrom(startTime, endTime); } else { - this._ele!.currentTime = this.layoutDoc._currentTimecode = - startTime; + this._ele!.currentTime = this.layoutDoc._currentTimecode = startTime; } } }); } } - // shows trim controls + @action - startTrim = () => { - if (!this.duration) { - this.timecodeChanged(); - } - if (this.mediaState === "playing") { - this.Pause(); - } - this._trimming = true; + timelineWhenChildContentsActiveChanged = (isActive: boolean) => + this.props.whenChildContentsActiveChanged(this._isAnyChildContentActive = isActive); + + timelineScreenToLocal = () => + this.props.ScreenToLocalTransform().translate(0, -AudioBox.bottomControlsHeight); + + setPlayheadTime = (time: number) => this._ele!.currentTime = this.layoutDoc._currentTimecode = time; + + playing = () => this.mediaState === media_state.Playing; + + isActiveChild = () => this._isAnyChildContentActive; + + // timeline dimensions + timelineWidth = () => this.props.PanelWidth(); + timelineHeight = () => (this.props.PanelHeight() - (AudioBox.topControlsHeight + AudioBox.bottomControlsHeight)) + + // ends trim, hides trim controls and displays new clip + @undoBatch + finishTrim = () => { + this.Pause(); + this.setPlayheadTime(Math.max(Math.min(this.timeline?.trimEnd || 0, this._ele!.currentTime), this.timeline?.trimStart || 0)); + this.timeline?.StopTrimming(); + } + + // displays trim controls to start trimming clip + startTrim = (scope: TrimScope) => { + this.Pause(); + this.timeline?.StartTrimming(scope); + } + + // for trim button, double click displays full clip, single displays curr trim bounds + onClipPointerDown = (e: React.PointerEvent) => { + e.stopPropagation(); + this.timeline && setupMoveUpEvents(this, e, returnFalse, returnFalse, action((e: PointerEvent, doubleTap?: boolean) => { + if (doubleTap) { + this.startTrim(TrimScope.All); + } else if (this.timeline) { + this.Pause(); + this.timeline.IsTrimming !== TrimScope.None ? this.finishTrim() : this.startTrim(TrimScope.Clip); + } + })); + } + + + // for zoom slider, sets timeline waveform zoom + zoom = (zoom: number) => { + this.timeline?.setZoom(zoom); } - // hides trim controls and displays new clip + // for volume slider sets volume @action - finishTrim = () => { - if (this.mediaState === "playing") { - this.Pause(); + setVolume = (volume: number) => { + if (this._ele) { + this._volume = volume; + this._ele.volume = volume; + if (this._muted) { + this.toggleMute(); + } } - this.layoutDoc.clipStart = this._trimStart; - this.layoutDoc.clipEnd = this._trimEnd; - this._trimming = false; - this.setAnchorTime(Math.max(Math.min(this._trimEnd, this._ele!.currentTime), this._trimStart)); } + // toggles audio muted @action - setStartTrim = (newStart: number) => { - this._trimStart = newStart; + toggleMute = () => { + if (this._ele) { + this._muted = !this._muted; + this._ele.muted = this._muted; + } } - @action - setEndTrim = (newEnd: number) => { - this._trimEnd = newEnd; + + setupTimelineDrop = (r: HTMLDivElement | null) => { + if (r && this.timeline) { + this._dropDisposer?.(); + this._dropDisposer = DragManager.MakeDropTarget(r, + (e, de) => { + const [xp, yp] = this.props.ScreenToLocalTransform().transformPoint(de.x, de.y); + de.complete.docDragData && this.timeline!.internalDocDrop(e, de, de.complete.docDragData, xp); + }, + this.layoutDoc, undefined); + } } - isActiveChild = () => this._isAnyChildContentActive; - timelineWhenChildContentsActiveChanged = (isActive: boolean) => - this.props.whenChildContentsActiveChanged( - runInAction(() => (this._isAnyChildContentActive = isActive)) - ) - timelineScreenToLocal = () => - this.props - .ScreenToLocalTransform() - .translate( - -AudioBox.playheadWidth, - (-(100 - this.heightPercent) / 200) * this.props.PanelHeight() - ) - setAnchorTime = (time: number) => { - (this._ele!.currentTime = this.layoutDoc._currentTimecode = time); - } - - timelineHeight = () => - (((this.props.PanelHeight() * this.heightPercent) / 100) * - this.heightPercent) / - 100 // panelHeight * heightPercent is player height. * heightPercent is timeline height (as per css inline) - timelineWidth = () => this.props.PanelWidth() - AudioBox.playheadWidth; + + // UI for recording, initially displayed when new audio created in Dash + @computed get recordingControls() { + return <div className="audiobox-recorder"> + <div className="audiobox-dictation" onClick={this.onFile}> + <FontAwesomeIcon + size="2x" + icon="file-alt" /> + </div> + {[media_state.Recording, media_state.Playing].includes(this.mediaState) ? + <div className="recording-controls" onClick={e => e.stopPropagation()}> + <div className="record-button" onClick={this.Record}> + <FontAwesomeIcon + size="2x" + icon="stop" /> + </div> + <div className="record-button" onClick={this._paused ? this.recordPlay : this.recordPause}> + <FontAwesomeIcon + size="2x" + icon={this._paused ? "play" : "pause"} /> + </div> + <div className="record-timecode"> + {formatTime(Math.round(NumCast(this.layoutDoc._currentTimecode)))} + </div> + </div> + : + <div className="audiobox-start-record"> + <FontAwesomeIcon icon="microphone" /> + RECORD + </div>} + </div> + } + + // UI for playback, displayed for imported or recorded clips, hides timeline and collapses controls when node is shrunk vertically + @computed get playbackControls() { + return <div className="audiobox-file" style={{ + pointerEvents: this._isAnyChildContentActive || this.props.isContentActive() ? "all" : "none", + flexDirection: this.miniPlayer ? "row" : "column", + justifyContent: this.miniPlayer ? "flex-start" : "space-between" + }}> + <div className="audiobox-controls"> + <div className="controls-left"> + <div className="audiobox-button" + title={this.mediaState === media_state.Paused ? "play" : "pause"} + onPointerDown={this.mediaState === media_state.Paused ? this.Play : (e) => { e.stopPropagation(); this.Pause(); }}> + <FontAwesomeIcon icon={this.mediaState === media_state.Paused ? "play" : "pause"} size={"1x"} /> + </div> + + {!this.miniPlayer && + <div className="audiobox-button" + title={this.timeline?.IsTrimming !== TrimScope.None ? "finish" : "trim"} + onPointerDown={this.onClipPointerDown}> + <FontAwesomeIcon icon={this.timeline?.IsTrimming !== TrimScope.None ? "check" : "cut"} size={"1x"} /> + </div>} + </div> + <div className="controls-right"> + <div className="audiobox-button" + title={this._muted ? "unmute" : "mute"} + onPointerDown={(e) => { e.stopPropagation(); this.toggleMute(); }}> + <FontAwesomeIcon icon={this._muted ? "volume-mute" : "volume-up"} /> + </div> + <input type="range" step="0.1" min="0" max="1" value={this._muted ? 0 : this._volume} + className="toolbar-slider volume" + onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => { this.setVolume(Number(e.target.value)) }} + /> + </div> + </div> + + <div className="audiobox-playback" style={{ width: this.miniPlayer ? 0 : "100%" }}> + <div className="audiobox-timeline"> + {this.renderTimeline} + </div> + </div> + + {this.audio} + + <div className="audiobox-timecodes"> + <div className="timecode-current"> + {this.timeline && formatTime(Math.round(NumCast(this.layoutDoc._currentTimecode) - NumCast(this.timeline.clipStart)))} + </div> + {!this.miniPlayer && + <div className="bottom-controls-middle"> + <FontAwesomeIcon icon="search-plus" /> + <input type="range" step="0.1" min="1" max="5" value={this.timeline?._zoomFactor} + className="toolbar-slider" id="zoom-slider" + onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => { this.zoom(Number(e.target.value)); }} + /> + </div>} + + <div className="timecode-duration"> + {this.timeline && formatTime(Math.round(this.timeline.clipDuration))} + </div> + </div> + + + </div> + } + + // gets CollectionStackedTimeline @computed get renderTimeline() { return ( <CollectionStackedTimeline - ref={this._stackedTimeline} - {...this.props} + ref={action((r: any) => this._stackedTimeline = r)} + {...OmitKeys(this.props, ["CollectionFreeFormDocumentView"]).omit} fieldKey={this.annotationKey} dictationKey={this.fieldKey + "-dictation"} mediaPath={this.path} @@ -547,13 +614,10 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp focus={DocUtils.DefaultFocus} bringToFront={emptyFunction} CollectionView={undefined} - duration={this.duration} playFrom={this.playFrom} - setTime={this.setAnchorTime} + setTime={this.setPlayheadTime} playing={this.playing} - whenChildContentsActiveChanged={ - this.timelineWhenChildContentsActiveChanged - } + whenChildContentsActiveChanged={this.timelineWhenChildContentsActiveChanged} moveDocument={this.moveDocument} addDocument={this.addDocument} removeDocument={this.removeDocument} @@ -565,142 +629,34 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp playLink={this.playLink} PanelWidth={this.timelineWidth} PanelHeight={this.timelineHeight} - trimming={this._trimming} - trimStart={this._trimStart} - trimEnd={this._trimEnd} - trimDuration={this.trimDuration} - setStartTrim={this.setStartTrim} - setEndTrim={this.setEndTrim} + rawDuration={this.rawDuration} /> ); } + // returns the html audio element + @computed get audio() { + return <audio ref={this.setRef} + className={`audiobox-control${this.props.isContentActive() ? "-interactive" : ""}`} + onLoadedData={action(e => + (this._ele?.duration && this._ele?.duration !== Infinity) && + (this.dataDoc[this.fieldKey + "-duration"] = this._ele.duration) + )} + > + <source src={this.path} type="audio/mpeg" /> + Not supported. + </audio>; + } + render() { - const interactive = - SnappingManager.GetIsDragging() || this.props.isContentActive() - ? "-interactive" - : ""; - return ( - <div - className="audiobox-container" - onContextMenu={this.specificContextMenu} - onClick={ - !this.path && !this._recorder ? this.recordAudioAnnotation : undefined - } - style={{ - pointerEvents: - this.props.layerProvider?.(this.layoutDoc) === false - ? "none" - : undefined, - }} - > - {!this.path ? ( - <div className="audiobox-buttons"> - <div className="audiobox-dictation" onClick={this.onFile}> - <FontAwesomeIcon - style={{ - width: "30px" - }} - icon="file-alt" - size={this.props.PanelHeight() < 36 ? "1x" : "2x"} - /> - </div> - {this.mediaState === "recording" || this.mediaState === "paused" ? ( - <div className="recording" onClick={(e) => e.stopPropagation()}> - <div className="recording-buttons" onClick={this.recordClick}> - <FontAwesomeIcon - icon={"stop"} - size={this.props.PanelHeight() < 36 ? "1x" : "2x"} - /> - </div> - <div - className="recording-buttons" - onClick={this._paused ? this.recordPlay : this.recordPause} - > - <FontAwesomeIcon - icon={this._paused ? "play" : "pause"} - size={this.props.PanelHeight() < 36 ? "1x" : "2x"} - /> - </div> - <div className="time"> - {formatTime( - Math.round(NumCast(this.layoutDoc._currentTimecode)) - )} - </div> - </div> - ) : ( - <div - className={`audiobox-record${interactive}`} - style={{ backgroundColor: Colors.DARK_GRAY }} - > - <FontAwesomeIcon icon="microphone" /> - RECORD - </div> - )} - </div> - ) : ( - <div - className="audiobox-controls" - style={{ - pointerEvents: - this._isAnyChildContentActive || this.props.isContentActive() - ? "all" - : "none", - }} - > - <div className="audiobox-dictation" /> - <div - className="audiobox-player" - style={{ height: `${AudioBox.heightPercent}%` }} - > - <div - className="audiobox-buttons" - title={this.mediaState === "paused" ? "play" : "pause"} - onClick={this.mediaState === "paused" ? this.Play : this.Pause} - > - {" "} - <FontAwesomeIcon - icon={this.mediaState === "paused" ? "play" : "pause"} - size={"1x"} - /> - </div> - <div - className="audiobox-buttons" - title={this._trimming ? "finish" : "trim"} - onClick={this._trimming ? this.finishTrim : this.startTrim} - > - <FontAwesomeIcon - icon={this._trimming ? "check" : "cut"} - size={"1x"} - /> - </div> - <div - className="audiobox-timeline" - style={{ - top: 0, - height: `100%`, - left: AudioBox.playheadWidth, - width: `calc(100% - ${AudioBox.playheadWidth}px)`, - background: "white", - }} - > - {this.renderTimeline} - </div> - {this.audio} - <div className="audioBox-current-time"> - {this._trimming ? - formatTime(Math.round(NumCast(this.layoutDoc._currentTimecode))) - : formatTime(Math.round(NumCast(this.layoutDoc._currentTimecode) - NumCast(this._trimStart)))} - </div> - <div className="audioBox-total-time"> - {this._trimming || !this._trimEnd ? - formatTime(Math.round(NumCast(this.duration))) - : formatTime(Math.round(NumCast(this.trimDuration)))} - </div> - </div> - </div> - )} - </div> - ); + return <div + ref={this.setupTimelineDrop} + className="audiobox-container" + onContextMenu={this.specificContextMenu} + onClick={!this.path && !this._recorder ? this.recordAudioAnnotation : undefined} + style={{ pointerEvents: this.props.layerProvider?.(this.layoutDoc) === false ? "none" : undefined }} + > + {!this.path ? this.recordingControls : this.playbackControls} + </div>; } } diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx index 5919cd8f2..cbc61ffdb 100644 --- a/src/client/views/nodes/ComparisonBox.tsx +++ b/src/client/views/nodes/ComparisonBox.tsx @@ -81,8 +81,8 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatabl </div>; }; const displayDoc = (which: string) => { - const whichDoc = Cast(this.dataDoc[which], Doc, null); - //if (whichDoc?.type === DocumentType.MARKER) + var whichDoc = Cast(this.dataDoc[which], Doc, null); + // if (whichDoc?.type === DocumentType.MARKER) whichDoc = Cast(whichDoc.annotationOn, Doc, null); const targetDoc = Cast(whichDoc?.annotationOn, Doc, null) ?? whichDoc; return whichDoc ? <> <DocumentView diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 2e822bb6a..4a5fca61a 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -746,9 +746,9 @@ export class DocumentViewInternal extends DocComponent<DocumentViewInternalProps } !zorders && cm.addItem({ description: "ZOrder...", subitems: zorderItems, icon: "compass" }); - onClicks.push({ description: "Enter Portal", event: this.makeIntoPortal, icon: "window-restore" }); + !Doc.UserDoc().noviceMode && onClicks.push({ description: "Enter Portal", event: this.makeIntoPortal, icon: "window-restore" }); !Doc.UserDoc().noviceMode && onClicks.push({ description: "Toggle Detail", event: this.setToggleDetail, icon: "concierge-bell" }); - onClicks.push({ description: (this.Document.followLinkZoom ? "Don't" : "") + " zoom following link", event: () => this.Document.followLinkZoom = !this.Document.followLinkZoom, icon: this.Document.ignoreClick ? "unlock" : "lock" }); + this.props.CollectionFreeFormDocumentView && onClicks.push({ description: (this.Document.followLinkZoom ? "Don't" : "") + " zoom following link", event: () => this.Document.followLinkZoom = !this.Document.followLinkZoom, icon: this.Document.ignoreClick ? "unlock" : "lock" }); if (!this.Document.annotationOn) { const options = cm.findByDescription("Options..."); @@ -798,6 +798,7 @@ export class DocumentViewInternal extends DocComponent<DocumentViewInternalProps if (this.props.removeDocument && !Doc.IsSystem(this.rootDoc) && CurrentUserUtils.ActiveDashboard !== this.props.Document) { // need option to gray out menu items ... preferably with a '?' that explains why they're grayed out (eg., no permissions) moreItems.push({ description: "Close", event: this.deleteClicked, icon: "times" }); } + !more && moreItems.length && cm.addItem({ description: "More...", subitems: moreItems, icon: "compass" }); const help = cm.findByDescription("Help..."); const helpItems: ContextMenuProps[] = help && "subitems" in help ? help.subitems : []; diff --git a/src/client/views/nodes/EquationBox.scss b/src/client/views/nodes/EquationBox.scss index 6c9d53d10..229a1a485 100644 --- a/src/client/views/nodes/EquationBox.scss +++ b/src/client/views/nodes/EquationBox.scss @@ -1,3 +1,39 @@ +@import "../global/globalCssVariables.scss"; + .equationBox-cont { transform-origin: top left; + overflow: visible; + width: 100%; + height: 100%; +} + +.button { + position: absolute; + display: flex; + justify-content: center; + align-items: center; + top: 0; + right: 0; + width: 20px; + height: 20px; + border-radius: 5px; + background: $dark-gray; + color: white; + + svg { + width: 12px; + height: 12px; + } +} + +.ink-editor { + top: 20px; + min-width: 500px; + min-height: 300px; + background: $light-gray; + pointer-events: all; + + button { + float: right; + } }
\ No newline at end of file diff --git a/src/client/views/nodes/EquationBox.tsx b/src/client/views/nodes/EquationBox.tsx index c170f9867..c0c4fab09 100644 --- a/src/client/views/nodes/EquationBox.tsx +++ b/src/client/views/nodes/EquationBox.tsx @@ -1,5 +1,7 @@ +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import EquationEditor from 'equation-editor-react'; -import { action, reaction } from 'mobx'; +import * as iink from 'iink-js'; +import { action, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { WidthSym } from '../../../fields/Doc'; @@ -12,12 +14,16 @@ import { LightboxView } from '../LightboxView'; import './EquationBox.scss'; import { FieldView, FieldViewProps } from './FieldView'; - @observer export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(EquationBox, fieldKey); } public static SelectOnLoad: string = ""; + _ref: React.RefObject<EquationEditor> = React.createRef(); + @observable _inkOpen = false; + @observable _inkEditor: any; + @observable _inkRef: any; + componentDidMount() { if (EquationBox.SelectOnLoad === this.rootDoc[Id] && (!LightboxView.LightboxDoc || LightboxView.IsLightboxDocView(this.props.docViewPath()))) { this.props.select(false); @@ -39,7 +45,11 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { } }, { fireImmediately: true }); } - plot: any; + + componentWillUnmount() { + this._inkRef.removeEventListener('exported', this.exportInk); + } + @action keyPressed = (e: KeyboardEvent) => { const _height = Number(getComputedStyle(this._ref.current!.element.current).height.replace("px", "")); @@ -65,6 +75,7 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { } if (e.key === "Backspace" && !this.dataDoc.text) this.props.removeDocument?.(this.rootDoc); } + onChange = (str: string) => { this.dataDoc.text = str; const style = this._ref.current && getComputedStyle(this._ref.current.element.current); @@ -75,6 +86,58 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { this.layoutDoc._height = Math.max(25, _height); } } + + @action + toggleInk = (e: React.PointerEvent) => { + e.stopPropagation(); + + this._inkOpen = !this._inkOpen; + + if (!this._inkEditor) { + this._inkEditor = this._inkRef ? iink.register(this._inkRef, { + recognitionParams: { + type: 'MATH', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + }, + iink: { + math: { + mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] + }, + export: { + jiix: { + strokes: true + } + } + } + } + }) : null; + + this._inkRef.addEventListener('exported', this.exportInk) + } + } + + convertInk = (e: React.MouseEvent) => { + this._inkRef.editor.convert(); + } + + clearInk = (e: React.MouseEvent) => { + this._inkRef.editor.clear(); + this.onChange(""); + } + + exportInk = (e: any) => { + const exports = e.detail.exports; + if (exports && exports['application/x-latex']) { + this.onChange(exports['application/x-latex']); + console.log(JSON.parse(exports['application/vnd.myscript.jiix']).expressions[0].items[0]); + } + } + + render() { TraceMobx(); const scale = (this.props.scaling?.() || 1) * NumCast(this.layoutDoc._viewScale, 1); @@ -82,18 +145,34 @@ export class EquationBox extends ViewBoxBaseComponent<FieldViewProps>() { onPointerDown={e => !e.ctrlKey && e.stopPropagation()} style={{ transform: `scale(${scale})`, - width: `${100 / scale}%`, + width: `calc(${100 / scale}% + 25px)`, height: `${100 / scale}%`, pointerEvents: !this.props.isSelected() ? "none" : undefined, }} - onKeyDown={e => e.stopPropagation()} - > + onKeyDown={e => e.stopPropagation()}> + <EquationEditor ref={this._ref} value={this.dataDoc.text || "x"} spaceBehavesLikeTab={true} onChange={this.onChange} autoCommands="pi theta sqrt sum prod alpha beta gamma rho" autoOperatorNames="sin cos tan" /> + + <div className="button" + style={{ display: this.props.isContentActive() ? "flex" : "none" }} + onPointerDown={this.toggleInk}> + <FontAwesomeIcon icon="pencil-alt" /> + </div> + + <div className='ink-editor' + ref={action((r: any) => this._inkRef = r)} + id="editor" onPointerDown={(e) => e.stopPropagation()} + touch-action="none" + style={{ display: this._inkOpen && this.props.isContentActive() ? "block" : "none" }}> + <button onClick={this.convertInk}>convert</button> + <button onClick={this.clearInk}>clear</button> + </div> + </div>); } }
\ No newline at end of file diff --git a/src/client/views/nodes/LabelBox.tsx b/src/client/views/nodes/LabelBox.tsx index 4e0461650..0015f0b71 100644 --- a/src/client/views/nodes/LabelBox.tsx +++ b/src/client/views/nodes/LabelBox.tsx @@ -33,7 +33,7 @@ export class LabelBox extends ViewBoxBaseComponent<(FieldViewProps & LabelBoxPro } getTitle() { - return this.props.label ? this.props.label : this.rootDoc["title-custom"] ? StrCast(this.rootDoc.title) : + return this.rootDoc["title-custom"] ? StrCast(this.rootDoc.title) : this.props.label ? this.props.label : typeof this.rootDoc[this.fieldKey] === "string" ? StrCast(this.rootDoc[this.fieldKey]) : StrCast(this.rootDoc.title); } @@ -47,14 +47,14 @@ export class LabelBox extends ViewBoxBaseComponent<(FieldViewProps & LabelBoxPro get paramsDoc() { return Doc.AreProtosEqual(this.layoutDoc, this.dataDoc) ? this.dataDoc : this.layoutDoc; } specificContextMenu = (e: React.MouseEvent): void => { const funcs: ContextMenuProps[] = []; - funcs.push({ + !Doc.UserDoc().noviceMode && funcs.push({ description: "Clear Script Params", event: () => { const params = Cast(this.paramsDoc["onClick-paramFieldKeys"], listSpec("string"), []); params?.map(p => this.paramsDoc[p] = undefined); }, icon: "trash" }); - ContextMenu.Instance.addItem({ description: "OnClick...", noexpand: true, subitems: funcs, icon: "mouse-pointer" }); + funcs.length && ContextMenu.Instance.addItem({ description: "OnClick...", noexpand: true, subitems: funcs, icon: "mouse-pointer" }); } @undoBatch @@ -87,7 +87,7 @@ export class LabelBox extends ViewBoxBaseComponent<(FieldViewProps & LabelBoxPro style={{ boxShadow: this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BoxShadow) }}> <div className="labelBox-mainButton" style={{ backgroundColor: this.hoverColor, - fontSize: StrCast(this.layoutDoc._fontSize) || Math.min(18, this.props.PanelHeight() / 2), + fontSize: StrCast(this.layoutDoc._fontSize), fontFamily: StrCast(this.layoutDoc._fontFamily) || "inherit", letterSpacing: StrCast(this.layoutDoc.letterSpacing), textTransform: StrCast(this.layoutDoc.textTransform) as any, diff --git a/src/client/views/nodes/MapBox/MapBox.scss b/src/client/views/nodes/MapBox/MapBox.scss index 854da5ed2..fb15520f6 100644 --- a/src/client/views/nodes/MapBox/MapBox.scss +++ b/src/client/views/nodes/MapBox/MapBox.scss @@ -35,7 +35,7 @@ .mapBox-wrapper { width: 100%; - .searchbox { + .mapBox-input { box-sizing: border-box; border: 1px solid transparent; width: 240px; diff --git a/src/client/views/nodes/MapBox/MapBox.tsx b/src/client/views/nodes/MapBox/MapBox.tsx index 9bf7c2e8c..52b0035bb 100644 --- a/src/client/views/nodes/MapBox/MapBox.tsx +++ b/src/client/views/nodes/MapBox/MapBox.tsx @@ -594,7 +594,7 @@ export class MapBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps <Autocomplete onLoad={this.setSearchBox} onPlaceChanged={this.handlePlaceChanged}> - <input ref={this.inputRef} className="searchbox" type="text" placeholder="Search anywhere:" /> + <input className="mapBox-input" ref={this.inputRef} type="text" placeholder="Enter location" /> </Autocomplete> {this.renderMarkers()} diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index 9807cee7c..fa23dfbe8 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -10,7 +10,7 @@ import { TraceMobx } from '../../../fields/util'; import { emptyFunction, returnOne, setupMoveUpEvents, Utils } from '../../../Utils'; import { Docs } from '../../documents/Documents'; import { KeyCodes } from '../../util/KeyCodes'; -import { undoBatch } from '../../util/UndoManager'; +import { undoBatch, UndoManager } from '../../util/UndoManager'; import { ContextMenu } from '../ContextMenu'; import { ContextMenuProps } from '../ContextMenuItem'; import { ViewBoxAnnotatableComponent, ViewBoxAnnotatableProps } from "../DocComponent"; @@ -137,18 +137,21 @@ export class PDFBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps return this.addDocument(doc, sidebarKey); } sidebarBtnDown = (e: React.PointerEvent, onButton: boolean) => { // onButton determines whether the width of the pdf box changes, or just the ratio of the sidebar to the pdf + const batch = UndoManager.StartBatch("sidebar"); setupMoveUpEvents(this, e, (e, down, delta) => { const localDelta = this.props.ScreenToLocalTransform().scale(this.props.scaling?.() || 1).transformDirection(delta[0], delta[1]); const nativeWidth = NumCast(this.layoutDoc[this.fieldKey + "-nativeWidth"]); + const nativeHeight = NumCast(this.layoutDoc[this.fieldKey + "-nativeHeight"]); const curNativeWidth = NumCast(this.layoutDoc.nativeWidth, nativeWidth); const ratio = (curNativeWidth + (onButton ? 1 : -1) * localDelta[0] / (this.props.scaling?.() || 1)) / nativeWidth; if (ratio >= 1) { this.layoutDoc.nativeWidth = nativeWidth * ratio; + this.layoutDoc.nativeHeight = nativeHeight * (1 + ratio); onButton && (this.layoutDoc._width = this.layoutDoc[WidthSym]() + localDelta[0]); this.layoutDoc._showSidebar = nativeWidth !== this.layoutDoc._nativeWidth; } return false; - }, emptyFunction, () => this.toggleSidebar()); + }, () => batch.end(), () => this.toggleSidebar()); } @observable _previewNativeWidth: Opt<number> = undefined; @observable _previewWidth: Opt<number> = undefined; diff --git a/src/client/views/nodes/VideoBox.scss b/src/client/views/nodes/VideoBox.scss index f0d7bd2f3..3cf10a033 100644 --- a/src/client/views/nodes/VideoBox.scss +++ b/src/client/views/nodes/VideoBox.scss @@ -1,4 +1,6 @@ -.mini-viewer{ +@import "../global/globalCssVariables.scss"; + +.mini-viewer { cursor: grab; position: absolute; right: 10; @@ -14,22 +16,21 @@ height: 100%; position: relative; .videoBox-viewer { - display:flex; + display: flex; flex-direction: column; height: 100%; border-radius: inherit; - opacity: 0.99; // hack! overcomes some kind of Chrome weirdness where buttons (e.g., snapshot) disappear at some point as the video is resized larger + opacity: 0.99; // hack! overcomes some kind of Chrome weirdness where buttons (e.g., snapshot) disappear at some point as the video is resized larger + background: $dark-gray; } .inkingCanvas-paths-markers { - opacity : 0.4; // we shouldn't have to do this, but since chrome crawls to a halt with z-index unset in videoBox-content, this is a workaround - } - .collectionStackedTimeline { - background: beige; + opacity: 0.4; // we shouldn't have to do this, but since chrome crawls to a halt with z-index unset in videoBox-content, this is a workaround } + .videoBox-stackPanel { z-index: -1; width: 100%; - position: relative; + position: relative; } .videoBox-annotationLayer { @@ -43,8 +44,11 @@ } } -.videoBox-content-YouTube, .videoBox-content-YouTube-fullScreen, -.videoBox-content, .videoBox-content-interactive, .videoBox-cont-fullScreen { +.videoBox-content-YouTube, +.videoBox-content-YouTube-fullScreen, +.videoBox-content, +.videoBox-content-interactive, +.videoBox-cont-fullScreen { width: 100%; z-index: -1; // 0; // logically this should be 0 (or unset) which would give us transparent brush strokes over videos. However, this makes Chrome crawl to a halt position: absolute; @@ -62,7 +66,8 @@ left: 0px; } -.videoBox-content-YouTube, .videoBox-content-YouTube-fullScreen { +.videoBox-content-YouTube, +.videoBox-content-YouTube-fullScreen { height: 100%; } @@ -73,25 +78,138 @@ .videoBox-ui { position: absolute; flex-direction: row; - right: 5px; - top: 5px; - display: none; - background-color: rgba(0, 0, 0, 0.6); + align-items: center; + justify-content: center; + display: flex; + visibility: none; + opacity: 0; + background-color: $dark-gray; + color: white; + border-radius: 100px; + top: calc(100% - 20px); + left: 50%; + transform: translate(-50%, -100%); + + transition: top 0.5s, width 0.5s, opacity 0.2s, visibility 0s; + height: 120px; + padding: 0 20px; + + .timecode-controls { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + margin: 0 5px; + flex-grow: 2; + font-size: 32px; + + .timecode { + margin: 0 5px; + } + + .timeline-slider { + margin: 0 20px 0 20px; + flex-grow: 2; + } + } + + .toolbar-slider.volume, .toolbar-slider.zoom { + width: 100px; + } + + .videobox-button { + margin: 5px; + cursor: pointer; + width: 70px; + height: 70px; + border-radius: 50%; + background: $dark-gray; + display: flex; + align-items: center; + justify-content: center; + + &:hover { + background: $black; + } + + svg { + width: 40px; + height: 40px; + } + } } -.videoBox-time, .videoBox-snapshot, .videoBox-timelineButton, .videoBox-play, .videoBox-full { - color : white; + +.videoBox-time, +.videoBox-snapshot, +.videoBox-timelineButton, +.videoBox-play, +.videoBox-full { + color: white; position: relative; transform-origin: left top; - pointer-events:all; + pointer-events: all; padding-right: 5px; cursor: pointer; &:hover { - background-color: gray; + background-color: $medium-gray; } } .videoBox:hover { .videoBox-ui { - display: flex; + visibility: visible; + opacity: 1; + z-index: 10000; } +} + +.videoBox-content-fullScreen, .videoBox-content-fullScreen-interactive { + display: flex; + justify-content: center; + align-items: center; + + &:hover { + .videoBox-ui { + opacity: 0; + } + } + + .videoBox-ui:hover { + opacity: 1; + } +} + +video::-webkit-media-controls { + display: none !important; +} + +input[type="range"] { + -webkit-appearance: none; + background: none; + margin: 10px; +} + +input[type="range"]:focus { + outline: none; +} + +input[type="range"]::-webkit-slider-runnable-track { + width: 100%; + height: 20px; + cursor: pointer; + box-shadow: 0; + background: $light-gray; + border-radius: 20px; +} + +input[type="range"]::-webkit-slider-thumb { + box-shadow: 0; + border: 0; + height: 26px; + width: 26px; + border-radius: 20px; + background: $medium-blue; + cursor: pointer; + -webkit-appearance: none; + margin-top: -3px; }
\ No newline at end of file diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index a5b8967c4..913123cda 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -1,6 +1,5 @@ import React = require("react"); import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { Tooltip } from "@material-ui/core"; import { action, computed, IReactionDisposer, observable, ObservableMap, reaction, runInAction, untracked } from "mobx"; import { observer } from "mobx-react"; import { basename } from "path"; @@ -17,8 +16,9 @@ import { CurrentUserUtils } from "../../util/CurrentUserUtils"; import { DocumentManager } from "../../util/DocumentManager"; import { SelectionManager } from "../../util/SelectionManager"; import { SnappingManager } from "../../util/SnappingManager"; +import { undoBatch } from "../../util/UndoManager"; import { CollectionFreeFormView } from "../collections/collectionFreeForm/CollectionFreeFormView"; -import { CollectionStackedTimeline } from "../collections/CollectionStackedTimeline"; +import { CollectionStackedTimeline, TrimScope } from "../collections/CollectionStackedTimeline"; import { ContextMenu } from "../ContextMenu"; import { ContextMenuProps } from "../ContextMenuItem"; import { ViewBoxAnnotatableComponent, ViewBoxAnnotatableProps } from "../DocComponent"; @@ -27,76 +27,142 @@ import { MarqueeAnnotator } from "../MarqueeAnnotator"; import { AnchorMenu } from "../pdf/AnchorMenu"; import { StyleProp } from "../StyleProvider"; import { FieldView, FieldViewProps } from './FieldView'; -import { LinkDocPreview } from "./LinkDocPreview"; import "./VideoBox.scss"; +const path = require('path'); + + +/** + * VideoBox + * Main component: VideoBox.tsx + * Supporting Components: CollectionStackedTimeline + * + * VideoBox is a node that supports the playback of video files in Dash. + * When a video file or YouTube video is importeed into Dash, it is immediately rendered as a VideoBox document. + * CollectionStackedTimline handles AudioBox and VideoBox shared behavior, but VideoBox handles playing, pausing, etc because it contains <video> element + * User can trim video: nondestructive, just sets new bounds for playback and rendering timeline + * Like images, users can zoom and pan and it has an overlay layer allowing for annotations on top of the video at different times + */ @observer export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps & FieldViewProps>() { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(VideoBox, fieldKey); } + static async convertDataUri(imageUri: string, returnedFilename: string) { + try { + const posting = Utils.prepend("/uploadURI"); + const returnedUri = await rp.post(posting, { + body: { + uri: imageUri, + name: returnedFilename + }, + json: true, + }); + return returnedUri; + + } catch (e) { + console.log("VideoBox :" + e); + } + } + static _youtubeIframeCounter: number = 0; - static Instance: VideoBox; - static heightPercent = 60; // height of timeline in percent of height of videoBox. + static heightPercent = 80; // height of video relative to videoBox when timeline is open private _disposers: { [name: string]: IReactionDisposer } = {}; private _youtubePlayer: YT.Player | undefined = undefined; - private _videoRef: HTMLVideoElement | null = null; + private _videoRef: HTMLVideoElement | null = null; // <video> ref + private _contentRef: HTMLDivElement | null = null; // ref to div that wraps video and controls for full screen private _youtubeIframeId: number = -1; private _youtubeContentCreated = false; - private _stackedTimeline = React.createRef<CollectionStackedTimeline>(); - private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); + private _audioPlayer: HTMLAudioElement | null = null; + private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); // outermost div private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); - private _playRegionTimer: any = null; - private _playRegionDuration = 0; - @observable static _nativeControls: boolean; - @observable _marqueeing: number[] | undefined; + private _playRegionTimer: any = null; // timeout for playback + @observable _stackedTimeline: any; // CollectionStackedTimeline ref + @observable static _nativeControls: boolean; // default html controls + @observable _marqueeing: number[] | undefined; // coords for marquee selection @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>(); @observable _screenCapture = false; - @observable _clicking = false; + @observable _clicking = false; // used for transition between showing/hiding timeline @observable _forceCreateYouTubeIFrame = false; @observable _playTimer?: NodeJS.Timeout = undefined; @observable _fullScreen = false; @observable _playing = false; + @observable _finished: boolean = false; // has playback reached end of clip + @observable _volume: number = 1; + @observable _muted: boolean = false; + @computed get links() { return DocListCast(this.dataDoc.links); } - @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } - @computed get duration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); } + @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } // current percent of video relative to VideoBox height + // @computed get rawDuration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); } + @observable rawDuration: number = 0; - private get transition() { return this._clicking ? "left 0.5s, width 0.5s, height 0.5s" : ""; } - public get player(): HTMLVideoElement | null { return this._videoRef; } - constructor(props: Readonly<ViewBoxAnnotatableProps & FieldViewProps>) { - super(props); - VideoBox.Instance = this; + @computed get youtubeVideoId() { + const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); + return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : ""; } - getAnchor = () => { - const timecode = Cast(this.layoutDoc._currentTimecode, "number", null); - const marquee = AnchorMenu.Instance.GetAnchor?.(); - return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined, undefined, marquee) || this.rootDoc; + + // returns the path of the audio file + @computed get audiopath() { + const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null); + const vfield = Cast(this.dataDoc[this.fieldKey], VideoField, null); + return field?.url.href ?? vfield?.url.href ?? ""; } - videoLoad = () => { - const aspect = this.player!.videoWidth / this.player!.videoHeight; - Doc.SetNativeWidth(this.dataDoc, this.player!.videoWidth); - Doc.SetNativeHeight(this.dataDoc, this.player!.videoHeight); - this.layoutDoc._height = NumCast(this.layoutDoc._width) / aspect; - if (Number.isFinite(this.player!.duration)) { - this.dataDoc[this.fieldKey + "-duration"] = this.player!.duration; + + @computed private get timeline() { return this._stackedTimeline; } + private get transition() { return this._clicking ? "left 0.5s, width 0.5s, height 0.5s" : ""; } // css transition for hiding/showing timeline + public get player(): HTMLVideoElement | null { return this._videoRef; } + + + componentDidMount() { + this.props.setContentView?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link. + if (this.youtubeVideoId) { + const youtubeaspect = 400 / 315; + const nativeWidth = Doc.NativeWidth(this.layoutDoc); + const nativeHeight = Doc.NativeHeight(this.layoutDoc); + if (!nativeWidth || !nativeHeight) { + if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 600); + Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 600) / youtubeaspect); + this.layoutDoc._height = NumCast(this.layoutDoc._width) / youtubeaspect; + } } + this.player && this.setPlayheadTime(0); + } + + componentWillUnmount() { + this.removeCurrentlyPlaying(); + this.Pause(); + Object.keys(this._disposers).forEach(d => this._disposers[d]?.()); } + + // plays video @action public Play = (update: boolean = true) => { this._playing = true; - try { - this._audioPlayer && this.player && (this._audioPlayer.currentTime = this.player?.currentTime); - update && this.player?.play(); - update && this._audioPlayer?.play(); - update && this._youtubePlayer?.playVideo(); - this._youtubePlayer && !this._playTimer && (this._playTimer = setInterval(this.updateTimecode, 5)); - } catch (e) { - console.log("Video Play Exception:", e); + const eleTime = this.player?.currentTime || 0; + if (this.timeline) { + let start = eleTime >= this.timeline.trimEnd || eleTime <= this.timeline.trimStart ? this.timeline.trimStart : eleTime; + + if (this._finished) { + // restarts video if reached end on previous play + this._finished = false; + start = this.timeline.trimStart; + } + + try { + this._audioPlayer && this.player && (this._audioPlayer.currentTime = this.player?.currentTime); + update && this.player && this.playFrom(start, undefined, true); + update && this._audioPlayer?.play(); + update && this._youtubePlayer?.playVideo(); + this._youtubePlayer && !this._playTimer && (this._playTimer = setInterval(this.updateTimecode, 5)); + } catch (e) { + console.log("Video Play Exception:", e); + } } this.updateTimecode(); } + // goes to time @action public Seek(time: number) { try { this._youtubePlayer?.seekTo(Math.round(time), true); @@ -107,8 +173,10 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this._audioPlayer && (this._audioPlayer.currentTime = time); } + // pauses video @action public Pause = (update: boolean = true) => { this._playing = false; + this.removeCurrentlyPlaying(); try { update && this.player?.pause(); update && this._audioPlayer?.pause(); @@ -120,12 +188,21 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } this._youtubePlayer && SelectionManager.DeselectAll(); // if we don't deselect the player, then we get an annoying YouTube spinner I guess telling us we're paused. this._playTimer = undefined; - this.props.renderDepth !== -1 && this.updateTimecode(); + this.updateTimecode(); + if (!this._finished) clearTimeout(this._playRegionTimer); // if paused in the middle of playback, prevents restart on next play } + // toggles video full screen @action public FullScreen = () => { - this._fullScreen = true; - this.player && this.player.requestFullscreen(); + if (document.fullscreenElement == this._contentRef) { + this._fullScreen = false; + this.player && this._contentRef && document.exitFullscreen(); + } + else { + this._fullScreen = true; + this.player && this._contentRef && this._contentRef.requestFullscreen(); + + } try { this._youtubePlayer && this.props.addDocTab(this.rootDoc, "add"); } catch (e) { @@ -133,6 +210,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // creates and links snapshot photo of current video frame @action public Snapshot(downX?: number, downY?: number) { const width = NumCast(this.layoutDoc._width); const canvas = document.createElement('canvas'); @@ -176,7 +255,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } - private createRealSummaryLink = (imagePath: string, downX?: number, downY?: number) => { + // creates link for snapshot + createRealSummaryLink = (imagePath: string, downX?: number, downY?: number) => { const url = !imagePath.startsWith("/") ? Utils.CorsProxy(imagePath) : imagePath; const width = NumCast(this.layoutDoc._width) || 1; const height = NumCast(this.layoutDoc._height); @@ -194,6 +274,27 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp (downX !== undefined && downY !== undefined) && DocumentManager.Instance.getFirstDocumentView(imageSummary)?.startDragging(downX, downY, "move", true)); } + + getAnchor = () => { + const timecode = Cast(this.layoutDoc._currentTimecode, "number", null); + const marquee = AnchorMenu.Instance.GetAnchor?.(); + return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined, undefined, marquee) || this.rootDoc; + } + + + // sets video info on load + videoLoad = action(() => { + const aspect = this.player!.videoWidth / this.player!.videoHeight; + Doc.SetNativeWidth(this.dataDoc, this.player!.videoWidth); + Doc.SetNativeHeight(this.dataDoc, this.player!.videoHeight); + this.layoutDoc._height = NumCast(this.layoutDoc._width) / aspect; + if (Number.isFinite(this.player!.duration)) { + this.rawDuration = this.player!.duration; + } + }) + + + // updates video time @action updateTimecode = () => { this.player && (this.layoutDoc._currentTimecode = this.player.currentTime); @@ -204,72 +305,32 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } - componentDidMount() { - this.props.setContentView?.(this); // this tells the DocumentView that this AudioBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. - this._disposers.triggerVideo = reaction( - () => !LinkDocPreview.LinkInfo && this.props.renderDepth !== -1 ? NumCast(this.Document._triggerVideo, null) : undefined, - time => time !== undefined && setTimeout(() => { - this.player && this.Play(); - setTimeout(() => this.Document._triggerVideo = undefined, 10); - }, this.player ? 0 : 250), // wait for mainCont and try again to play - { fireImmediately: true } - ); - this._disposers.triggerStop = reaction( - () => this.props.renderDepth !== -1 && !LinkDocPreview.LinkInfo ? NumCast(this.Document._triggerVideoStop, null) : undefined, - stop => stop !== undefined && setTimeout(() => { - this.player && this.Pause(); - setTimeout(() => this.Document._triggerVideoStop = undefined, 10); - }, this.player ? 0 : 250), // wait for mainCont and try again to play - { fireImmediately: true } - ); - if (this.youtubeVideoId) { - const youtubeaspect = 400 / 315; - const nativeWidth = Doc.NativeWidth(this.layoutDoc); - const nativeHeight = Doc.NativeHeight(this.layoutDoc); - if (!nativeWidth || !nativeHeight) { - if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 600); - Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 600) / youtubeaspect); - this.layoutDoc._height = NumCast(this.layoutDoc._width) / youtubeaspect; - } - } - } - - componentWillUnmount() { - this.Pause(); - Object.keys(this._disposers).forEach(d => this._disposers[d]?.()); - } + // sets video element ref @action setVideoRef = (vref: HTMLVideoElement | null) => { this._videoRef = vref; if (vref) { this._videoRef!.ontimeupdate = this.updateTimecode; // @ts-ignore - vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen); + // vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen); this._disposers.reactionDisposer?.(); this._disposers.reactionDisposer = reaction(() => NumCast(this.layoutDoc._currentTimecode), time => !this._playing && (vref.currentTime = time), { fireImmediately: true }); } } - public static async convertDataUri(imageUri: string, returnedFilename: string, nosuffix = false) { - try { - const posting = Utils.prepend("/uploadURI"); - const returnedUri = await rp.post(posting, { - body: { - uri: imageUri, - name: returnedFilename, - nosuffix - }, - json: true, - }); - return returnedUri; - - } catch (e) { - console.log("VideoBox :" + e); + // set ref for div that wraps video and controls for fullscreen + @action + setContentRef = (cref: HTMLDivElement | null) => { + this._contentRef = cref; + if (cref) { + cref.onfullscreenchange = action((e) => this._fullScreen = (document.fullscreenElement == cref)); } } + + // context menu specificContextMenu = (e: React.MouseEvent): void => { const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); if (field) { @@ -283,31 +344,32 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this._videoRef!.srcObject = !this._screenCapture ? undefined : await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); }), icon: "expand-arrows-alt" }); + subitems.push({ description: (this.layoutDoc.dontAutoFollowLinks ? "" : "Don't") + " follow links when encountered", event: () => this.layoutDoc.dontAutoFollowLinks = !this.layoutDoc.dontAutoFollowLinks, icon: "expand-arrows-alt" }); subitems.push({ description: (this.layoutDoc.dontAutoPlayFollowedLinks ? "" : "Don't") + " play when link is selected", event: () => this.layoutDoc.dontAutoPlayFollowedLinks = !this.layoutDoc.dontAutoPlayFollowedLinks, icon: "expand-arrows-alt" }); subitems.push({ description: (this.layoutDoc.autoPlayAnchors ? "Don't auto play" : "Auto play") + " anchors onClick", event: () => this.layoutDoc.autoPlayAnchors = !this.layoutDoc.autoPlayAnchors, icon: "expand-arrows-alt" }); - subitems.push({ description: "Toggle Native Controls", event: action(() => VideoBox._nativeControls = !VideoBox._nativeControls), icon: "expand-arrows-alt" }); + // subitems.push({ description: "Toggle Native Controls", event: action(() => VideoBox._nativeControls = !VideoBox._nativeControls), icon: "expand-arrows-alt" }); + // subitems.push({ description: "Start Trim All", event: () => this.startTrim(TrimScope.All), icon: "expand-arrows-alt" }); + // subitems.push({ description: "Start Trim Clip", event: () => this.startTrim(TrimScope.Clip), icon: "expand-arrows-alt" }); + // subitems.push({ description: "Stop Trim", event: () => this.finishTrim(), icon: "expand-arrows-alt" }); subitems.push({ description: "Copy path", event: () => { Utils.CopyText(url); }, icon: "expand-arrows-alt" }); ContextMenu.Instance.addItem({ description: "Options...", subitems: subitems, icon: "video" }); } } - // returns the path of the audio file - @computed get audiopath() { - const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null); - const vfield = Cast(this.dataDoc[this.fieldKey], VideoField, null); - return field?.url.href ?? vfield?.url.href ?? ""; - } + // ref for updating time - _audioPlayer: HTMLAudioElement | null = null; setAudioRef = (e: HTMLAudioElement | null) => this._audioPlayer = e; + + // renders the video and audio @computed get content() { const field = Cast(this.dataDoc[this.fieldKey], VideoField); const interactive = CurrentUserUtils.SelectedTool !== InkTool.None || !this.props.isSelected() ? "" : "-interactive"; const classname = "videoBox-content" + (this._fullScreen ? "-fullScreen" : "") + interactive; return !field ? <div key="loading">Loading</div> : <div className="videoBox-contentContainer" key="container" style={{ mixBlendMode: "multiply" }}> - <div className={classname}> - <video key="video" autoPlay={this._screenCapture} ref={this.setVideoRef} + <div className={classname} ref={this.setContentRef} onPointerDown={(e) => this._fullScreen && e.stopPropagation()}> + {this.uIButtons} + <video key="video" autoPlay={this._screenCapture} ref={this.setVideoRef} style={this._fullScreen ? this.fullScreenSize() : {}} onCanPlay={this.videoLoad} controls={VideoBox._nativeControls} onPlay={() => this.Play()} @@ -326,10 +388,6 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp </div>; } - @computed get youtubeVideoId() { - const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); - return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : ""; - } @action youtubeIframeLoaded = (e: any) => { if (!this._youtubeContentCreated) { @@ -340,7 +398,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this.loadYouTube(e.target); } - private loadYouTube = (iframe: any) => { + + loadYouTube = (iframe: any) => { let started = true; const onYoutubePlayerStateChange = (event: any) => runInAction(() => { if (started && event.data === YT.PlayerState.PLAYING) { @@ -372,48 +431,19 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp }); } } - private get uIButtons() { - const curTime = NumCast(this.layoutDoc._currentTimecode); - const nonNativeControls = [ - <Tooltip title={<div className="dash-tooltip">{"playback"}</div>} key="play" placement="bottom"> - <div className="videoBox-play" onPointerDown={this.onPlayDown} > - <FontAwesomeIcon icon={this._playing ? "pause" : "play"} size="lg" /> - </div> - </Tooltip>, - <Tooltip title={<div className="dash-tooltip">{"timecode"}</div>} key="time" placement="bottom"> - <div className="videoBox-time" onPointerDown={this.onResetDown} > - <span>{formatTime(curTime)}</span> - <span style={{ fontSize: 8 }}>{" " + Math.floor((curTime - Math.trunc(curTime)) * 100).toString().padStart(2, "0")}</span> - </div> - </Tooltip>, - <Tooltip title={<div className="dash-tooltip">{"view full screen"}</div>} key="full" placement="bottom"> - <div className="videoBox-full" onPointerDown={this.FullScreen}> - <FontAwesomeIcon icon="expand" size="lg" /> - </div> - </Tooltip>]; - return <div className="videoBox-ui"> - {[...(VideoBox._nativeControls ? [] : nonNativeControls), - <Tooltip title={<div className="dash-tooltip">{"snapshot current frame"}</div>} key="snap" placement="bottom"> - <div className="videoBox-snapshot" onPointerDown={this.onSnapshotDown} > - <FontAwesomeIcon icon="camera" size="lg" /> - </div> - </Tooltip>, - <Tooltip title={<div className="dash-tooltip">{"show annotation timeline"}</div>} key="timeline" placement="bottom"> - <div className="videoBox-timelineButton" onPointerDown={this.onTimelineHdlDown}> - <FontAwesomeIcon icon="eye" size="lg" /> - </div> - </Tooltip>,]} - </div>; - } + + // for play button onPlayDown = () => this._playing ? this.Pause() : this.Play(); + // for fullscreen button onFullDown = (e: React.PointerEvent) => { this.FullScreen(); e.stopPropagation(); e.preventDefault(); } + // for snapshot button onSnapshotDown = (e: React.PointerEvent) => { setupMoveUpEvents(this, e, (e) => { this.Snapshot(e.clientX, e.clientY); @@ -421,14 +451,18 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp }, emptyFunction, () => this.Snapshot()); } - onTimelineHdlDown = action((e: React.PointerEvent) => { + // for show/hide timeline button, transitions between show/hide + @action + onTimelineHdlDown = (e: React.PointerEvent) => { this._clicking = true; setupMoveUpEvents(this, e, - action((e: PointerEvent) => { + action(encodeURIComponent => { this._clicking = false; if (this.props.isContentActive()) { - const local = this.props.ScreenToLocalTransform().scale(this.props.scaling?.() || 1).transformPoint(e.clientX, e.clientY); - this.layoutDoc._timelineHeightPercent = Math.max(0, Math.min(100, local[1] / this.props.PanelHeight() * 100)); + // const local = this.props.ScreenToLocalTransform().scale(this.props.scaling?.() || 1).transformPoint(e.clientX, e.clientY); + // this.layoutDoc._timelineHeightPercent = Math.max(0, Math.min(100, local[1] / this.props.PanelHeight() * 100)); + + this.layoutDoc._timelineHeightPercent = 80; } return false; }), emptyFunction, @@ -436,19 +470,30 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this.layoutDoc._timelineHeightPercent = this.heightPercent !== 100 ? 100 : VideoBox.heightPercent; setTimeout(action(() => this._clicking = false), 500); }, this.props.isContentActive(), this.props.isContentActive()); - }); + } - onResetDown = (e: React.PointerEvent) => { - setupMoveUpEvents(this, e, - (e: PointerEvent) => { - this.Seek(Math.max(0, NumCast(this.layoutDoc._currentTimecode) + Math.sign(e.movementX) * 0.0333)); - e.stopImmediatePropagation(); - return false; - }, - emptyFunction, - (e: PointerEvent) => this.layoutDoc._currentTimecode = 0); + + // removes video from currently playing display + @action + removeCurrentlyPlaying = () => { + if (CollectionStackedTimeline.CurrentlyPlaying) { + const index = CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc.doc as Doc); + index !== -1 && CollectionStackedTimeline.CurrentlyPlaying.splice(index, 1); + } } + // adds video to currently playing display + @action + addCurrentlyPlaying = () => { + if (!CollectionStackedTimeline.CurrentlyPlaying) { + CollectionStackedTimeline.CurrentlyPlaying = []; + } + if (CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc.doc as Doc) == -1) { + CollectionStackedTimeline.CurrentlyPlaying.push(this.layoutDoc.doc as Doc); + } + } + + @computed get youtubeContent() { this._youtubeIframeId = VideoBox._youtubeIframeCounter++; this._youtubeContentCreated = this._forceCreateYouTubeIFrame ? true : true; @@ -460,6 +505,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp src={`https://www.youtube.com/embed/${this.youtubeVideoId}?enablejsapi=1&rel=0&showinfo=1&autoplay=0&mute=1&start=${start}&modestbranding=1&controls=${VideoBox._nativeControls ? 1 : 0}`} />; } + + // for annotating, adds doc with time info @action.bound addDocWithTimecode(doc: Doc | Doc[]): boolean { const docs = doc instanceof Doc ? [doc] : doc; @@ -468,52 +515,245 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp return this.addDocument(doc); } - // play back the video from time + + // play back the audio from seekTimeInSeconds, fullPlay tells whether clip is being played to end vs link range @action - playFrom = (seekTimeInSeconds: number, endTime: number = this.duration) => { + playFrom = (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => { clearTimeout(this._playRegionTimer); - this._playRegionDuration = endTime - seekTimeInSeconds; if (Number.isNaN(this.player?.duration)) { setTimeout(() => this.playFrom(seekTimeInSeconds, endTime), 500); - } else if (this.player) { - if (seekTimeInSeconds < 0) { - if (seekTimeInSeconds > -1) { - setTimeout(() => this.playFrom(0), -seekTimeInSeconds * 1000); - } else { - this.Pause(); - } - } else if (seekTimeInSeconds <= this.player.duration) { - this.player.currentTime = seekTimeInSeconds; + } + else if (this.player) { + // trimBounds override requested playback bounds + const end = Math.min(this.timeline?.trimEnd ?? this.rawDuration, endTime ?? this.timeline?.trimEnd ?? this.rawDuration); + const start = Math.max(this.timeline?.trimStart ?? 0, seekTimeInSeconds); + const playRegionDuration = end - start; + // checks if times are within clip range + if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) { + this.player.currentTime = start; this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds); this.player.play(); this._audioPlayer?.play(); - runInAction(() => this._playing = true); - if (endTime !== this.duration) { - this._playRegionTimer = setTimeout(() => this.Pause(), (this._playRegionDuration) * 1000); // use setTimeout to play a specific duration - } + this._playing = true; + this.addCurrentlyPlaying(); + this._playRegionTimer = setTimeout( + () => { + // need to keep track of if end of clip is reached so on next play, clip restarts + if (fullPlay) this._finished = true; + // removes from currently playing if playback has reached end of range marker + else this.removeCurrentlyPlaying(); + this.Pause(); + }, playRegionDuration * 1000); } else { this.Pause(); } } } + + // ends trim, hides trim controls and displays new clip + @undoBatch + finishTrim = action(() => { + this.Pause(); + this.setPlayheadTime(Math.max(Math.min(this.timeline?.trimEnd || 0, this.player!.currentTime), this.timeline?.trimStart || 0)); + this.timeline?.StopTrimming(); + }); + + // displays trim controls to start trimming clip + startTrim = (scope: TrimScope) => { + this.Pause(); + this.timeline?.StartTrimming(scope); + } + + // for trim button, double click displays full clip, single displays curr trim bounds + onClipPointerDown = (e: React.PointerEvent) => { + // if timeline isn't shown, show first then trim + this.heightPercent >= 100 && this.onTimelineHdlDown(e); + this.timeline && setupMoveUpEvents(this, e, returnFalse, returnFalse, action((e: PointerEvent, doubleTap?: boolean) => { + if (doubleTap) { + this.startTrim(TrimScope.All); + } else if (this.timeline) { + this.Pause(); + this.timeline.IsTrimming !== TrimScope.None ? this.finishTrim() : this.startTrim(TrimScope.Clip); + } + })); + } + + + // for volume slider sets volume + @action + setVolume = (volume: number) => { + if (this.player) { + this._volume = volume; + this.player.volume = volume; + if (this._muted) { + this.toggleMute(); + } + } + } + + // toggles video mute + @action + toggleMute = () => { + if (this.player) { + this._muted = !this._muted; + this.player.muted = this._muted; + } + } + + + // stretches vertically or horizontally depending on video orientation so video fits full screen + fullScreenSize() { + if (this._videoRef && this._videoRef.videoHeight / this._videoRef.videoWidth > 1) { + return { height: "100%" } + } + else { + return { width: "100%" } + } + } + + + // for zoom slider, sets timeline waveform zoom + zoom = (zoom: number) => { + this.timeline?.setZoom(zoom); + } + + + // plays link playLink = (doc: Doc) => { - const startTime = Math.max(0, (this._stackedTimeline.current?.anchorStart(doc) || 0)); - const endTime = this._stackedTimeline.current?.anchorEnd(doc); + const startTime = Math.max(0, (this._stackedTimeline?.anchorStart(doc) || 0)); + const endTime = this.timeline?.anchorEnd(doc); if (startTime !== undefined) { if (!this.layoutDoc.dontAutoPlayFollowedLinks) endTime ? this.playFrom(startTime, endTime) : this.playFrom(startTime); else this.Seek(startTime); } } - playing = () => this._playing; + + // starts marquee selection + marqueeDown = (e: React.PointerEvent) => { + if (!e.altKey && e.button === 0 && this.layoutDoc._viewScale === 1 && this.props.isContentActive(true) && ![InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) { + setupMoveUpEvents(this, e, action(e => { + MarqueeAnnotator.clearAnnotations(this._savedAnnotations); + this._marqueeing = [e.clientX, e.clientY]; + return true; + }), returnFalse, () => MarqueeAnnotator.clearAnnotations(this._savedAnnotations), false); + } + } + + // ends marquee selection + @action + finishMarquee = () => { + this._marqueeing = undefined; + this.props.select(true); + } + timelineWhenChildContentsActiveChanged = action((isActive: boolean) => this.props.whenChildContentsActiveChanged(this._isAnyChildContentActive = isActive)); + timelineScreenToLocal = () => this.props.ScreenToLocalTransform().scale(this.scaling()).translate(0, -this.heightPercent / 100 * this.props.PanelHeight()); - setAnchorTime = (time: number) => this.player!.currentTime = this.layoutDoc._currentTimecode = time; + + setPlayheadTime = (time: number) => this.player!.currentTime = this.layoutDoc._currentTimecode = time; + timelineHeight = () => this.props.PanelHeight() * (100 - this.heightPercent) / 100; + + playing = () => this._playing; + + contentFunc = () => [this.youtubeVideoId ? this.youtubeContent : this.content]; + + scaling = () => this.props.scaling?.() || 1; + + panelWidth = () => this.props.PanelWidth() * this.heightPercent / 100; + panelHeight = () => this.layoutDoc._fitWidth ? this.panelWidth() / (Doc.NativeAspect(this.rootDoc) || 1) : this.props.PanelHeight() * this.heightPercent / 100; + + screenToLocalTransform = () => { + const offset = (this.props.PanelWidth() - this.panelWidth()) / 2 / this.scaling(); + return this.props.ScreenToLocalTransform().translate(-offset, 0).scale(100 / this.heightPercent); + } + + marqueeFitScaling = () => (this.props.scaling?.() || 1) * this.heightPercent / 100; + marqueeOffset = () => [this.panelWidth() / 2 * (1 - this.heightPercent / 100) / (this.heightPercent / 100), 0]; + + timelineDocFilter = () => [`_timelineLabel:true,${Utils.noRecursionHack}:x`]; + + + // renders video controls + @computed get uIButtons() { + const curTime = NumCast(this.layoutDoc._currentTimecode) - (this.timeline?.clipStart || 0); + return <div className="videoBox-ui" style={this._fullScreen || this.heightPercent == 100 ? { fontSize: "40px", minWidth: "80%" } : {}}> + <div className="videobox-button" + title={this._playing ? "play" : "pause"} + onPointerDown={this.onPlayDown}> + <FontAwesomeIcon icon={this._playing ? "pause" : "play"} /> + </div> + + {this.timeline && <div className="timecode-controls"> + <div className="timecode-current"> + {formatTime(curTime)} + </div> + + {this._fullScreen || this.heightPercent == 100 ? + <div className="timeline-slider"> + <input type="range" step="0.1" min={this.timeline.clipStart} max={this.timeline.clipEnd} value={curTime} + className="toolbar-slider time-progress" + onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => { this.setPlayheadTime(Number(e.target.value)) }} + /> + </div> + : + <div>/</div>} + + <div className="timecode-end"> + {formatTime(this.timeline.clipDuration)} + </div> + </div>} + + <div className="videobox-button" + title={"full screen"} + onPointerDown={this.onFullDown}> + <FontAwesomeIcon icon="expand" /> + </div> + + {!this._fullScreen && <div className="videobox-button" + title={"show timeline"} + onPointerDown={this.onTimelineHdlDown}> + <FontAwesomeIcon icon="eye" /> + </div>} + + {!this._fullScreen && <div className="videobox-button" + title={this.timeline?.IsTrimming !== TrimScope.None ? "finish trimming" : "start trim"} + onPointerDown={this.onClipPointerDown}> + <FontAwesomeIcon icon={this.timeline?.IsTrimming !== TrimScope.None ? "check" : "cut"} /> + </div>} + + <div className="videobox-button show-slider" + title={this._muted ? "unmute" : "mute"} + onPointerDown={(e) => { e.stopPropagation(); this.toggleMute(); }}> + <FontAwesomeIcon icon={this._muted ? "volume-mute" : "volume-up"} /> + </div> + <input type="range" step="0.1" min="0" max="1" value={this._muted ? 0 : this._volume} + className="toolbar-slider volume" + onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => { this.setVolume(Number(e.target.value)) }} + /> + + {!this._fullScreen && this.heightPercent != 100 && + <> + <div className="videobox-button" title="zoom"> + <FontAwesomeIcon icon="search-plus" /> + </div> + <input type="range" step="0.1" min="1" max="5" value={this.timeline?._zoomFactor} + className="toolbar-slider zoom" + onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => { this.zoom(Number(e.target.value)); }} + /> + </>} + </div> + } + + // renders CollectionStackedTimeline @computed get renderTimeline() { return <div className="videoBox-stackPanel" style={{ transition: this.transition, height: `${100 - this.heightPercent}%` }}> - <CollectionStackedTimeline ref={this._stackedTimeline} {...this.props} + <CollectionStackedTimeline ref={action((r: any) => this._stackedTimeline = r)} {...this.props} fieldKey={this.annotationKey} dictationKey={this.fieldKey + "-dictation"} mediaPath={this.audiopath} @@ -522,59 +762,29 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp endTag={"_timecodeToHide" /* videoEnd */} bringToFront={emptyFunction} CollectionView={undefined} - duration={this.duration} playFrom={this.playFrom} - setTime={this.setAnchorTime} + setTime={this.setPlayheadTime} playing={this.playing} isAnyChildContentActive={this.isAnyChildContentActive} whenChildContentsActiveChanged={this.timelineWhenChildContentsActiveChanged} + moveDocument={this.moveDocument} + addDocument={this.addDocument} removeDocument={this.removeDocument} ScreenToLocalTransform={this.timelineScreenToLocal} Play={this.Play} Pause={this.Pause} playLink={this.playLink} PanelHeight={this.timelineHeight} - trimming={false} - trimStart={0} - trimEnd={this.duration} - trimDuration={this.duration} - setStartTrim={emptyFunction} - setEndTrim={emptyFunction} + rawDuration={this.rawDuration} /> </div>; } + // renders annotation layer @computed get annotationLayer() { return <div className="videoBox-annotationLayer" style={{ transition: this.transition, height: `${this.heightPercent}%` }} ref={this._annotationLayer} />; } - marqueeDown = (e: React.PointerEvent) => { - if (!e.altKey && e.button === 0 && this.layoutDoc._viewScale === 1 && this.props.isContentActive(true) && ![InkTool.Highlighter, InkTool.Pen, InkTool.Write].includes(CurrentUserUtils.SelectedTool)) { - setupMoveUpEvents(this, e, action(e => { - MarqueeAnnotator.clearAnnotations(this._savedAnnotations); - this._marqueeing = [e.clientX, e.clientY]; - return true; - }), returnFalse, () => MarqueeAnnotator.clearAnnotations(this._savedAnnotations), false); - } - } - - finishMarquee = action(() => { - this._marqueeing = undefined; - this.props.select(true); - }); - - @computed get fitWidth() { return this.props.docViewPath?.().slice(-1)[0].fitWidth; } - contentFunc = () => [this.youtubeVideoId ? this.youtubeContent : this.content]; - scaling = () => this.props.scaling?.() || 1; - panelWidth = (): number => this.fitWidth ? this.props.PanelWidth() : (Doc.NativeAspect(this.rootDoc) || 1) * this.panelHeight(); - panelHeight = (): number => this.fitWidth ? this.panelWidth() / (Doc.NativeAspect(this.rootDoc) || 1) : this.heightPercent / 100 * this.props.PanelHeight(); - screenToLocalTransform = () => { - const offset = (this.props.PanelWidth() - this.panelWidth()) / 2 / this.scaling(); - return this.props.ScreenToLocalTransform().translate(-offset, 0).scale(100 / this.heightPercent); - } - marqueeFitScaling = () => (this.props.scaling?.() || 1) * this.heightPercent / 100; - marqueeOffset = () => [this.panelWidth() / 2 * (1 - this.heightPercent / 100) / (this.heightPercent / 100), 0]; - timelineDocFilter = () => [`_timelineLabel:true,${Utils.noRecursionHack}:x`]; render() { const borderRad = this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BorderRounding); const borderRadius = borderRad?.includes("px") ? `${Number(borderRad.split("px")[0]) / this.scaling()}px` : borderRad; @@ -627,7 +837,6 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp mainCont={this._mainCont.current} />} {this.renderTimeline} - {this.uIButtons} </div> </div >); } diff --git a/src/client/views/nodes/WebBox.tsx b/src/client/views/nodes/WebBox.tsx index 54c49ecad..c740644d4 100644 --- a/src/client/views/nodes/WebBox.tsx +++ b/src/client/views/nodes/WebBox.tsx @@ -590,10 +590,12 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps this._draggingSidebar = true; const localDelta = this.props.ScreenToLocalTransform().scale(this.props.scaling?.() || 1).transformDirection(delta[0], delta[1]); const nativeWidth = NumCast(this.layoutDoc[this.fieldKey + "-nativeWidth"]); + const nativeHeight = NumCast(this.layoutDoc[this.fieldKey + "-nativeHeight"]); const curNativeWidth = NumCast(this.layoutDoc.nativeWidth, nativeWidth); const ratio = (curNativeWidth + (onButton ? 1 : -1) * localDelta[0] / (this.props.scaling?.() || 1)) / nativeWidth; if (ratio >= 1) { this.layoutDoc.nativeWidth = nativeWidth * ratio; + this.layoutDoc.nativeHeight = nativeHeight * (1 + ratio); onButton && (this.layoutDoc._width = this.layoutDoc[WidthSym]() + localDelta[0]); this.layoutDoc._showSidebar = nativeWidth !== this.layoutDoc._nativeWidth; } diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index 391d84cfa..ea2f63aff 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -245,7 +245,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<(FieldViewProp this._editorView.updateState(state); const tsel = this._editorView.state.selection.$from; - tsel.marks().filter(m => m.type === this._editorView!.state.schema.marks.user_mark).map(m => AudioBox.SetScrubTime(Math.max(0, m.attrs.modified * 1000))); + //tsel.marks().filter(m => m.type === this._editorView!.state.schema.marks.user_mark).map(m => AudioBox.SetScrubTime(Math.max(0, m.attrs.modified * 1000))); const curText = state.doc.textBetween(0, state.doc.content.size, " \n"); const curTemp = this.layoutDoc.resolvedDataDoc ? Cast(this.layoutDoc[this.props.fieldKey], RichTextField) : undefined; // the actual text in the text box const curProto = Cast(Cast(this.dataDoc.proto, Doc, null)?.[this.fieldKey], RichTextField, null); // the default text inherited from a prototype diff --git a/src/client/views/nodes/trails/PresBox.tsx b/src/client/views/nodes/trails/PresBox.tsx index 9b74bb618..2e312ee51 100644 --- a/src/client/views/nodes/trails/PresBox.tsx +++ b/src/client/views/nodes/trails/PresBox.tsx @@ -201,26 +201,13 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() { const targMedia = DocumentManager.Instance.getDocumentView(targetDoc); targMedia?.ComponentView?.playFrom?.(NumCast(activeItem.presStartTime), NumCast(activeItem.presStartTime) + duration); } - // if (targetDoc.type === DocumentType.AUDIO) { - // if (this._mediaTimer && this._mediaTimer[1] === targetDoc) clearTimeout(this._mediaTimer[0]); - // targetDoc._triggerAudio = NumCast(activeItem.presStartTime); - // this._mediaTimer = [setTimeout(() => targetDoc._audioStop = true, duration * 1000), targetDoc]; - // } else if (targetDoc.type === DocumentType.VID) { - // targetDoc._triggerVideoStop = true; - // setTimeout(() => targetDoc._currentTimecode = NumCast(activeItem.presStartTime), 10); - // setTimeout(() => targetDoc._triggerVideo = true, 20); - // this._mediaTimer = [setTimeout(() => targetDoc._triggerVideoStop = true, (duration * 1000) + 20), targetDoc]; - // } } stopTempMedia = (targetDocField: FieldResult) => { const targetDoc = Cast(targetDocField, Doc, null); - if (targetDoc?.type === DocumentType.AUDIO) { - if (this._mediaTimer && this._mediaTimer[1] === targetDoc) clearTimeout(this._mediaTimer[0]); - targetDoc._audioStop = true; - } else if (targetDoc?.type === DocumentType.VID) { - if (this._mediaTimer && this._mediaTimer[1] === targetDoc) clearTimeout(this._mediaTimer[0]); - targetDoc._triggerVideoStop = true; + if ([DocumentType.VID, DocumentType.AUDIO].includes(targetDoc.type as any)) { + const targMedia = DocumentManager.Instance.getDocumentView(targetDoc); + targMedia?.ComponentView?.Pause?.(); } } @@ -1484,6 +1471,8 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() { @computed get mediaOptionsDropdown() { const activeItem: Doc = this.activeItem; const targetDoc: Doc = this.targetDoc; + const clipStart: number = NumCast(activeItem.clipStart); + const clipEnd: number = NumCast(activeItem.clipEnd); const duration = Math.round(NumCast(activeItem[`${Doc.LayoutFieldKey(activeItem)}-duration`]) * 10); const mediaStopDocInd: number = NumCast(activeItem.mediaStopDoc); const mediaStopDocStr: string = mediaStopDocInd ? mediaStopDocInd + ". " + this.childDocs[mediaStopDocInd - 1].title : ""; @@ -1529,7 +1518,7 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() { </div> </div> <div className="multiThumb-slider"> - <input type="range" step="0.1" min="0" max={duration / 10} value={NumCast(activeItem.presEndTime)} + <input type="range" step="0.1" min={clipStart} max={clipEnd} value={NumCast(activeItem.presEndTime)} style={{ gridColumn: 1, gridRow: 1 }} className={`toolbar-slider ${"end"}`} id="toolbar-slider" @@ -1553,7 +1542,7 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() { e.stopPropagation(); activeItem.presEndTime = Number(e.target.value); }} /> - <input type="range" step="0.1" min="0" max={duration / 10} value={NumCast(activeItem.presStartTime)} + <input type="range" step="0.1" min={clipStart} max={clipEnd} value={NumCast(activeItem.presStartTime)} style={{ gridColumn: 1, gridRow: 1 }} className={`toolbar-slider ${"start"}`} id="toolbar-slider" @@ -1579,9 +1568,9 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() { }} /> </div> <div className={`slider-headers ${activeItem.presMovement === PresMovement.Pan || activeItem.presMovement === PresMovement.Zoom ? "" : "none"}`}> - <div className="slider-text">0 s</div> + <div className="slider-text">{clipStart} s</div> <div className="slider-text"></div> - <div className="slider-text">{duration / 10} s</div> + <div className="slider-text">{clipEnd} s</div> </div> </div> <div className="ribbon-final-box"> diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 1253cf9c7..642becb46 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -346,6 +346,7 @@ export namespace Doc { return GetT(doc, "system", "boolean", true); } export async function SetInPlace(doc: Doc, key: string, value: Field | undefined, defaultProto: boolean) { + if (key.startsWith("_")) key = key.substring(1); const hasProto = doc.proto instanceof Doc; const onDeleg = Object.getOwnPropertyNames(doc).indexOf(key) !== -1; const onProto = hasProto && Object.getOwnPropertyNames(doc.proto).indexOf(key) !== -1; diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts index c34e8f93d..6d24c6cbc 100644 --- a/src/fields/InkField.ts +++ b/src/fields/InkField.ts @@ -20,6 +20,7 @@ export enum InkTool { export interface PointData { X: number; Y: number; + time?: number; } export type Segment = Array<Bezier>; @@ -84,7 +85,7 @@ export class InkField extends ObjectField { } [ToScriptString]() { - return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}} `) + "])"; + return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}, time: ${i.time || 0}} `) + "])"; } [ToString]() { return "InkField"; diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts index 65f2bf80c..cf59cb3c6 100644 --- a/src/pen-gestures/GestureUtils.ts +++ b/src/pen-gestures/GestureUtils.ts @@ -8,7 +8,8 @@ export namespace GestureUtils { readonly gesture: Gestures, readonly points: PointData[], readonly bounds: Rect, - readonly text?: any + readonly text?: any, + readonly times?: number[] ) { } } diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts deleted file mode 100644 index ee8dd5b5b..000000000 --- a/src/scraping/buxton/final/BuxtonImporter.ts +++ /dev/null @@ -1,604 +0,0 @@ -import { readdirSync, writeFile, mkdirSync, createReadStream, createWriteStream, existsSync, statSync } from "fs"; -import * as path from "path"; -import { red, cyan, yellow } from "colors"; -import { Utils } from "../../../Utils"; -import rimraf = require("rimraf"); -import { DashUploadUtils } from "../../../server/DashUploadUtils"; -const StreamZip = require('node-stream-zip'); -const createImageSizeStream = require("image-size-stream"); -import { parseXml } from "libxmljs"; -import { strictEqual } from "assert"; -import { Readable, PassThrough } from "stream"; -import { Directory, serverPathToFile, pathToDirectory } from "../../../server/ApiManagers/UploadManager"; - -/** - * This is an arbitrary bundle of data that gets populated - * in extractFileContents - */ -interface DocumentContents { - body: string; - imageData: ImageData[]; - hyperlinks: string[]; - tableData: TableData[]; - longDescription: string; -} - -/** - * A rough schema for everything that Bill has - * included for each document - */ -export interface DeviceDocument { - title: string; - shortDescription: string; - longDescription: string; - company: string; - year: number; - originalPrice?: number; - degreesOfFreedom?: number; - dimensions?: string; - primaryKey: string; - secondaryKey: string; - attribute: string; - __images: ImageData[]; - additionalMedia: ({ [type: string]: string } | undefined)[]; - hyperlinks: string[]; - captions: string[]; // from the table column - embeddedFileNames: string[]; // from the table column -} - -/** - * A layer of abstraction around a single parsing - * attempt. The error is not a TypeScript error, but - * rather an invalidly formatted value for a given key. - */ -export interface AnalysisResult { - device?: DeviceDocument; - invalid?: { [deviceProperty: string]: string }; -} - -/** - * A mini API that takes in a string and returns - * either the given T or an error indicating that the - * transformation was rejected. - */ -type Transformer<T> = (raw: string) => TransformResult<T>; -interface TransformResult<T> { - transformed?: T; - error?: string; -} - -/** - * Simple bundle counting successful and failed imports - */ -export interface ImportResults { - deviceCount: number; - errorCount: number; -} - -/** - * Definitions for callback functions. Such instances are - * just invoked by when a single document has been parsed - * or the entire import is over. As of this writing, these - * callbacks are supplied by WebSocket.ts and used to inform - * the client of these events. - */ -type ResultCallback = (result: AnalysisResult) => void; -type TerminatorCallback = (result: ImportResults) => void; - -/** - * Defines everything needed to define how a single key should be - * formatted within the plain body text. The association between - * keys and their format definitions is stored FormatMap - */ -interface ValueFormatDefinition<T> { - exp: RegExp; // the expression that the key's value should match - matchIndex?: number; // defaults to 0, but can be overridden to account for grouping in @param exp - transformer?: Transformer<T>; // if desirable, how to transform the Regex match - required?: boolean; // defaults to true, confirms that for a whole document to be counted successful, - // all of its required values should be present and properly formatted -} - -/** - * The basic data we extract from each image in the document - */ -interface ImageData { - url: string; - nativeWidth: number; - nativeHeight: number; -} - -namespace Utilities { - - /** - * Numeric 'try parse', fits with the Transformer API - * @param raw the serialized number - */ - export function numberValue(raw: string): TransformResult<number> { - const transformed = Number(raw); - if (isNaN(transformed)) { - return { error: `${raw} cannot be parsed to a numeric value.` }; - } - return { transformed }; - } - - /** - * A simple tokenizer that splits along 'and' and commas, and removes duplicates - * Helpful mainly for attribute and primary key lists - * @param raw the string to tokenize - */ - export function collectUniqueTokens(raw: string): TransformResult<string[]> { - const pieces = raw.replace(/,|\s+and\s+/g, " ").split(/\s+/).filter(piece => piece.length); - const unique = new Set(pieces.map(token => token.toLowerCase().trim())); - return { transformed: Array.from(unique).map(capitalize).sort() }; - } - - /** - * Tries to correct XML text parsing artifact where some sentences lose their separating space, - * and others gain excess whitespace - * @param raw - */ - export function correctSentences(raw: string): TransformResult<string> { - raw = raw.replace(/\./g, ". ").replace(/\:/g, ": ").replace(/\,/g, ", ").replace(/\?/g, "? ").trimRight(); - raw = raw.replace(/\s{2,}/g, " "); - return { transformed: raw }; - } - - /** - * Simple capitalization - * @param word to capitalize - */ - export function capitalize(word: string): string { - const clean = word.trim(); - if (!clean.length) { - return word; - } - return word.charAt(0).toUpperCase() + word.slice(1); - } - - /** - * Streams the requeted file at the relative path to the - * root of the zip, then parses it with a library - * @param zip the zip instance data source - * @param relativePath the path to a .xml file within the zip to parse - */ - export async function readAndParseXml(zip: any, relativePath: string) { - console.log(`Text streaming ${relativePath}`); - const contents = await new Promise<string>((resolve, reject) => { - let body = ""; - zip.stream(relativePath, (error: any, stream: any) => { - if (error) { - reject(error); - } - stream.on('data', (chunk: any) => body += chunk.toString()); - stream.on('end', () => resolve(body)); - }); - }); - return parseXml(contents); - } -} - -/** - * Defines how device values should be formatted. As you can see, the formatting is - * not super consistent and has changed over time as edge cases have been found, but this - * at least imposes some constraints, and will notify you if a document doesn't match the specifications - * in this map. - */ -const FormatMap = new Map<keyof DeviceDocument, ValueFormatDefinition<any>>([ - ["title", { - exp: /contact\s+(.*)Short Description:/ - }], - ["company", { - exp: /Company:\s+([^\|]*)\s+\|/, - transformer: (raw: string) => ({ transformed: raw.replace(/\./g, "") }) - }], - ["year", { - exp: /Year:\s+([^\|]*)\s+\|/, - transformer: (raw: string) => Utilities.numberValue(/[0-9]{4}/.exec(raw)![0]) - }], - ["primaryKey", { - exp: /Primary:\s+(.*)(Secondary|Additional):/, - transformer: raw => { - const { transformed, error } = Utilities.collectUniqueTokens(raw); - return transformed ? { transformed: transformed[0] } : { error }; - } - }], - ["secondaryKey", { - exp: /(Secondary|Additional):\s+(.*)Attributes?:/, - transformer: raw => { - const { transformed, error } = Utilities.collectUniqueTokens(raw); - return transformed ? { transformed: transformed[0] } : { error }; - }, - matchIndex: 2 - }], - ["attribute", { - exp: /Attributes?:\s+(.*)Links/, - transformer: raw => { - const { transformed, error } = Utilities.collectUniqueTokens(raw); - return transformed ? { transformed: transformed[0] } : { error }; - }, - }], - ["originalPrice", { - exp: /Original Price \(USD\)\:\s+(\$[0-9\,]+\.[0-9]+|NFS)/, - transformer: (raw: string) => { - raw = raw.replace(/\,/g, ""); - if (raw === "NFS") { - return { transformed: -1 }; - } - return Utilities.numberValue(raw.slice(1)); - }, - required: false - }], - ["degreesOfFreedom", { - exp: /Degrees of Freedom:\s+([0-9]+)/, - transformer: Utilities.numberValue, - required: false - }], - ["dimensions", { - exp: /Dimensions\s+\(L x W x H\):\s+([0-9\.]+\s+x\s+[0-9\.]+\s+x\s+[0-9\.]+\s\([A-Za-z]+\))/, - transformer: (raw: string) => { - const [length, width, group] = raw.split(" x "); - const [height, unit] = group.split(" "); - return { - transformed: { - dim_length: Number(length), - dim_width: Number(width), - dim_height: Number(height), - dim_unit: unit.replace(/[\(\)]+/g, "") - } - }; - }, - required: false - }], - ["shortDescription", { - exp: /Short Description:\s+(.*)Bill Buxton[’']s Notes/, - transformer: Utilities.correctSentences - }], -]); - -const sourceDir = path.resolve(__dirname, "source"); // where the Word documents are assumed to be stored -const assetDir = path.resolve(__dirname, "assets"); // where any additional media content like pdfs will be stored. Each subdirectory of this -// must follow the enum Directory.<type> naming scheme -const outDir = path.resolve(__dirname, "json"); // where the JSON output of these device documents will be written -const imageDir = path.resolve(__dirname, "../../../server/public/files/images/buxton"); // where, in the server, these images will be written -const successOut = "buxton.json"; // the JSON list representing properly formatted documents -const failOut = "incomplete.json"; // the JSON list representing improperly formatted documents -const deviceKeys = Array.from(FormatMap.keys()); // a way to iterate through all keys of the DeviceDocument interface - -/** - * Starts by REMOVING ALL EXISTING BUXTON RESOURCES. This might need to be - * changed going forward - * @param emitter the callback when each document is completed - * @param terminator the callback when the entire import is completed - */ -export default async function executeImport(emitter: ResultCallback, terminator: TerminatorCallback) { - try { - // get all Word documents in the source directory - const contents = readdirSync(sourceDir); - const wordDocuments = contents.filter(file => /.*\.docx?$/.test(file)).map(file => `${sourceDir}/${file}`); - // removal takes place here - [outDir, imageDir].forEach(dir => { - rimraf.sync(dir); - mkdirSync(dir); - }); - await transferAssets(); - return parseFiles(wordDocuments, emitter, terminator); - } catch (e: any) { - const message = [ - "Unable to find a source directory.", - "Please ensure that the following directory exists:", - `${e.message}` - ].join('\n'); - console.log(red(message)); - return { error: message }; - } -} - -/** - * Builds a mirrored directory structure of all media / asset files - * within the server's public directory. - */ -async function transferAssets() { - for (const assetType of readdirSync(assetDir)) { - const subroot = path.resolve(assetDir, assetType); - if (!statSync(subroot).isDirectory()) { - continue; - } - const outputSubroot = serverPathToFile(assetType as Directory, "buxton"); - if (existsSync(outputSubroot)) { - continue; - } else { - mkdirSync(outputSubroot); - } - for (const fileName of readdirSync(subroot)) { - const readStream = createReadStream(path.resolve(subroot, fileName)); - const writeStream = createWriteStream(path.resolve(outputSubroot, fileName)); - await new Promise<void>(resolve => { - readStream.pipe(writeStream).on("close", resolve); - }); - } - } -} - -/** - * Parse every Word document in the directory, notifying any callers as needed - * at each iteration via the emitter. - * @param wordDocuments the string list of Word document names to parse - * @param emitter the callback when each document is completed - * @param terminator the callback when the entire import is completed - */ -async function parseFiles(wordDocuments: string[], emitter: ResultCallback, terminator: TerminatorCallback): Promise<DeviceDocument[]> { - // execute parent-most parse function - const results: AnalysisResult[] = []; - for (const filePath of wordDocuments) { - const fileName = path.basename(filePath).replace("Bill_Notes_", ""); // not strictly needed, but cleaner - console.log(cyan(`\nExtracting contents from ${fileName}...`)); - const result = analyze(fileName, await extractFileContents(filePath)); - emitter(result); - results.push(result); - } - - // collect information about errors and successes - const masterDevices: DeviceDocument[] = []; - const masterErrors: { [key: string]: string }[] = []; - results.forEach(({ device, invalid: errors }) => { - if (device) { - masterDevices.push(device); - } else if (errors) { - masterErrors.push(errors); - } - }); - - // something went wrong, since errors and successes should sum to total inputs - const total = wordDocuments.length; - if (masterDevices.length + masterErrors.length !== total) { - throw new Error(`Encountered a ${masterDevices.length} to ${masterErrors.length} mismatch in device / error split!`); - } - - // write the external JSON representations of this import - console.log(); - await writeOutputFile(successOut, masterDevices, total, true); - await writeOutputFile(failOut, masterErrors, total, false); - console.log(); - - // notify the caller that the import has finished - terminator({ deviceCount: masterDevices.length, errorCount: masterErrors.length }); - - return masterDevices; -} - -/** - * XPath definitions for desired XML targets in respective hierarchies. - * - * For table cells, can be read as: "find me anything that looks like <w:tc> in XML, whose - * parent looks like <w:tr>, whose parent looks like <w:tbl>" - * - * <w:tbl> - * <w:tr> - * <w:tc> - * - * These are found by trial and error, and using an online XML parser / prettifier - * to inspect the structure, since the Node XML library does not expose the parsed - * structure very well for searching, say in the debug console. - */ -const xPaths = { - paragraphs: '//*[name()="w:p"]', - tableCells: '//*[name()="w:tbl"]/*[name()="w:tr"]/*[name()="w:tc"]', - hyperlinks: '//*[name()="Relationship" and contains(@Type, "hyperlink")]' -}; - -interface TableData { - fileName: string; - caption: string; - additionalMedia?: { [type: string]: string }; -} - -const SuffixDirectoryMap = new Map<string, Directory>([ - ["p", Directory.pdfs] -]); - -/** - * The meat of the script, images and text content are extracted here - * @param pathToDocument the path to the document relative to the root of the zip - */ -async function extractFileContents(pathToDocument: string): Promise<DocumentContents> { - console.log('Extracting text...'); - const zip = new StreamZip({ file: pathToDocument, storeEntries: true }); - await new Promise<void>(resolve => zip.on('ready', resolve)); - - // extract the body of the document and, specifically, its captions - const document = await Utilities.readAndParseXml(zip, "word/document.xml"); - // get plain text - const body = document.root()?.text() ?? "No body found. Check the import script's XML parser."; - const captions: string[] = []; - const tableData: TableData[] = []; - // preserve paragraph formatting and line breaks that would otherwise get lost in the plain text parsing - // of the XML hierarchy - const paragraphs = document.find(xPaths.paragraphs).map(node => Utilities.correctSentences(node.text()).transformed!); - const start = paragraphs.indexOf(paragraphs.find(el => /Bill Buxton[’']s Notes/.test(el))!) + 1; - const end = paragraphs.indexOf("Device Details"); - const longDescription = paragraphs.slice(start, end).filter(paragraph => paragraph.length).join("\n\n"); - - // extract captions from the table cells - const tableRowsFlattened = document.find(xPaths.tableCells).map(node => node.text().trim()); - const { length } = tableRowsFlattened; - const numCols = 4; - strictEqual(length > numCols, true, "No captions written."); // first row has the headers, not content - strictEqual(length % numCols === 0, true, "Improper caption formatting."); - - // break the flat list of strings into groups of numColumns. Thus, each group represents - // a row in the table, where the first row has no text content since it's - // the image, the second has the file name and the third has the caption (maybe additional columns - // have been added or reordered since this was written, but follow the same appraoch) - for (let i = numCols; i < tableRowsFlattened.length; i += numCols) { - const row = tableRowsFlattened.slice(i, i + numCols); - const entry: TableData = { fileName: row[1], caption: row[2] }; - const key = SuffixDirectoryMap.get(row[3].toLowerCase()); - if (key) { - const media: any = {}; - media[key] = `${entry.fileName.split(".")[0]}.pdf`; - entry.additionalMedia = media; - } - tableData.push(entry); - } - - // extract all hyperlinks embedded in the document - const rels = await Utilities.readAndParseXml(zip, "word/_rels/document.xml.rels"); - const hyperlinks = rels.find(xPaths.hyperlinks).map(el => el.attrs()[2].value()); - console.log("Text extracted."); - - // write out the images for this document - console.log("Beginning image extraction..."); - const imageData = await writeImages(zip); - console.log(`Extracted ${imageData.length} images.`); - - // cleanup - zip.close(); - - return { body, longDescription, imageData, tableData, hyperlinks }; -} - -// zip relative path from root expression / filter used to isolate only media assets -const imageEntry = /^word\/media\/\w+\.(jpeg|jpg|png|gif)/; - -/** - * Image dimensions and file suffix, - */ -interface ImageAttrs { - width: number; - height: number; - type: string; -} - -/** - * For each image, stream the file, get its size, check if it's an icon - * (if it is, ignore it) - * @param zip the zip instance data source - */ -async function writeImages(zip: any): Promise<ImageData[]> { - const allEntries = Object.values<any>(zip.entries()).map(({ name }) => name); - const imageEntries = allEntries.filter(name => imageEntry.test(name)); - - const imageUrls: ImageData[] = []; - const valid: any[] = []; - - const getImageStream = (mediaPath: string) => new Promise<Readable>((resolve, reject) => { - zip.stream(mediaPath, (error: any, stream: any) => error ? reject(error) : resolve(stream)); - }); - - for (const mediaPath of imageEntries) { - const { width, height, type } = await new Promise<ImageAttrs>(async resolve => { - const sizeStream = (createImageSizeStream() as PassThrough).on('size', (dimensions: ImageAttrs) => { - readStream.destroy(); - resolve(dimensions); - }).on("error", () => readStream.destroy()); - const readStream = await getImageStream(mediaPath); - readStream.pipe(sizeStream); - }); - - // if it's not an icon, by this rough heuristic, i.e. is it not square - const number = Number(/image(\d+)/.exec(mediaPath)![1]); - if (number > 5 || width - height > 10) { - valid.push({ width, height, type, mediaPath, number }); - } - } - - valid.sort((a, b) => a.number - b.number); - - const [{ width: first_w, height: first_h }, { width: second_w, height: second_h }] = valid; - if (Math.abs(first_w / second_w - first_h / second_h) < 0.01) { - const first_size = first_w * first_h; - const second_size = second_w * second_h; - const target = first_size >= second_size ? 1 : 0; - valid.splice(target, 1); - console.log(`Heuristically removed image with size ${target ? second_size : first_size}`); - } - - // for each valid image, output the _o, _l, _m, and _s files - // THIS IS WHERE THE SCRIPT SPENDS MOST OF ITS TIME - for (const { type, width, height, mediaPath } of valid) { - const generatedFileName = `upload_${Utils.GenerateGuid()}.${type.toLowerCase()}`; - await DashUploadUtils.outputResizedImages(() => getImageStream(mediaPath), generatedFileName, imageDir); - imageUrls.push({ - url: `/files/images/buxton/${generatedFileName}`, - nativeWidth: width, - nativeHeight: height - }); - } - - return imageUrls; -} - -/** - * Takes the results of extractFileContents, which relative to this is sort of the - * external media / preliminary text processing, and now tests the given file name to - * with those value definitions to make sure the body of the document contains all - * required fields, properly formatted - * @param fileName the file whose body to inspect - * @param contents the data already computed / parsed by extractFileContents - */ -function analyze(fileName: string, contents: DocumentContents): AnalysisResult { - const { body, imageData, hyperlinks, tableData, longDescription } = contents; - const device: any = { - hyperlinks, - captions: tableData.map(({ caption }) => caption), - embeddedFileNames: tableData.map(({ fileName }) => fileName), - additionalMedia: tableData.map(({ additionalMedia }) => additionalMedia), - longDescription, - __images: imageData - }; - const errors: { [key: string]: string } = { fileName }; - - for (const key of deviceKeys) { - const { exp, transformer, matchIndex, required } = FormatMap.get(key)!; - const matches = exp.exec(body); - - let captured: string; - // if we matched and we got the specific match we're after - if (matches && (captured = matches[matchIndex ?? 1])) { // matchIndex defaults to 1 - captured = captured.replace(/\s{2,}/g, " "); // remove excess whitespace - // if supplied, apply the required transformation (recall this is specified in FormatMap) - if (transformer) { - const { error, transformed } = transformer(captured); - if (error) { - // we hit a snag trying to transform the valid match - // still counts as a fundamental error - errors[key] = `__ERR__${key.toUpperCase()}__TRANSFORM__: ${error}`; - continue; - } - captured = transformed; - } - device[key] = captured; - } else if (required ?? true) { - // the field was either implicitly or explicitly required, and failed to match the definition in - // FormatMap - errors[key] = `ERR__${key.toUpperCase()}__: outer match ${matches === null ? "wasn't" : "was"} captured.`; - continue; - } - } - - // print errors - this can be removed - const errorKeys = Object.keys(errors); - if (errorKeys.length > 1) { - console.log(red(`@ ${cyan(fileName.toUpperCase())}...`)); - errorKeys.forEach(key => key !== "filename" && console.log(red(errors[key]))); - return { invalid: errors }; - } - - return { device }; -} - -/** - * A utility function that writes the JSON results for this import out to the desired path - * @param relativePath where to write the JSON file - * @param data valid device document objects, or errors - * @param total used for more informative printing - * @param success whether or not the caller is writing the successful parses or the failures - */ -async function writeOutputFile(relativePath: string, data: any[], total: number, success: boolean) { - console.log(yellow(`Encountered ${data.length} ${success ? "valid" : "invalid"} documents out of ${total} candidates. Writing ${relativePath}...`)); - return new Promise<void>((resolve, reject) => { - const destination = path.resolve(outDir, relativePath); - const contents = JSON.stringify(data, undefined, 4); // format the JSON - writeFile(destination, contents, err => err ? reject(err) : resolve()); - }); -}
\ No newline at end of file diff --git a/src/server/Message.ts b/src/server/Message.ts index 59b24cd82..d87ae5027 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -1,5 +1,4 @@ import { Point } from "../pen-gestures/ndollar"; -import { AnalysisResult, ImportResults } from "../scraping/buxton/final/BuxtonImporter"; import { Utils } from "../Utils"; export class Message<T> { @@ -82,9 +81,6 @@ export namespace MessageStore { export const GetDocument = new Message<string>("Get Document"); export const DeleteAll = new Message<any>("Delete All"); export const ConnectionTerminated = new Message<string>("Connection Terminated"); - export const BeginBuxtonImport = new Message<string>("Begin Buxton Import"); - export const BuxtonDocumentResult = new Message<AnalysisResult>("Buxton Document Result"); - export const BuxtonImportComplete = new Message<ImportResults>("Buxton Import Complete"); export const GesturePoints = new Message<GestureContent>("Gesture Points"); export const MobileInkOverlayTrigger = new Message<MobileInkOverlayContent>("Trigger Mobile Ink Overlay"); diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 0c79c1bbb..1b7f5919f 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -4,7 +4,6 @@ import { createServer, Server } from "https"; import { networkInterfaces } from "os"; import * as sio from 'socket.io'; import { Socket } from "socket.io"; -import executeImport from "../scraping/buxton/final/BuxtonImporter"; import { Utils } from "../Utils"; import { logPort } from './ActionUtilities'; import { timeMap } from "./ApiManagers/UserManager"; @@ -125,12 +124,6 @@ export namespace WebSocket { * as an emitter and a terminator the functions that simply broadcast a result * or indicate termination to the client via the web socket */ - Utils.AddServerHandler(socket, MessageStore.BeginBuxtonImport, () => { - executeImport( - deviceOrError => Utils.Emit(socket, MessageStore.BuxtonDocumentResult, deviceOrError), - results => Utils.Emit(socket, MessageStore.BuxtonImportComplete, results) - ); - }); disconnect = () => { socket.broadcast.emit("connection_terminated", Date.now()); diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 068ac2159..bc7727426 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -8,6 +8,7 @@ declare module 'webrtc-adapter'; declare module 'bezier-curve'; declare module 'fit-curve'; declare module 'react-audio-waveform'; +declare module 'iink-js'; declare module 'reveal'; declare module 'react-reveal'; |