diff options
| author | Stanley Yip <stanley_yip@brown.edu> | 2020-02-12 21:54:58 -0500 |
|---|---|---|
| committer | Stanley Yip <stanley_yip@brown.edu> | 2020-02-12 21:54:58 -0500 |
| commit | a8b19e82d8c9a8350609e64e60848dbfbbbb6dcd (patch) | |
| tree | 35d1e015a4b6f4913f162faec50c79c36e402350 | |
| parent | 33d5a12af14e1ed50e5c3164b363fbbc253506a0 (diff) | |
| parent | 864cba561db8e26240b093da7ab524e76c8823d1 (diff) | |
merge
30 files changed, 881 insertions, 89 deletions
diff --git a/package-lock.json b/package-lock.json index 049c91fcf..ec265527d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14665,19 +14665,22 @@ "dev": true }, "resilient-server-session": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/resilient-server-session/-/resilient-server-session-1.1.2.tgz", - "integrity": "sha512-eLoXxTc5bFOYH2JejSCYc2O8emoo80p2zOuwVVWVoK6/2NJBzLP8Yl7kU8m7tJDrOoqDSZGghsVD5ob9BbUgAQ==", + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/resilient-server-session/-/resilient-server-session-1.1.9.tgz", + "integrity": "sha512-XSVujTyJOQMACllXUvWOSHY4GK4JI6aECjCrQR0UBvd2+hdjM1euffspn2b+7M0fepo+bJ71YrAOA9M34ChBZw==", "requires": { "@types/chai": "^4.2.7", + "@types/express": "^4.17.2", "@types/mocha": "^5.2.7", "@types/node": "^10.12.30", "@types/request-promise": "^4.1.42", "@types/uuid": "^3.4.6", "chai": "^4.2.0", "colors": "^1.4.0", + "express": "^4.17.1", "jsonschema": "^1.2.5", "mocha": "^7.0.0", + "request": "^2.88.0", "request-promise": "^4.2.5", "typescript": "^3.7.4", "uuid": "^3.3.3" @@ -14842,9 +14845,9 @@ } }, "mocha": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-7.0.0.tgz", - "integrity": "sha512-CirsOPbO3jU86YKjjMzFLcXIb5YiGLUrjrXFHoJ3e2z9vWiaZVCZQ2+gtRGMPWF+nFhN6AWwLM/juzAQ6KRkbA==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-7.0.1.tgz", + "integrity": "sha512-9eWmWTdHLXh72rGrdZjNbG3aa1/3NRPpul1z0D979QpEnFdCG0Q5tv834N+94QEN2cysfV72YocQ3fn87s70fg==", "requires": { "ansi-colors": "3.2.3", "browser-stdout": "1.3.1", diff --git a/package.json b/package.json index 7e0adfba6..6a2b5fac4 100644 --- a/package.json +++ b/package.json @@ -233,7 +233,7 @@ "readline": "^1.3.0", "request": "^2.88.0", "request-promise": "^4.2.5", - "resilient-server-session": "^1.1.2", + "resilient-server-session": "^1.1.9", "rimraf": "^3.0.0", "serializr": "^1.5.4", "sharp": "^0.23.4", diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 5fcd2547e..33f7c1d35 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -1,10 +1,12 @@ import * as OpenSocket from 'socket.io-client'; -import { MessageStore, YoutubeQueryTypes } from "./../server/Message"; +import { MessageStore, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent } from "./../server/Message"; import { Opt, Doc } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; import { SerializationHelper } from './util/SerializationHelper'; import { RefField } from '../new_fields/RefField'; import { Id, HandleUpdate } from '../new_fields/FieldSymbols'; +import GestureOverlay from './views/GestureOverlay'; +import MobileInkOverlay from '../mobile/MobileInkOverlay'; /** * This class encapsulates the transfer and cross-client synchronization of @@ -64,6 +66,26 @@ export namespace DocServer { } } + export namespace Mobile { + + export function dispatchGesturePoints(content: GestureContent) { + Utils.Emit(_socket, MessageStore.GesturePoints, content); + } + + export function dispatchOverlayTrigger(content: MobileInkOverlayContent) { + // _socket.emit("dispatchBoxTrigger"); + Utils.Emit(_socket, MessageStore.MobileInkOverlayTrigger, content); + } + + export function dispatchOverlayPositionUpdate(content: UpdateMobileInkOverlayPositionContent) { + Utils.Emit(_socket, MessageStore.UpdateMobileInkOverlayPosition, content); + } + + export function dispatchMobileDocumentUpload(content: MobileDocumentUploadContent) { + Utils.Emit(_socket, MessageStore.MobileDocumentUpload, content); + } + } + const instructions = "This page will automatically refresh after this alert is closed. Expect to reconnect after about 30 seconds."; function alertUser(connectionTerminationReason: string) { switch (connectionTerminationReason) { @@ -101,6 +123,21 @@ export namespace DocServer { Utils.AddServerHandler(_socket, MessageStore.DeleteField, respondToDelete); Utils.AddServerHandler(_socket, MessageStore.DeleteFields, respondToDelete); Utils.AddServerHandler(_socket, MessageStore.ConnectionTerminated, alertUser); + + // mobile ink overlay socket events to communicate between mobile view and desktop view + _socket.addEventListener("receiveGesturePoints", (content: GestureContent) => { + MobileInkOverlay.Instance.drawStroke(content); + }); + _socket.addEventListener("receiveOverlayTrigger", (content: MobileInkOverlayContent) => { + GestureOverlay.Instance.enableMobileInkOverlay(content); + MobileInkOverlay.Instance.initMobileInkOverlay(content); + }); + _socket.addEventListener("receiveUpdateOverlayPosition", (content: UpdateMobileInkOverlayPositionContent) => { + MobileInkOverlay.Instance.updatePosition(content); + }); + _socket.addEventListener("receiveMobileDocumentUpload", (content: MobileDocumentUploadContent) => { + MobileInkOverlay.Instance.uploadDocument(content); + }); } function errorFunc(): never { diff --git a/src/client/util/DragManager.ts b/src/client/util/DragManager.ts index 5d2f1bc06..bf64e0bdb 100644 --- a/src/client/util/DragManager.ts +++ b/src/client/util/DragManager.ts @@ -180,7 +180,7 @@ export namespace DragManager { ); } element.dataset.canDrop = "true"; - const handler = (e: Event) => dropFunc(e, (e as CustomEvent<DropEvent>).detail); + const handler = (e: Event) => { dropFunc(e, (e as CustomEvent<DropEvent>).detail); }; element.addEventListener("dashOnDrop", handler); return () => { element.removeEventListener("dashOnDrop", handler); @@ -196,7 +196,7 @@ export namespace DragManager { dragData.userDropAction === "alias" || (!dragData.userDropAction && dragData.dropAction === "alias") ? Doc.MakeAlias(d) : dragData.userDropAction === "copy" || (!dragData.userDropAction && dragData.dropAction === "copy") ? Doc.MakeCopy(d, true) : d) ); - e.docDragData?.droppedDocuments.forEach((drop: Doc, i: number) => + e.docDragData ?.droppedDocuments.forEach((drop: Doc, i: number) => Cast(dragData.draggedDocuments[i].removeDropProperties, listSpec("string"), []).map(prop => drop[prop] = undefined)); }; dragData.draggedDocuments.map(d => d.dragFactory); // does this help? trying to make sure the dragFactory Doc is loaded @@ -266,6 +266,10 @@ export namespace DragManager { StartDrag([ele], dragData, downX, downY, options); } + export function StartImgDrag(ele: HTMLElement, downX: number, downY: number) { + StartDrag([ele], {}, downX, downY); + } + function StartDrag(eles: HTMLElement[], dragData: { [id: string]: any }, downX: number, downY: number, options?: DragOptions, finishDrag?: (dropData: DragCompleteEvent) => void) { eles = eles.filter(e => e); if (!dragDiv) { @@ -285,6 +289,7 @@ export namespace DragManager { if (!ele.parentNode) dragDiv.appendChild(ele); const dragElement = ele.parentNode === dragDiv ? ele : ele.cloneNode(true) as HTMLElement; const rect = ele.getBoundingClientRect(); + console.log("boudning", rect); const scaleX = rect.width / ele.offsetWidth, scaleY = rect.height / ele.offsetHeight; xs.push(rect.left); @@ -302,7 +307,7 @@ export namespace DragManager { dragElement.style.transformOrigin = "0 0"; dragElement.style.borderRadius = getComputedStyle(ele).borderRadius; dragElement.style.zIndex = globalCssVariables.contextMenuZindex;// "1000"; - dragElement.style.transform = `translate(${rect.left + (options?.offsetX || 0)}px, ${rect.top + (options?.offsetY || 0)}px) scale(${scaleX}, ${scaleY})`; + dragElement.style.transform = `translate(${rect.left + (options ?.offsetX || 0)}px, ${rect.top + (options ?.offsetY || 0)}px) scale(${scaleX}, ${scaleY})`; dragElement.style.width = `${rect.width / scaleX}px`; dragElement.style.height = `${rect.height / scaleY}px`; @@ -331,8 +336,8 @@ export namespace DragManager { return dragElement; }); - const hideSource = options?.hideSource ? true : false; - eles.map(ele => ele.parentElement && ele.parentElement?.className === dragData.dragDivName ? (ele.parentElement.hidden = hideSource) : (ele.hidden = hideSource)); + const hideSource = options ?.hideSource ? true : false; + eles.map(ele => ele.parentElement && ele.parentElement ?.className === dragData.dragDivName ? (ele.parentElement.hidden = hideSource) : (ele.hidden = hideSource)); let lastX = downX; let lastY = downY; @@ -343,7 +348,7 @@ export namespace DragManager { } if (e.shiftKey && CollectionDockingView.Instance && dragData.droppedDocuments.length === 1) { AbortDrag(); - finishDrag?.(new DragCompleteEvent(true, dragData)); + finishDrag ?.(new DragCompleteEvent(true, dragData)); CollectionDockingView.Instance.StartOtherDrag({ pageX: e.pageX, pageY: e.pageY, @@ -357,13 +362,13 @@ export namespace DragManager { lastX = e.pageX; lastY = e.pageY; dragElements.map((dragElement, i) => (dragElement.style.transform = - `translate(${(xs[i] += moveX) + (options?.offsetX || 0)}px, ${(ys[i] += moveY) + (options?.offsetY || 0)}px) scale(${scaleXs[i]}, ${scaleYs[i]})`) + `translate(${(xs[i] += moveX) + (options ?.offsetX || 0)}px, ${(ys[i] += moveY) + (options ?.offsetY || 0)}px) scale(${scaleXs[i]}, ${scaleYs[i]})`) ); }; const hideDragShowOriginalElements = () => { dragElements.map(dragElement => dragElement.parentNode === dragDiv && dragDiv.removeChild(dragElement)); - eles.map(ele => ele.parentElement && ele.parentElement?.className === dragData.dragDivName ? (ele.parentElement.hidden = false) : (ele.hidden = false)); + eles.map(ele => ele.parentElement && ele.parentElement ?.className === dragData.dragDivName ? (ele.parentElement.hidden = false) : (ele.hidden = false)); }; const endDrag = () => { document.removeEventListener("pointermove", moveHandler, true); @@ -373,14 +378,14 @@ export namespace DragManager { AbortDrag = () => { hideDragShowOriginalElements(); SelectionManager.SetIsDragging(false); - options?.dragComplete?.(new DragCompleteEvent(true, dragData)); + options ?.dragComplete ?.(new DragCompleteEvent(true, dragData)); endDrag(); }; const upHandler = (e: PointerEvent) => { hideDragShowOriginalElements(); dispatchDrag(eles, e, dragData, options, finishDrag); SelectionManager.SetIsDragging(false); - options?.dragComplete?.(new DragCompleteEvent(false, dragData)); + options ?.dragComplete ?.(new DragCompleteEvent(false, dragData)); endDrag(); }; document.addEventListener("pointermove", moveHandler, true); @@ -401,7 +406,7 @@ export namespace DragManager { }); if (target) { const complete = new DragCompleteEvent(false, dragData); - finishDrag?.(complete); + finishDrag ?.(complete); console.log(complete.aborted); target.dispatchEvent( new CustomEvent<DropEvent>("dashOnDrop", { diff --git a/src/client/util/InteractionUtils.tsx b/src/client/util/InteractionUtils.tsx index 184e37ba5..cf51b8e89 100644 --- a/src/client/util/InteractionUtils.tsx +++ b/src/client/util/InteractionUtils.tsx @@ -1,3 +1,5 @@ +import React = require("react"); + export namespace InteractionUtils { export const MOUSETYPE = "mouse"; export const TOUCHTYPE = "touch"; @@ -90,6 +92,21 @@ export namespace InteractionUtils { return myTouches; } + // TODO: find a way to reference this function from InkingStroke instead of copy pastign here. copied bc of weird error when on mobile view + export function CreatePolyline(points: { X: number, Y: number }[], left: number, top: number, color: string, width: number) { + const pts = points.reduce((acc: string, pt: { X: number, Y: number }) => acc + `${pt.X - left},${pt.Y - top} `, ""); + return ( + <polyline + points={pts} + style={{ + fill: "none", + stroke: color, + strokeWidth: width + }} + /> + ); + } + export function IsType(e: PointerEvent | React.PointerEvent, type: string): boolean { console.log(e.button); switch (type) { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 9a601e07b..460a764e8 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -29,7 +29,9 @@ import { CollectionViewType } from "./collections/CollectionView"; import TouchScrollableMenu, { TouchScrollableMenuItem } from "./TouchScrollableMenu"; import { RadialMenu } from "./nodes/RadialMenu"; import { SelectionManager } from "../util/SelectionManager"; - +import MobileInterface from "../../mobile/MobileInterface"; +import { MobileInkOverlayContent } from "../../server/Message"; +import MobileInkOverlay from "../../mobile/MobileInkOverlay"; @observer export default class GestureOverlay extends Touchable { @@ -53,9 +55,11 @@ export default class GestureOverlay extends Touchable { @observable private _clipboardDoc?: JSX.Element; @observable private _possibilities: JSX.Element[] = []; - @computed private get height(): number { return 2 * Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); } + @computed private get height(): number { return 2 * Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 100, 100); } @computed private get showBounds() { return this.Tool !== ToolglassTools.None; } + @observable private showMobileInkOverlay: boolean = false; + private _d1: Doc | undefined; private _inkToTextDoc: Doc | undefined; private _thumbDoc: Doc | undefined; @@ -141,8 +145,6 @@ export default class GestureOverlay extends Touchable { const nts = this.getNewTouches(te); if (nts.nt.length < 5) { const target = document.elementFromPoint(te.changedTouches.item(0).clientX, te.changedTouches.item(0).clientY); - te.changedTouches.item(0).identifier; - console.log(te.touches); target?.dispatchEvent( new CustomEvent<InteractionUtils.MultiTouchEvent<React.TouchEvent>>("dashOnTouchStart", { @@ -558,6 +560,16 @@ export default class GestureOverlay extends Touchable { const B = this.svgBounds; const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); + if (MobileInterface.Instance && MobileInterface.Instance.drawingInk) { + const { selectedColor, selectedWidth } = InkingControl.Instance; + DocServer.Mobile.dispatchGesturePoints({ + points: this._points, + bounds: B, + color: selectedColor, + width: selectedWidth + }); + } + const initialPoint = this._points[0.]; const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height); const yInGlass = initialPoint.Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - (this.height) && initialPoint.Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER); @@ -717,10 +729,16 @@ export default class GestureOverlay extends Touchable { this._clipboardDoc = undefined; } + @action + enableMobileInkOverlay = (content: MobileInkOverlayContent) => { + this.showMobileInkOverlay = content.enableOverlay; + } + render() { trace(); return ( <div className="gestureOverlay-cont" onPointerDown={this.onPointerDown} onTouchStart={this.onReactTouchStart}> + {this.showMobileInkOverlay ? <MobileInkOverlay /> : <></>} {this.elements} <div className="clipboardDoc-cont" style={{ diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx index f315ce12a..a791eed40 100644 --- a/src/client/views/InkingStroke.tsx +++ b/src/client/views/InkingStroke.tsx @@ -29,13 +29,13 @@ export class InkingStroke extends DocExtendableComponent<FieldViewProps, InkDocu @computed get PanelHeight() { return this.props.PanelHeight(); } private analyzeStrokes = () => { - const data: InkData = Cast(this.Document.data, InkField)?.inkData ?? []; + const data: InkData = Cast(this.Document.data, InkField) ?.inkData ?? []; CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.Document, ["inkAnalysis", "handwriting"], [data]); } render() { TraceMobx(); - const data: InkData = Cast(this.Document.data, InkField)?.inkData ?? []; + const data: InkData = Cast(this.Document.data, InkField) ?.inkData ?? []; const xs = data.map(p => p.X); const ys = data.map(p => p.Y); const left = Math.min(...xs); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 997266266..ff6e79836 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -56,6 +56,7 @@ export class MainView extends React.Component { private _flyoutSizeOnDown = 0; private _urlState: HistoryUtil.DocUrl; private _docBtnRef = React.createRef<HTMLDivElement>(); + private _mainViewRef = React.createRef<HTMLDivElement>(); @observable private _panelWidth: number = 0; @observable private _panelHeight: number = 0; @@ -484,7 +485,7 @@ export class MainView extends React.Component { return new Transform(-translateX, -translateY, 1 / scale); } @computed get docButtons() { - if (CurrentUserUtils.UserDocument?.expandingButtons instanceof Doc) { + if (CurrentUserUtils.UserDocument ?.expandingButtons instanceof Doc) { return <div className="mainView-docButtons" ref={this._docBtnRef} style={{ height: !CurrentUserUtils.UserDocument.expandingButtons.isExpanded ? "42px" : undefined }} > <MainViewNotifs /> @@ -519,8 +520,16 @@ export class MainView extends React.Component { return (null); } + get mainViewElement() { + return document.getElementById("mainView-container"); + } + + get mainViewRef() { + return this._mainViewRef; + } + render() { - return (<div id="mainView-container"> + return (<div id="mainView-container" ref={this._mainViewRef}> <DictationOverlay /> <SharingManager /> <SettingsManager /> diff --git a/src/client/views/collections/CollectionMasonryViewFieldRow.tsx b/src/client/views/collections/CollectionMasonryViewFieldRow.tsx index e25a2f5eb..e84b3b0dd 100644 --- a/src/client/views/collections/CollectionMasonryViewFieldRow.tsx +++ b/src/client/views/collections/CollectionMasonryViewFieldRow.tsx @@ -73,6 +73,7 @@ export class CollectionMasonryViewFieldRow extends React.Component<CMVFieldRowPr @undoBatch rowDrop = action((e: Event, de: DragManager.DropEvent) => { + console.log("masronry row drop"); this._createAliasSelected = false; if (de.complete.docDragData) { (this.props.parent.Document.dropConverter instanceof ScriptField) && diff --git a/src/client/views/collections/CollectionStackingView.tsx b/src/client/views/collections/CollectionStackingView.tsx index 91c7ca76e..7592712e4 100644 --- a/src/client/views/collections/CollectionStackingView.tsx +++ b/src/client/views/collections/CollectionStackingView.tsx @@ -227,6 +227,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { @undoBatch @action drop = (e: Event, de: DragManager.DropEvent) => { + console.log("DROP STACKIN G2"); const where = [de.x, de.y]; let targInd = -1; let plusOne = 0; @@ -257,6 +258,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { @undoBatch @action onDrop = async (e: React.DragEvent): Promise<void> => { + console.log("DROP STACKING"); const where = [e.clientX, e.clientY]; let targInd = -1; this._docXfs.map((cd, i) => { diff --git a/src/client/views/collections/CollectionStackingViewFieldColumn.tsx b/src/client/views/collections/CollectionStackingViewFieldColumn.tsx index 2a9f903bb..c4680fc28 100644 --- a/src/client/views/collections/CollectionStackingViewFieldColumn.tsx +++ b/src/client/views/collections/CollectionStackingViewFieldColumn.tsx @@ -63,6 +63,7 @@ export class CollectionStackingViewFieldColumn extends React.Component<CSVFieldC @undoBatch columnDrop = action((e: Event, de: DragManager.DropEvent) => { + console.log("column drop stacking"); this._createAliasSelected = false; if (de.complete.docDragData) { const key = StrCast(this.props.parent.props.Document.sectionFilter); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 12906ee83..c5028d16e 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -6,7 +6,7 @@ import { Id } from "../../../new_fields/FieldSymbols"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { ScriptField } from "../../../new_fields/ScriptField"; -import { Cast } from "../../../new_fields/Types"; +import { Cast, StrCast } from "../../../new_fields/Types"; import { CurrentUserUtils } from "../../../server/authentication/models/current_user_utils"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; @@ -53,9 +53,9 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) { private _childLayoutDisposer?: IReactionDisposer; protected _mainCont?: HTMLDivElement; protected createDashEventsTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view - this.dropDisposer?.(); - this.gestureDisposer?.(); - this.multiTouchDisposer?.(); + this.dropDisposer ?.(); + this.gestureDisposer ?.(); + this.multiTouchDisposer ?.(); if (ele) { this._mainCont = ele; this.dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this)); @@ -68,7 +68,7 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) { } componentDidMount() { - this._childLayoutDisposer = reaction(() => [this.childDocs, (Cast(this.props.Document.childLayout, Doc) as Doc)?.[Id]], + this._childLayoutDisposer = reaction(() => [this.childDocs, (Cast(this.props.Document.childLayout, Doc) as Doc) ?.[Id]], (args) => { const childLayout = Cast(this.props.Document.childLayout, Doc); if (childLayout instanceof Doc) { @@ -173,7 +173,6 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) { @undoBatch protected onGesture(e: Event, ge: GestureUtils.GestureEvent) { - } @undoBatch @@ -196,7 +195,7 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) { const movedDocs = docDragData.draggedDocuments; added = movedDocs.reduce((added: boolean, d, i) => docDragData.droppedDocuments[i] !== d ? this.props.addDocument(docDragData.droppedDocuments[i]) : - docDragData.moveDocument?.(d, this.props.Document, this.props.addDocument) || added, false); + docDragData.moveDocument ?.(d, this.props.Document, this.props.addDocument) || added, false); } else { added = docDragData.droppedDocuments.reduce((added: boolean, d) => this.props.addDocument(d) || added, false); } @@ -219,6 +218,7 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) { } const html = e.dataTransfer.getData("text/html"); const text = e.dataTransfer.getData("text/plain"); + console.log(html); if (text && text.startsWith("<div")) { return; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index dab0ce08e..ab6ac0eaf 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -145,7 +145,7 @@ export class CollectionView extends Touchable<FieldViewProps> { let index = value.reduce((p, v, i) => (v instanceof Doc && v === doc) ? i : p, -1); index = index !== -1 ? index : value.reduce((p, v, i) => (v instanceof Doc && Doc.AreProtosEqual(v, doc)) ? i : p, -1); - ContextMenu.Instance.clearItems(); + ContextMenu.Instance && ContextMenu.Instance.clearItems(); if (index !== -1) { value.splice(index, 1); return true; diff --git a/src/client/views/collections/CollectionViewChromes.tsx b/src/client/views/collections/CollectionViewChromes.tsx index 073a30330..0811654bf 100644 --- a/src/client/views/collections/CollectionViewChromes.tsx +++ b/src/client/views/collections/CollectionViewChromes.tsx @@ -166,7 +166,7 @@ export class CollectionViewBaseChrome extends React.Component<CollectionViewChro this._viewSpecsOpen = true; //@ts-ignore - if (!e.target?.classList[0]?.startsWith("qs")) { + if (!e.target ?.classList[0] ?.startsWith("qs")) { this.closeDatePicker(); } @@ -315,7 +315,7 @@ export class CollectionViewBaseChrome extends React.Component<CollectionViewChro @action protected drop(e: Event, de: DragManager.DropEvent): boolean { if (de.complete.docDragData && de.complete.docDragData.draggedDocuments.length) { - this._buttonizableCommands.filter(c => c.title === this._currentKey).map(c => c.immediate(de.complete.docDragData?.draggedDocuments || [])); + this._buttonizableCommands.filter(c => c.title === this._currentKey).map(c => c.immediate(de.complete.docDragData ?.draggedDocuments || [])); e.stopPropagation(); } return true; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 4b2ac94c1..2f50fd710 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -86,7 +86,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { @observable private _pullCoords: number[] = [0, 0]; @observable private _pullDirection: string = ""; - public get displayName() { return "CollectionFreeFormView(" + this.props.Document.title?.toString() + ")"; } // this makes mobx trace() statements more descriptive + public get displayName() { return "CollectionFreeFormView(" + this.props.Document.title ?.toString() + ")"; } // this makes mobx trace() statements more descriptive @observable.shallow _layoutElements: ViewDefResult[] = []; // shallow because some layout items (eg pivot labels) are just generated 'divs' and can't be frozen as observables @observable _clusterSets: (Doc[])[] = []; @@ -255,7 +255,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { docs.map(doc => this._clusterSets[doc.cluster = NumCast(docFirst.cluster)].push(doc)); } childLayouts.map(child => !this._clusterSets.some((set, i) => Doc.IndexOf(child, set) !== -1 && child.cluster === i) && this.updateCluster(child)); - childLayouts.map(child => Doc.GetProto(child).clusterStr = child.cluster?.toString()); + childLayouts.map(child => Doc.GetProto(child).clusterStr = child.cluster ?.toString()); } } @@ -434,9 +434,9 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { console.log("end"); if (this._inkToTextStartX && this._inkToTextStartY) { const end = this.getTransform().transformPoint(Math.max(...ge.points.map(p => p.X)), Math.max(...ge.points.map(p => p.Y))); - const setDocs = this.getActiveDocuments().filter(s => s.proto?.type === "text" && s.color); + const setDocs = this.getActiveDocuments().filter(s => s.proto ?.type === "text" && s.color); const sets = setDocs.map((sd) => { - return Cast(sd.data, RichTextField)?.Text as string; + return Cast(sd.data, RichTextField) ?.Text as string; }); if (sets.length && sets[0]) { this._wordPalette.clear(); @@ -523,7 +523,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { @action pan = (e: PointerEvent | React.Touch | { clientX: number, clientY: number }): void => { // I think it makes sense for the marquee menu to go away when panned. -syip2 - MarqueeOptionsMenu.Instance.fadeOut(true); + MarqueeOptionsMenu.Instance && MarqueeOptionsMenu.Instance.fadeOut(true); let x = this.Document._panX || 0; let y = this.Document._panY || 0; @@ -883,8 +883,8 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } getCalculatedPositions(params: { doc: Doc, index: number, collection: Doc, docs: Doc[], state: any }): { x?: number, y?: number, z?: number, width?: number, height?: number, transition?: string, state?: any } { - const result = this.Document.arrangeScript?.script.run(params, console.log); - if (result?.success) { + const result = this.Document.arrangeScript ?.script.run(params, console.log); + if (result ?.success) { return { ...result, transition: "transform 1s" }; } const layoutDoc = Doc.Layout(params.doc); @@ -1114,7 +1114,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } @computed get placeholder() { return <div className="collectionfreeformview-placeholder" style={{ background: this.Document.backgroundColor }}> - <span className="collectionfreeformview-placeholderSpan">{this.props.Document.title?.toString()}</span> + <span className="collectionfreeformview-placeholderSpan">{this.props.Document.title ?.toString()}</span> </div>; } @computed get marqueeView() { diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index a69de60df..9182eb4c0 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -475,6 +475,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu if (!(InteractionUtils.IsType(e, InteractionUtils.MOUSETYPE) || InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) { if (!InteractionUtils.IsType(e, InteractionUtils.PENTYPE)) { e.stopPropagation(); + // TODO: check here for panning/inking } return; } diff --git a/src/client/views/nodes/WebBox.scss b/src/client/views/nodes/WebBox.scss index fbe9bf063..226103a53 100644 --- a/src/client/views/nodes/WebBox.scss +++ b/src/client/views/nodes/WebBox.scss @@ -90,4 +90,19 @@ width: 100%; margin-right: 10px; height: 100%; +} + +.touch-iframe-overlay { + width: 100%; + height: 100%; + position: absolute; + pointer-events: all; + + .indicator { + position: absolute; + + &.active { + background-color: rgba(0, 0, 0, 0.1); + } + } }
\ No newline at end of file diff --git a/src/client/views/nodes/WebBox.tsx b/src/client/views/nodes/WebBox.tsx index a48dc286e..6cae4e878 100644 --- a/src/client/views/nodes/WebBox.tsx +++ b/src/client/views/nodes/WebBox.tsx @@ -21,6 +21,9 @@ import "./WebBox.scss"; import React = require("react"); import { DocAnnotatableComponent } from "../DocComponent"; import { documentSchema } from "../../../new_fields/documentSchemas"; +import { Id } from "../../../new_fields/FieldSymbols"; +import { DragManager } from "../../util/DragManager"; +import { ImageUtils } from "../../util/Import & Export/ImageUtils"; library.add(faStickyNote); @@ -34,6 +37,13 @@ export class WebBox extends DocAnnotatableComponent<FieldViewProps, WebDocument> @observable private collapsed: boolean = true; @observable private url: string = ""; + private _longPressSecondsHack?: NodeJS.Timeout; + private _iframeRef = React.createRef<HTMLIFrameElement>(); + private _iframeIndicatorRef = React.createRef<HTMLDivElement>(); + private _iframeDragRef = React.createRef<HTMLDivElement>(); + @observable private _pressX: number = 0; + @observable private _pressY: number = 0; + componentDidMount() { const field = Cast(this.props.Document[this.props.fieldKey], WebField); @@ -49,6 +59,14 @@ export class WebBox extends DocAnnotatableComponent<FieldViewProps, WebDocument> } this.setURL(); + + document.addEventListener("pointerup", this.onLongPressUp); + document.addEventListener("pointermove", this.onLongPressMove); + } + + componentWillUnmount() { + document.removeEventListener("pointerup", this.onLongPressUp); + document.removeEventListener("pointermove", this.onLongPressMove); } @action @@ -164,6 +182,86 @@ export class WebBox extends DocAnnotatableComponent<FieldViewProps, WebDocument> } } + onLongPressDown = (e: React.PointerEvent) => { + this._pressX = e.clientX; + this._pressY = e.clientY; + + // find the pressed element in the iframe (currently only works if its an img) + let pressedElement: HTMLElement | undefined; + let pressedBound: ClientRect | undefined; + if (this._iframeRef.current) { + const B = this._iframeRef.current.getBoundingClientRect(); + const iframeDoc = this._iframeRef.current.contentDocument; + if (B && iframeDoc) { + // check if there is selected text + const selectedText = iframeDoc.getSelection(); + if (selectedText && selectedText.toString.length > -1) { + + } + console.log("selectedText", selectedText ? selectedText.toString() : ""); + + // TODO: this only works when scale = 1 as it is currently only inteded for mobile upload + const element = iframeDoc.elementFromPoint(this._pressX - B.left, this._pressY - B.top); + console.log("found element", element, element && element.nodeName); + if (element && element.nodeName) {//} === "IMG") { + pressedBound = element.getBoundingClientRect(); + pressedElement = element.cloneNode(true) as HTMLElement; + } + } + } + + // mark the pressed element + if (pressedElement && pressedBound) { + console.log("clones b", pressedElement.getBoundingClientRect(), pressedBound); + if (this._iframeIndicatorRef.current) { + this._iframeIndicatorRef.current.style.top = pressedBound.top + "px"; + this._iframeIndicatorRef.current.style.left = pressedBound.left + "px"; + this._iframeIndicatorRef.current.style.width = pressedBound.width + "px"; + this._iframeIndicatorRef.current.style.height = pressedBound.height + "px"; + this._iframeIndicatorRef.current.classList.add("active"); + } + } + + // start dragging the pressed element if long pressed + this._longPressSecondsHack = setTimeout(() => { + if (pressedElement && pressedBound) { + e.stopPropagation(); + e.preventDefault(); + if (pressedElement.nodeName === "IMG") { + const src = pressedElement.getAttribute("src"); // TODO: may not always work + if (src) { + const doc = Docs.Create.ImageDocument(src, { _width: 300 }); + ImageUtils.ExtractExif(doc); + + // add clone to div so that dragging ghost is placed properly + if (this._iframeDragRef.current) this._iframeDragRef.current.appendChild(pressedElement); + + const dragData = new DragManager.DocumentDragData([doc]); + DragManager.StartDocumentDrag([pressedElement], dragData, this._pressX, this._pressY, { hideSource: true }); + } + } + } + }, 1500); + } + + onLongPressMove = (e: PointerEvent) => { + // this._pressX = e.clientX; + // this._pressY = e.clientY; + } + + onLongPressUp = (e: PointerEvent) => { + if (this._longPressSecondsHack) { + clearTimeout(this._longPressSecondsHack); + } + if (this._iframeIndicatorRef.current) { + this._iframeIndicatorRef.current.classList.remove("active"); + } + if (this._iframeDragRef.current) { + while (this._iframeDragRef.current.firstChild) this._iframeDragRef.current.removeChild(this._iframeDragRef.current.firstChild); + } + } + + @computed get content() { const field = this.dataDoc[this.props.fieldKey]; @@ -171,9 +269,9 @@ export class WebBox extends DocAnnotatableComponent<FieldViewProps, WebDocument> if (field instanceof HtmlField) { view = <span id="webBox-htmlSpan" dangerouslySetInnerHTML={{ __html: field.html }} />; } else if (field instanceof WebField) { - view = <iframe src={Utils.CorsProxy(field.url.href)} style={{ position: "absolute", width: "100%", height: "100%", top: 0 }} />; + view = <iframe ref={this._iframeRef} src={Utils.CorsProxy(field.url.href)} style={{ position: "absolute", width: "100%", height: "100%", top: 0 }} />; } else { - view = <iframe src={"https://crossorigin.me/https://cs.brown.edu"} style={{ position: "absolute", width: "100%", height: "100%", top: 0 }} />; + view = <iframe ref={this._iframeRef} src={"https://crossorigin.me/https://cs.brown.edu"} style={{ position: "absolute", width: "100%", height: "100%", top: 0 }} />; } const content = <div style={{ width: "100%", height: "100%", position: "absolute" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}> @@ -181,15 +279,23 @@ export class WebBox extends DocAnnotatableComponent<FieldViewProps, WebDocument> {view} </div>; - const frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; + const decInteracting = DocumentDecorations.Instance && DocumentDecorations.Instance.Interacting; + + const frozen = !this.props.isSelected() || decInteracting; - const classname = "webBox-cont" + (this.props.isSelected() && InkingControl.Instance.selectedTool === InkTool.None && !DocumentDecorations.Instance.Interacting ? "-interactive" : ""); + const classname = "webBox-cont" + (this.props.isSelected() && InkingControl.Instance.selectedTool === InkTool.None && !decInteracting ? "-interactive" : ""); return ( <> <div className={classname} > {content} </div> - {!frozen ? (null) : <div className="webBox-overlay" onWheel={this.onPreWheel} onPointerDown={this.onPrePointer} onPointerMove={this.onPrePointer} onPointerUp={this.onPrePointer} />} + {!frozen ? (null) : + <div className="webBox-overlay" onWheel={this.onPreWheel} onPointerDown={this.onPrePointer} onPointerMove={this.onPrePointer} onPointerUp={this.onPrePointer}> + <div className="touch-iframe-overlay" onPointerDown={this.onLongPressDown} > + <div className="indicator" ref={this._iframeIndicatorRef}></div> + <div className="dragger" ref={this._iframeDragRef}></div> + </div> + </div>} </>); } render() { diff --git a/src/mobile/ImageUpload.tsx b/src/mobile/ImageUpload.tsx index 1583e3d5d..5903a2ce9 100644 --- a/src/mobile/ImageUpload.tsx +++ b/src/mobile/ImageUpload.tsx @@ -13,6 +13,7 @@ import { observable } from 'mobx'; import { Utils } from '../Utils'; import MobileInterface from './MobileInterface'; import { CurrentUserUtils } from '../server/authentication/models/current_user_utils'; +import { Scripting } from '../client/util/Scripting'; @@ -27,12 +28,11 @@ const inputRef = React.createRef<HTMLInputElement>(); @observer class Uploader extends React.Component { - @observable - error: string = ""; - @observable - status: string = ""; + @observable error: string = ""; + @observable status: string = ""; onClick = async () => { + console.log("uploader click"); try { this.status = "initializing protos"; await Docs.Prototypes.initialize(); @@ -47,6 +47,7 @@ class Uploader extends React.Component { const upload = window.location.origin + "/uploadFormData"; this.status = "uploading image"; + console.log("uploading image", formData); const res = await fetch(upload, { method: 'POST', body: formData diff --git a/src/mobile/MobileInkOverlay.scss b/src/mobile/MobileInkOverlay.scss new file mode 100644 index 000000000..b9c1fb146 --- /dev/null +++ b/src/mobile/MobileInkOverlay.scss @@ -0,0 +1,39 @@ +.mobileInkOverlay { + border: 10px dashed red; + background-color: rgba(0, 0, 0, .05); +} + +.mobileInkOverlay-border { + // background-color: rgba(0, 255, 0, .4); + position: absolute; + pointer-events: auto; + cursor: pointer; + + &.top { + width: calc(100% + 20px); + height: 10px; + top: -10px; + left: -10px; + } + + &.left { + width: 10px; + height: calc(100% + 20px); + top: -10px; + left: -10px; + } + + &.right { + width: 10px; + height: calc(100% + 20px); + top: -10px; + right: -10px; + } + + &.bottom { + width: calc(100% + 20px); + height: 10px; + bottom: -10px; + left: -10px; + } +}
\ No newline at end of file diff --git a/src/mobile/MobileInkOverlay.tsx b/src/mobile/MobileInkOverlay.tsx new file mode 100644 index 000000000..4dde3a075 --- /dev/null +++ b/src/mobile/MobileInkOverlay.tsx @@ -0,0 +1,190 @@ +import React = require('react'); +import { observer } from "mobx-react"; +import { MobileInkOverlayContent, GestureContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent } from "../server/Message"; +import { observable, action } from "mobx"; +import { GestureUtils } from "../pen-gestures/GestureUtils"; +import "./MobileInkOverlay.scss"; +import { StrCast, Cast } from '../new_fields/Types'; +import { DragManager } from "../client/util/DragManager"; +import { DocServer } from '../client/DocServer'; +import { Doc, DocListCastAsync } from '../new_fields/Doc'; +import { listSpec } from '../new_fields/Schema'; + + +@observer +export default class MobileInkOverlay extends React.Component { + public static Instance: MobileInkOverlay; + + @observable private _scale: number = 1; + @observable private _width: number = 0; + @observable private _height: number = 0; + @observable private _x: number = -300; + @observable private _y: number = -300; + @observable private _text: string = ""; + + @observable private _offsetX: number = 0; + @observable private _offsetY: number = 0; + @observable private _isDragging: boolean = false; + private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); + + constructor(props: Readonly<{}>) { + super(props); + MobileInkOverlay.Instance = this; + } + + initialSize(mobileWidth: number, mobileHeight: number) { + const maxWidth = window.innerWidth - 30; + const maxHeight = window.innerHeight - 30; // -30 for padding + if (mobileWidth > maxWidth || mobileHeight > maxHeight) { + const scale = Math.min(maxWidth / mobileWidth, maxHeight / mobileHeight); + return { width: mobileWidth * scale, height: mobileHeight * scale, scale: scale }; + } + return { width: mobileWidth, height: mobileHeight, scale: 1 }; + } + + @action + initMobileInkOverlay(content: MobileInkOverlayContent) { + const { width, height, text } = content; + const scaledSize = this.initialSize(width ? width : 0, height ? height : 0); + this._width = scaledSize.width; + this._height = scaledSize.height; + this._scale = scaledSize.scale; + this._x = 300; // TODO: center on screen + this._y = 25; // TODO: center on screen + this._text = text ? text : ""; + } + + @action + updatePosition(content: UpdateMobileInkOverlayPositionContent) { + const { dx, dy, dsize } = content; + if (dx) this._x += dx; + if (dy) this._y += dy; + // TODO: scale dsize + } + + drawStroke = (content: GestureContent) => { + // TODO: figure out why strokes drawn in corner of mobile interface dont get inserted + + const { points, bounds } = content; + console.log("received points", points, bounds); + + const B = { + right: (bounds.right * this._scale) + this._x, + left: (bounds.left * this._scale) + this._x, // TODO: scale + bottom: (bounds.bottom * this._scale) + this._y, + top: (bounds.top * this._scale) + this._y, // TODO: scale + width: bounds.width * this._scale, + height: bounds.height * this._scale, + }; + + const target = document.elementFromPoint(this._x + 10, this._y + 10); + target?.dispatchEvent( + new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture", + { + bubbles: true, + detail: { + points: points, + gesture: GestureUtils.Gestures.Stroke, + bounds: B + } + } + ) + ); + } + + uploadDocument = async (content: MobileDocumentUploadContent) => { + const { docId } = content; + const doc = await DocServer.GetRefField(docId); + + if (doc && doc instanceof Doc) { + const target = document.elementFromPoint(this._x + 10, this._y + 10); + const dragData = new DragManager.DocumentDragData([doc]); + const complete = new DragManager.DragCompleteEvent(false, dragData); + + if (target) { + target.dispatchEvent( + new CustomEvent<DragManager.DropEvent>("dashOnDrop", + { + bubbles: true, + detail: { + x: this._x, + y: this._y, + complete: complete, + altKey: false, + metaKey: false, + ctrlKey: false, + shiftKey: false + } + } + ) + ); + } else { + alert("TARGET IS UNDEFINED"); + } + } + } + + @action + dragStart = (e: React.PointerEvent) => { + document.removeEventListener("pointermove", this.dragging); + document.removeEventListener("pointerup", this.dragEnd); + document.addEventListener("pointermove", this.dragging); + document.addEventListener("pointerup", this.dragEnd); + + this._isDragging = true; + this._offsetX = e.pageX - this._mainCont.current!.getBoundingClientRect().left; + this._offsetY = e.pageY - this._mainCont.current!.getBoundingClientRect().top; + + e.preventDefault(); + e.stopPropagation(); + } + + @action + dragging = (e: PointerEvent) => { + const x = e.pageX - this._offsetX; + const y = e.pageY - this._offsetY; + + // TODO: don't allow drag over library? + this._x = Math.min(Math.max(x, 0), window.innerWidth - this._width); + this._y = Math.min(Math.max(y, 0), window.innerHeight - this._height); + + e.preventDefault(); + e.stopPropagation(); + } + + @action + dragEnd = (e: PointerEvent) => { + document.removeEventListener("pointermove", this.dragging); + document.removeEventListener("pointerup", this.dragEnd); + + this._isDragging = false; + + e.preventDefault(); + e.stopPropagation(); + } + + render() { + + return ( + <div className="mobileInkOverlay" + style={{ + width: this._width, + height: this._height, + position: "absolute", + transform: `translate(${this._x}px, ${this._y}px)`, + zIndex: 30000, + pointerEvents: "none", + borderStyle: this._isDragging ? "solid" : "dashed", + } + } + ref={this._mainCont} + > + <p>{this._text}</p> + <div className="mobileInkOverlay-border top" onPointerDown={this.dragStart}></div> + <div className="mobileInkOverlay-border bottom" onPointerDown={this.dragStart}></div> + <div className="mobileInkOverlay-border left" onPointerDown={this.dragStart}></div> + <div className="mobileInkOverlay-border right" onPointerDown={this.dragStart}></div> + </div > + ); + } +}
\ No newline at end of file diff --git a/src/mobile/MobileInterface.scss b/src/mobile/MobileInterface.scss new file mode 100644 index 000000000..4d86e208f --- /dev/null +++ b/src/mobile/MobileInterface.scss @@ -0,0 +1,19 @@ +.mobileInterface-inkInterfaceButtons { + position: absolute; + top: 0px; + display: flex; + justify-content: space-between; + width: 100%; + z-index: 9999; + height: 50px; + + .mobileInterface-button { + height: 100%; + } +} + +.mobileInterface-container { + height: 100%; + position: relative; + touch-action: none; +}
\ No newline at end of file diff --git a/src/mobile/MobileInterface.tsx b/src/mobile/MobileInterface.tsx index b1eaeaa0a..83410b99d 100644 --- a/src/mobile/MobileInterface.tsx +++ b/src/mobile/MobileInterface.tsx @@ -1,40 +1,123 @@ import React = require('react'); import { observer } from 'mobx-react'; -import { computed, action } from 'mobx'; +import { computed, action, observable } from 'mobx'; import { CurrentUserUtils } from '../server/authentication/models/current_user_utils'; -import { FieldValue, Cast } from '../new_fields/Types'; -import { Doc } from '../new_fields/Doc'; +import { FieldValue, Cast, StrCast } from '../new_fields/Types'; +import { Doc, DocListCast } from '../new_fields/Doc'; import { Docs } from '../client/documents/Documents'; import { CollectionView } from '../client/views/collections/CollectionView'; import { DocumentView } from '../client/views/nodes/DocumentView'; import { emptyPath, emptyFunction, returnFalse, returnOne, returnEmptyString, returnTrue } from '../Utils'; import { Transform } from '../client/util/Transform'; import { library } from '@fortawesome/fontawesome-svg-core'; -import { faPenNib, faHighlighter, faEraser, faMousePointer } from '@fortawesome/free-solid-svg-icons'; +import { faPenNib, faHighlighter, faEraser, faMousePointer, faBreadSlice, faTrash, faCheck, faLongArrowAltLeft } from '@fortawesome/free-solid-svg-icons'; +import { Scripting } from '../client/util/Scripting'; +import { CollectionFreeFormView } from '../client/views/collections/collectionFreeForm/CollectionFreeFormView'; +import GestureOverlay from '../client/views/GestureOverlay'; +import { InkingControl } from '../client/views/InkingControl'; +import { InkTool } from '../new_fields/InkField'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import "./MobileInterface.scss"; +import { SelectionManager } from '../client/util/SelectionManager'; +import { DateField } from '../new_fields/DateField'; +import { GestureUtils } from '../pen-gestures/GestureUtils'; +import { DocServer } from '../client/DocServer'; +import { DocumentDecorations } from '../client/views/DocumentDecorations'; +import { OverlayView } from '../client/views/OverlayView'; +import { DictationOverlay } from '../client/views/DictationOverlay'; +import SharingManager from '../client/util/SharingManager'; +import { PreviewCursor } from '../client/views/PreviewCursor'; +import { ContextMenu } from '../client/views/ContextMenu'; +import { RadialMenu } from '../client/views/nodes/RadialMenu'; +import PDFMenu from '../client/views/pdf/PDFMenu'; +import MarqueeOptionsMenu from '../client/views/collections/collectionFreeForm/MarqueeOptionsMenu'; +import GoogleAuthenticationManager from '../client/apis/GoogleAuthenticationManager'; +import { listSpec } from '../new_fields/Schema'; +import { Id } from '../new_fields/FieldSymbols'; +import { DocumentManager } from '../client/util/DocumentManager'; + +library.add(faLongArrowAltLeft); @observer export default class MobileInterface extends React.Component { + @observable static Instance: MobileInterface; @computed private get userDoc() { return CurrentUserUtils.UserDocument; } @computed private get mainContainer() { return this.userDoc ? FieldValue(Cast(this.userDoc.activeMobile, Doc)) : CurrentUserUtils.GuestMobile; } + // @observable private currentView: "main" | "ink" | "upload" = "main"; + private mainDoc: any = CurrentUserUtils.setupMobileDoc(this.userDoc); + @observable private renderView?: () => JSX.Element; + + // private inkDoc?: Doc; + public drawingInk: boolean = false; + + // private uploadDoc?: Doc; + + constructor(props: Readonly<{}>) { + super(props); + MobileInterface.Instance = this; + } @action componentDidMount = () => { library.add(...[faPenNib, faHighlighter, faEraser, faMousePointer]); if (this.userDoc && !this.mainContainer) { - const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); - this.userDoc.activeMobile = doc; + this.userDoc.activeMobile = this.mainDoc; + } + } + + @action + switchCurrentView = (doc: (userDoc: Doc) => Doc, renderView?: () => JSX.Element, onSwitch?: () => void) => { + if (!this.userDoc) return; + + this.userDoc.activeMobile = doc(this.userDoc); + onSwitch && onSwitch(); + + this.renderView = renderView; + } + + onSwitchInking = () => { + InkingControl.Instance.switchTool(InkTool.Pen); + MobileInterface.Instance.drawingInk = true; + + DocServer.Mobile.dispatchOverlayTrigger({ + enableOverlay: true, + width: window.innerWidth, + height: window.innerHeight + }); + } + + onSwitchUpload = async () => { + let width = 300; + let height = 300; + + // get width and height of the collection doc + if (this.mainContainer) { + const data = Cast(this.mainContainer.data, listSpec(Doc)); + if (data) { + const collectionDoc = await data[1]; // this should be the collection doc since the positions should be locked + const docView = DocumentManager.Instance.getDocumentView(collectionDoc); + if (docView) { + width = docView.nativeWidth ? docView.nativeWidth : 300; + height = docView.nativeHeight ? docView.nativeHeight : 300; + } + } } + DocServer.Mobile.dispatchOverlayTrigger({ + enableOverlay: true, + width: width, + height: height, + text: "Documents uploaded from mobile will show here", + }); } - @computed - get mainContent() { + renderDefaultContent = () => { if (this.mainContainer) { return <DocumentView Document={this.mainContainer} DataDoc={undefined} LibraryPath={emptyPath} - addDocument={undefined} + addDocument={returnFalse} addDocTab={returnFalse} pinToPres={emptyFunction} removeDocument={undefined} @@ -58,11 +141,178 @@ export default class MobileInterface extends React.Component { return "hello"; } + onBack = (e: React.MouseEvent) => { + this.switchCurrentView((userDoc: Doc) => this.mainDoc); + InkingControl.Instance.switchTool(InkTool.None); // TODO: switch to previous tool + + DocServer.Mobile.dispatchOverlayTrigger({ + enableOverlay: false, + width: window.innerWidth, + height: window.innerHeight + }); + + // this.inkDoc = undefined; + this.drawingInk = false; + } + + shiftLeft = (e: React.MouseEvent) => { + DocServer.Mobile.dispatchOverlayPositionUpdate({ + dx: -10 + }); + e.preventDefault(); + e.stopPropagation(); + } + + shiftRight = (e: React.MouseEvent) => { + DocServer.Mobile.dispatchOverlayPositionUpdate({ + dx: 10 + }); + e.preventDefault(); + e.stopPropagation(); + } + + renderInkingContent = () => { + console.log("rendering inking content"); + // TODO: support panning and zooming + // TODO: handle moving of ink strokes + if (this.mainContainer) { + return ( + <div className="mobileInterface"> + <div className="mobileInterface-inkInterfaceButtons"> + <div className="navButtons"> + <button className="mobileInterface-button cancel" onClick={this.onBack} title="Cancel drawing">BACK</button> + </div> + <div className="inkSettingButtons"> + <button className="mobileInterface-button cancel" onClick={this.onBack} title="Cancel drawing"><FontAwesomeIcon icon="long-arrow-alt-left" /></button> + </div> + <div className="navButtons"> + <button className="mobileInterface-button" onClick={this.shiftLeft} title="Shift left">left</button> + <button className="mobileInterface-button" onClick={this.shiftRight} title="Shift right">right</button> + </div> + </div> + <CollectionView + Document={this.mainContainer} + DataDoc={undefined} + LibraryPath={emptyPath} + fieldKey={""} + addDocTab={returnFalse} + pinToPres={emptyFunction} + PanelHeight={() => window.innerHeight} + PanelWidth={() => window.innerWidth} + focus={emptyFunction} + isSelected={returnFalse} + select={emptyFunction} + active={returnFalse} + ContentScaling={returnOne} + whenActiveChanged={returnFalse} + ScreenToLocalTransform={Transform.Identity} + renderDepth={0} + ContainingCollectionView={undefined} + ContainingCollectionDoc={undefined}> + </CollectionView> + </div> + ); + } + } + + upload = async (e: React.MouseEvent) => { + if (this.mainContainer) { + const data = Cast(this.mainContainer.data, listSpec(Doc)); + if (data) { + const collectionDoc = await data[1]; // this should be the collection doc since the positions should be locked + const children = DocListCast(collectionDoc.data); + const uploadDoc = children.length === 1 ? children[0] : Docs.Create.StackingDocument(children, { + title: "Mobile Upload Collection", backgroundColor: "white", lockedPosition: true, _width: 300, _height: 300 + }); + if (uploadDoc) { + DocServer.Mobile.dispatchMobileDocumentUpload({ + docId: uploadDoc[Id], + }); + } + } + } + e.stopPropagation(); + e.preventDefault(); + } + + renderUploadContent() { + if (this.mainContainer) { + return ( + <div className="mobileInterface" onDragOver={this.onDragOver}> + <div className="mobileInterface-inkInterfaceButtons"> + <div className="navButtons"> + <button className="mobileInterface-button cancel" onClick={this.onBack} title="Back">BACK</button> + </div> + <div className="uploadSettings"> + <button className="mobileInterface-button" onClick={this.upload} title="Upload">UPLOAD</button> + </div> + </div> + <DocumentView + Document={this.mainContainer} + DataDoc={undefined} + LibraryPath={emptyPath} + addDocument={returnFalse} + addDocTab={returnFalse} + pinToPres={emptyFunction} + removeDocument={undefined} + onClick={undefined} + ScreenToLocalTransform={Transform.Identity} + ContentScaling={returnOne} + PanelWidth={() => window.screen.width} + PanelHeight={() => window.screen.height} + renderDepth={0} + focus={emptyFunction} + backgroundColor={returnEmptyString} + parentActive={returnTrue} + whenActiveChanged={emptyFunction} + bringToFront={emptyFunction} + ContainingCollectionView={undefined} + ContainingCollectionDoc={undefined} + zoomToScale={emptyFunction} + getScale={returnOne}> + </DocumentView> + </div> + ); + } + } + + onDragOver = (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + } + render() { + // const content = this.currentView === "main" ? this.mainContent : + // this.currentView === "ink" ? this.inkContent : + // this.currentView === "upload" ? this.uploadContent : <></>; return ( - <div className="mobile-container"> - {this.mainContent} + <div className="mobileInterface-container" onDragOver={this.onDragOver}> + {/* <DocumentDecorations /> + <GestureOverlay> + {this.renderView ? this.renderView() : this.renderDefaultContent()} + </GestureOverlay> */} + + {/* <DictationOverlay /> + <SharingManager /> + <GoogleAuthenticationManager /> */} + <DocumentDecorations /> + <GestureOverlay> + {this.renderView ? this.renderView() : this.renderDefaultContent()} + </GestureOverlay> + <PreviewCursor /> + {/* <ContextMenu /> */} + <RadialMenu /> + {/* <PDFMenu /> + <MarqueeOptionsMenu /> + <OverlayView /> */} </div> ); } -}
\ No newline at end of file +} + +Scripting.addGlobal(function switchMobileView(doc: (userDoc: Doc) => Doc, renderView?: () => JSX.Element, onSwitch?: () => void) { return MobileInterface.Instance.switchCurrentView(doc, renderView, onSwitch); }); +Scripting.addGlobal(function onSwitchMobileInking() { return MobileInterface.Instance.onSwitchInking(); }); +Scripting.addGlobal(function renderMobileInking() { return MobileInterface.Instance.renderInkingContent(); }); +Scripting.addGlobal(function onSwitchMobileUpload() { return MobileInterface.Instance.onSwitchUpload(); }); +Scripting.addGlobal(function renderMobileUpload() { return MobileInterface.Instance.renderUploadContent(); }); + diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index a92b613b7..e18b6826e 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -42,6 +42,7 @@ export default class UploadManager extends ApiManager { method: Method.POST, subscription: "/uploadFormData", secureHandler: async ({ req, res }) => { + console.log("/upload register"); const form = new formidable.IncomingForm(); form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; diff --git a/src/server/DashSession/DashSessionAgent.ts b/src/server/DashSession/DashSessionAgent.ts index c55e01243..c74b50555 100644 --- a/src/server/DashSession/DashSessionAgent.ts +++ b/src/server/DashSession/DashSessionAgent.ts @@ -25,15 +25,18 @@ export class DashSessionAgent extends AppliedSessionAgent { * The core method invoked when the single master thread is initialized. * Installs event hooks, repl commands and additional IPC listeners. */ - protected async initializeMonitor(monitor: Monitor, sessionKey: string): Promise<void> { - await this.dispatchSessionPassword(sessionKey); - monitor.addReplCommand("pull", [], () => monitor.exec("git pull")); - monitor.addReplCommand("solr", [/start|stop|index/], this.executeSolrCommand); - monitor.addReplCommand("backup", [], this.backup); - monitor.addReplCommand("debug", [/\S+\@\S+/], async ([to]) => this.dispatchZippedDebugBackup(to)); - monitor.on("backup", this.backup); - monitor.on("debug", async ({ to }) => this.dispatchZippedDebugBackup(to)); - monitor.coreHooks.onCrashDetected(this.dispatchCrashReport); + // protected async initializeMonitor(monitor: Monitor, sessionKey: string): Promise<void> { + protected async initializeMonitor(monitor: Monitor): Promise<void> { + + // await this.dispatchSessionPassword(sessionKey); + // monitor.addReplCommand("pull", [], () => monitor.exec("git pull")); + // monitor.addReplCommand("solr", [/start|stop|index/], this.executeSolrCommand); + // monitor.addReplCommand("backup", [], this.backup); + // monitor.addReplCommand("debug", [/\S+\@\S+/], async ([to]) => this.dispatchZippedDebugBackup(to)); + // monitor.on("backup", this.backup); + // monitor.on("debug", async ({ to }) => this.dispatchZippedDebugBackup(to)); + // monitor.coreHooks.onCrashDetected(this.dispatchCrashReport); + return; } /** diff --git a/src/server/Message.ts b/src/server/Message.ts index 22d2fa8a8..02ca2ceda 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -1,4 +1,6 @@ import { Utils } from "../Utils"; +import { Point } from "../pen-gestures/ndollar"; +import { Doc } from "../new_fields/Doc"; import { Image } from "canvas"; export class Message<T> { @@ -43,6 +45,30 @@ export interface Diff extends Reference { readonly diff: any; } +export interface GestureContent { + readonly points: Array<Point>; + readonly bounds: { right: number, left: number, bottom: number, top: number, width: number, height: number }; + readonly width?: string; + readonly color?: string; +} + +export interface MobileInkOverlayContent { + readonly enableOverlay: boolean; + readonly width?: number; + readonly height?: number; + readonly text?: string; +} + +export interface UpdateMobileInkOverlayPositionContent { + readonly dx?: number; + readonly dy?: number; + readonly dsize?: number; +} + +export interface MobileDocumentUploadContent { + readonly docId: string; +} + export namespace MessageStore { export const Foo = new Message<string>("Foo"); export const Bar = new Message<string>("Bar"); @@ -53,6 +79,11 @@ export namespace MessageStore { export const DeleteAll = new Message<any>("Delete All"); export const ConnectionTerminated = new Message<string>("Connection Terminated"); + export const GesturePoints = new Message<GestureContent>("Gesture Points"); + export const MobileInkOverlayTrigger = new Message<MobileInkOverlayContent>("Trigger Mobile Ink Overlay"); + export const UpdateMobileInkOverlayPosition = new Message<UpdateMobileInkOverlayPositionContent>("Update Mobile Ink Overlay Position"); + export const MobileDocumentUpload = new Message<MobileDocumentUploadContent>("Upload Document From Mobile"); + export const GetRefField = new Message<string>("Get Ref Field"); export const GetRefFields = new Message<string[]>("Get Ref Fields"); export const UpdateField = new Message<Diff>("Update Ref Field"); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index f485e1dcd..66f7019a4 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -1,5 +1,5 @@ import { Utils } from "../../Utils"; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes } from "../Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent } from "../Message"; import { Client } from "../Client"; import { Socket } from "socket.io"; import { Database } from "../database"; @@ -65,6 +65,10 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); + Utils.AddServerHandler(socket, MessageStore.GesturePoints, content => processGesturePoints(socket, content)); + Utils.AddServerHandler(socket, MessageStore.MobileInkOverlayTrigger, content => processOverlayTrigger(socket, content)); + Utils.AddServerHandler(socket, MessageStore.UpdateMobileInkOverlayPosition, content => processUpdateOverlayPosition(socket, content)); + Utils.AddServerHandler(socket, MessageStore.MobileDocumentUpload, content => processMobileDocumentUpload(socket, content)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); @@ -79,6 +83,22 @@ export namespace WebSocket { logPort("websocket", socketPort); } + function processGesturePoints(socket: Socket, content: GestureContent) { + socket.broadcast.emit("receiveGesturePoints", content); + } + + function processOverlayTrigger(socket: Socket, content: MobileInkOverlayContent) { + socket.broadcast.emit("receiveOverlayTrigger", content); + } + + function processUpdateOverlayPosition(socket: Socket, content: UpdateMobileInkOverlayPositionContent) { + socket.broadcast.emit("receiveUpdateOverlayPosition", content); + } + + function processMobileDocumentUpload(socket: Socket, content: MobileDocumentUploadContent) { + socket.broadcast.emit("receiveMobileDocumentUpload", content); + } + async function RecognizeImage([query, callback]: [string, (result: any) => any]) { const path = serverPathToFile(Directory.images, "handwriting.jpg"); imageDataUri.outputFile(query, path).then((savedName: string) => { diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index b0ea2f9ad..6c916689a 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -14,6 +14,7 @@ import { Utils } from "../../../Utils"; import { nullAudio } from "../../../new_fields/URLField"; import { DragManager } from "../../../client/util/DragManager"; import { InkingControl } from "../../../client/views/InkingControl"; +import { Scripting } from "../../../client/util/Scripting"; import { CollectionViewType } from "../../../client/views/collections/CollectionView"; export class CurrentUserUtils { @@ -102,6 +103,9 @@ export class CurrentUserUtils { { title: "use eraser", icon: "eraser", click: 'activateEraser(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "pink", activePen: doc }, { title: "use scrubber", icon: "eraser", click: 'activateScrubber(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "green", activePen: doc }, { title: "use drag", icon: "mouse-pointer", click: 'deactivateInk();this.activePen.pen = this;', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "white", activePen: doc }, + // { title: "draw", icon: "pen-nib", click: 'switchMobileView(setupMobileInkingDoc, renderMobileInking, onSwitchMobileInking);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "red", activePen: doc }, + { title: "upload", icon: "upload", click: 'switchMobileView(setupMobileUploadDoc, renderMobileUpload, onSwitchMobileUpload);', backgroundColor: "orange" }, + // { title: "upload", icon: "upload", click: 'uploadImageMobile();', backgroundColor: "cyan" }, ]; return docProtoData.filter(d => !buttons || !buttons.includes(d.title)).map(data => Docs.Create.FontIconDocument({ _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, _dropAction: data.click ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick, @@ -146,6 +150,22 @@ export class CurrentUserUtils { }); } + static setupMobileInkingDoc(userDoc: Doc) { + return Docs.Create.FreeformDocument([], { title: "Mobile Inking", backgroundColor: "white" }); + } + + static setupMobileUploadDoc(userDoc: Doc) { + const webDoc = Docs.Create.WebDocument("https://www.britannica.com/animal/cat", { + title: "Upload Images From the Web", _chromeStatus: "enabled", lockedPosition: true + }); + const uploadDoc = Docs.Create.StackingDocument([], { + title: "Mobile Upload Collection", backgroundColor: "white", lockedPosition: true + }); + return Docs.Create.StackingDocument([webDoc, uploadDoc], { + _width: screen.width, lockedPosition: true, _chromeStatus: "disabled", title: "Upload", _autoHeight: true, _yMargin: 80, backgroundColor: "lightgray" + }); + } + // setup the Creator button which will display the creator panel. This panel will include the drag creators and the color picker. when clicked, this panel will be displayed in the target container (ie, sidebarContainer) static setupToolsPanel(sidebarContainer: Doc, doc: Doc) { // setup a masonry view of all he creators @@ -383,3 +403,6 @@ export class CurrentUserUtils { return recurs([] as Attribute[], schema ? schema.rootAttributeGroup : undefined); } } + +Scripting.addGlobal(function setupMobileInkingDoc(userDoc: Doc) { return CurrentUserUtils.setupMobileInkingDoc(userDoc); }); +Scripting.addGlobal(function setupMobileUploadDoc(userDoc: Doc) { return CurrentUserUtils.setupMobileUploadDoc(userDoc); }); diff --git a/src/server/index.ts b/src/server/index.ts index 313a2f0e2..55ba71dba 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -141,7 +141,7 @@ export async function launchServer() { * So, the 'else' clause is exactly what we've always run when executing npm start. */ if (process.env.RELEASE) { - (sessionAgent = new DashSessionAgent()).launch(); + // (sessionAgent = new DashSessionAgent()).launch(); } else { launchServer(); } diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index 9f67c1dda..bb661a124 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -42,18 +42,18 @@ export default async function InitializeServer(routeSetter: RouteSetter) { } }; app.use(cors(corsOptions)); - app.use("*", ({ user, originalUrl }, res, next) => { - if (user && !originalUrl.includes("Heartbeat")) { - const userEmail = (user as any).email; - if (userEmail) { - timeMap[userEmail] = Date.now(); - } - } - if (!user && originalUrl === "/") { - return res.redirect("/login"); - } - next(); - }); + // app.use("*", ({ user, originalUrl }, res, next) => { + // if (user && !originalUrl.includes("Heartbeat")) { + // const userEmail = (user as any).email; + // if (userEmail) { + // timeMap[userEmail] = Date.now(); + // } + // } + // if (!user && originalUrl === "/") { + // return res.redirect("/login"); + // } + // next(); + // }); app.use(wdm(compiler, { publicPath: config.output.publicPath })); app.use(whm(compiler)); |