aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/client/util/DictationManager.ts144
-rw-r--r--src/client/views/GlobalKeyHandler.ts32
-rw-r--r--src/client/views/Main.scss30
-rw-r--r--src/client/views/MainView.tsx55
-rw-r--r--src/client/views/collections/CollectionBaseView.tsx18
-rw-r--r--src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx92
-rw-r--r--src/client/views/nodes/DocumentView.tsx10
7 files changed, 330 insertions, 51 deletions
diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
index b58bdb6c7..80efe12cd 100644
--- a/src/client/util/DictationManager.ts
+++ b/src/client/util/DictationManager.ts
@@ -1,3 +1,15 @@
+import { string } from "prop-types";
+import { observable, action, autorun } from "mobx";
+import { SelectionManager } from "./SelectionManager";
+import { DocumentView } from "../views/nodes/DocumentView";
+import { UndoManager } from "./UndoManager";
+import * as converter from "words-to-numbers";
+import { Doc } from "../../new_fields/Doc";
+import { List } from "../../new_fields/List";
+import { Docs } from "../documents/Documents";
+import { CollectionViewType } from "../views/collections/CollectionBaseView";
+import { MainView } from "../views/MainView";
+
namespace CORE {
export interface IWindow extends Window {
webkitSpeechRecognition: any;
@@ -5,11 +17,14 @@ namespace CORE {
}
const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+export type IndependentAction = (target: DocumentView) => any | Promise<any>;
+export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise<any>;
+export type RegexEntry = { key: RegExp, value: DependentAction };
export default class DictationManager {
public static Instance = new DictationManager();
- private isListening = false;
private recognizer: any;
+ private isListening = false;
constructor() {
this.recognizer = new webkitSpeechRecognition();
@@ -19,8 +34,12 @@ export default class DictationManager {
finish = (handler: any, data: any) => {
handler(data);
- this.isListening = false;
+ this.stop();
+ }
+
+ stop = () => {
this.recognizer.stop();
+ this.isListening = false;
}
listen = () => {
@@ -33,7 +52,128 @@ export default class DictationManager {
this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript);
this.recognizer.onerror = (e: any) => this.finish(reject, e);
});
+ }
+
+ private sanitize = (title: string) => {
+ return title.replace("...", "").toLowerCase().trim();
+ }
+
+ public registerStatic = (keys: Array<string>, action: IndependentAction, overwrite = false) => {
+ let success = true;
+ keys.forEach(key => {
+ key = this.sanitize(key);
+ let existing = RegisteredCommands.Independent.get(key);
+ if (!existing || overwrite) {
+ RegisteredCommands.Independent.set(key, action);
+ } else {
+ success = false;
+ }
+ });
+ return success;
+ }
+
+ public interpretNumber = (number: string) => {
+ let initial = parseInt(number);
+ if (!isNaN(initial)) {
+ return initial;
+ }
+ let converted = converter.wordsToNumbers(number, { fuzzy: true });
+ if (converted === null) {
+ return NaN;
+ }
+ return typeof converted === "string" ? parseInt(converted) : converted;
+ }
+
+ public registerDynamic = (dynamicKey: RegExp, action: DependentAction) => {
+ RegisteredCommands.Dependent.push({
+ key: dynamicKey,
+ value: action
+ });
+ }
+
+ public execute = async (phrase: string) => {
+ let target = SelectionManager.SelectedDocuments()[0];
+ if (!target) {
+ return;
+ }
+ let batch = UndoManager.StartBatch("Dictation Action");
+ phrase = this.sanitize(phrase);
+
+ let independentAction = RegisteredCommands.Independent.get(phrase);
+ if (independentAction) {
+ await independentAction(target);
+ return true;
+ }
+ let success = false;
+ for (let entry of RegisteredCommands.Dependent) {
+ let regex = entry.key;
+ let dependentAction = entry.value;
+ let matches = regex.exec(phrase);
+ regex.lastIndex = 0;
+ if (matches !== null) {
+ await dependentAction(target, matches);
+ success = true;
+ break;
+ }
+ }
+ batch.end();
+
+ return success;
}
+}
+
+export namespace RegisteredCommands {
+
+ export const Independent = new Map<string, IndependentAction>([
+
+ ["clear", (target: DocumentView) => {
+ Doc.GetProto(target.props.Document).data = new List();
+ }],
+
+ ["open fields", (target: DocumentView) => {
+ let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 });
+ target.props.addDocTab(kvp, target.dataDoc, "onRight");
+ }]
+
+ ]);
+
+ export const Dependent = new Array<RegexEntry>(
+
+ {
+ key: /create (\w+) documents of type (image|nested collection)/g,
+ value: (target: DocumentView, matches: RegExpExecArray) => {
+ let count = DictationManager.Instance.interpretNumber(matches[1]);
+ let what = matches[2];
+ if (!("viewType" in target.props.Document)) {
+ return;
+ }
+ let dataDoc = Doc.GetProto(target.props.Document);
+ let fieldKey = "data";
+ for (let i = 0; i < count; i++) {
+ let created: Doc | undefined;
+ switch (what) {
+ case "image":
+ created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg");
+ break;
+ case "nested collection":
+ created = Docs.Create.FreeformDocument([], {});
+ break;
+ }
+ created && Doc.AddDocToList(dataDoc, fieldKey, created);
+ }
+ }
+ },
+
+ {
+ key: /view as (freeform|stacking|masonry|schema|tree)/g,
+ value: (target: DocumentView, matches: RegExpExecArray) => {
+ let mode = CollectionViewType.ValueOf(matches[1]);
+ mode && (target.props.Document.viewType = mode);
+ }
+ }
+
+ );
+
} \ No newline at end of file
diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts
index ea2e3e196..59d120974 100644
--- a/src/client/views/GlobalKeyHandler.ts
+++ b/src/client/views/GlobalKeyHandler.ts
@@ -3,7 +3,7 @@ import { SelectionManager } from "../util/SelectionManager";
import { CollectionDockingView } from "./collections/CollectionDockingView";
import { MainView } from "./MainView";
import { DragManager } from "../util/DragManager";
-import { action } from "mobx";
+import { action, runInAction } from "mobx";
import { Doc } from "../../new_fields/Doc";
import { CognitiveServices } from "../cognitive_services/CognitiveServices";
import DictationManager from "../util/DictationManager";
@@ -62,7 +62,8 @@ export default class KeyManager {
private unmodified = action((keyname: string, e: KeyboardEvent) => {
switch (keyname) {
case "escape":
- if (MainView.Instance.isPointerDown) {
+ let main = MainView.Instance;
+ if (main.isPointerDown) {
DragManager.AbortDrag();
} else {
if (CollectionDockingView.Instance.HasFullScreen()) {
@@ -71,8 +72,12 @@ export default class KeyManager {
SelectionManager.DeselectAll();
}
}
- MainView.Instance.toggleColorPicker(true);
+ main.toggleColorPicker(true);
SelectionManager.DeselectAll();
+ DictationManager.Instance.stop();
+ main.dictationOverlayVisible = false;
+ main.dictationSuccess = undefined;
+ main.overlayTimeout && clearTimeout(main.overlayTimeout);
break;
case "delete":
case "backspace":
@@ -106,13 +111,24 @@ export default class KeyManager {
switch (keyname) {
case " ":
- let transcript = await DictationManager.Instance.listen();
- console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`);
- let command: ContextMenuProps | undefined;
- transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event();
+ let main = MainView.Instance;
+ main.dictationOverlayVisible = true;
+ main.isListening = true;
+ let manager = DictationManager.Instance;
+ let command = await manager.listen();
+ main.isListening = false;
+ if (!command) {
+ break;
+ }
+ command = command.toLowerCase();
+ main.dictatedPhrase = command;
+ main.dictationSuccess = await manager.execute(command);
+ main.overlayTimeout = setTimeout(() => {
+ main.dictationOverlayVisible = false;
+ main.dictationSuccess = undefined;
+ }, 3000);
stopPropagation = true;
preventDefault = true;
- break;
}
return {
diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss
index eed2ae4fa..8e57b88c3 100644
--- a/src/client/views/Main.scss
+++ b/src/client/views/Main.scss
@@ -266,4 +266,34 @@ ul#add-options-list {
height: 25%;
position: relative;
display: flex;
+}
+
+.dictation-prompt {
+ position: absolute;
+ z-index: 1000;
+ text-align: center;
+ justify-content: center;
+ align-self: center;
+ align-content: center;
+ padding: 20px;
+ background: gainsboro;
+ border-radius: 10px;
+ border: 3px solid black;
+ box-shadow: #00000044 5px 5px 10px;
+ transform: translate(-50%, -50%);
+ top: 50%;
+ font-style: italic;
+ left: 50%;
+ transition: 0.5s all ease;
+ pointer-events: none;
+}
+
+.dictation-prompt-overlay {
+ width: 100%;
+ height: 100%;
+ position: absolute;
+ background: darkslategray;
+ z-index: 999;
+ transition: 0.5s all ease;
+ pointer-events: none;
} \ No newline at end of file
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 2ecf5fd85..4a5e4a3d1 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -47,6 +47,14 @@ export class MainView extends React.Component {
@observable private _workspacesShown: boolean = false;
@observable public pwidth: number = 0;
@observable public pheight: number = 0;
+
+ @observable private dictationState = "Listening...";
+ @observable private dictationSuccessState: boolean | undefined = undefined;
+ @observable private dictationDisplayState = false;
+ @observable private dictationListeningState = false;
+
+ public overlayTimeout: NodeJS.Timeout | undefined;
+
@computed private get mainContainer(): Opt<Doc> {
return FieldValue(Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc));
}
@@ -64,6 +72,38 @@ export class MainView extends React.Component {
}
}
+ public get dictatedPhrase() {
+ return this.dictationState;
+ }
+
+ public set dictatedPhrase(value: string) {
+ runInAction(() => this.dictationState = value);
+ }
+
+ public get dictationSuccess() {
+ return this.dictationSuccessState;
+ }
+
+ public set dictationSuccess(value: boolean | undefined) {
+ runInAction(() => this.dictationSuccessState = value);
+ }
+
+ public get dictationOverlayVisible() {
+ return this.dictationDisplayState;
+ }
+
+ public set dictationOverlayVisible(value: boolean) {
+ runInAction(() => this.dictationDisplayState = value);
+ }
+
+ public get isListening() {
+ return this.dictationListeningState;
+ }
+
+ public set isListening(value: boolean) {
+ runInAction(() => this.dictationListeningState = value);
+ }
+
componentWillMount() {
var tag = document.createElement('script');
@@ -463,8 +503,23 @@ export class MainView extends React.Component {
}
render() {
+ let display = this.dictationOverlayVisible;
+ let success = this.dictationSuccess;
+ let result = this.isListening ? "Listening..." : `"${this.dictatedPhrase}"`;
return (
<div id="main-div">
+ <div
+ className={"dictation-prompt"}
+ style={{
+ opacity: display ? 1 : 0,
+ background: success === undefined ? "gainsboro" : success ? "lawngreen" : "red",
+ borderColor: this.isListening ? "red" : "black",
+ }}
+ >{result}</div>
+ <div
+ className={"dictation-prompt-overlay"}
+ style={{ opacity: display ? 0.4 : 0 }}
+ />
<DocumentDecorations />
{this.mainContent}
<PreviewCursor />
diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx
index c595a4c56..3f88ed98c 100644
--- a/src/client/views/collections/CollectionBaseView.tsx
+++ b/src/client/views/collections/CollectionBaseView.tsx
@@ -22,6 +22,24 @@ export enum CollectionViewType {
Masonry
}
+export namespace CollectionViewType {
+
+ const stringMapping = new Map<string, CollectionViewType>([
+ ["invalid", CollectionViewType.Invalid],
+ ["freeform", CollectionViewType.Freeform],
+ ["schema", CollectionViewType.Schema],
+ ["docking", CollectionViewType.Docking],
+ ["tree", CollectionViewType.Tree],
+ ["stacking", CollectionViewType.Stacking],
+ ["masonry", CollectionViewType.Masonry]
+ ]);
+
+ export const ValueOf = (value: string) => {
+ return stringMapping.get(value.toLowerCase());
+ };
+
+}
+
export interface CollectionRenderProps {
addDocument: (document: Doc, allowDuplicates?: boolean) => boolean;
removeDocument: (document: Doc) => boolean;
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 33a584c6f..59c77f1c9 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -29,14 +29,15 @@ import { DocumentViewProps, positionSchema } from "../../nodes/DocumentView";
import { pageSchema } from "../../nodes/ImageBox";
import { OverlayElementOptions, OverlayView } from "../../OverlayView";
import PDFMenu from "../../pdf/PDFMenu";
+import { CollectionSubView, SubCollectionViewProps } from "../CollectionSubView";
import { ScriptBox } from "../../ScriptBox";
-import { CollectionSubView } from "../CollectionSubView";
import { CollectionFreeFormLinksView } from "./CollectionFreeFormLinksView";
import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCursors";
import "./CollectionFreeFormView.scss";
import { MarqueeView } from "./MarqueeView";
import React = require("react");
import v5 = require("uuid/v5");
+import DictationManager from "../../../util/DictationManager";
library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload);
@@ -122,11 +123,18 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
});
}
+ constructor(props: SubCollectionViewProps) {
+ super(props);
+ let fixed = DictationManager.Instance.registerStatic;
+ fixed(["Unset Fit To Container", "Set Fit To Container"], this.fitToContainer);
+ fixed(["Arrange contents in grid"], this.arrangeContents);
+ fixed(["Analyze Strokes"], this.analyzeStrokes);
+ }
+
@computed get fieldExtensionDoc() {
return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true");
}
-
@undoBatch
@action
drop = (e: Event, de: DragManager.DropEvent) => {
@@ -523,50 +531,62 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
}
+ fitToContainer = async () => this.props.Document.fitToBox = !this.fitToBox;
+
+ arrangeContents = async () => {
+ const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
+ UndoManager.RunInBatch(() => {
+ if (docs) {
+ let startX = this.Document.panX || 0;
+ let x = startX;
+ let y = this.Document.panY || 0;
+ let i = 0;
+ const width = Math.max(...docs.map(doc => NumCast(doc.width)));
+ const height = Math.max(...docs.map(doc => NumCast(doc.height)));
+ for (const doc of docs) {
+ doc.x = x;
+ doc.y = y;
+ x += width + 20;
+ if (++i === 6) {
+ i = 0;
+ x = startX;
+ y += height + 20;
+ }
+ }
+ }
+ }, "arrange contents");
+ }
+
+ analyzeStrokes = async () => {
+ let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
+ if (!data) {
+ return;
+ }
+ let relevantKeys = ["inkAnalysis", "handwriting"];
+ CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
+ }
+
onContextMenu = (e: React.MouseEvent) => {
let layoutItems: ContextMenuProps[] = [];
layoutItems.push({
description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
- event: async () => this.props.Document.fitToBox = !this.fitToBox,
+ event: this.fitToContainer,
icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt"
});
layoutItems.push({
description: "Arrange contents in grid",
- icon: "table",
- event: async () => {
- const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
- UndoManager.RunInBatch(() => {
- if (docs) {
- let startX = this.Document.panX || 0;
- let x = startX;
- let y = this.Document.panY || 0;
- let i = 0;
- const width = Math.max(...docs.map(doc => NumCast(doc.width)));
- const height = Math.max(...docs.map(doc => NumCast(doc.height)));
- for (const doc of docs) {
- doc.x = x;
- doc.y = y;
- x += width + 20;
- if (++i === 6) {
- i = 0;
- x = startX;
- y += height + 20;
- }
- }
- }
- }, "arrange contents");
- }
+ event: this.arrangeContents,
+ icon: "table"
});
- ContextMenu.Instance.addItem({ description: "Layout...", subitems: layoutItems, icon: "compass" });
ContextMenu.Instance.addItem({
- description: "Analyze Strokes", event: async () => {
- let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
- if (!data) {
- return;
- }
- let relevantKeys = ["inkAnalysis", "handwriting"];
- CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
- }, icon: "paint-brush"
+ description: "Layout...",
+ subitems: layoutItems,
+ icon: "compass"
+ });
+ ContextMenu.Instance.addItem({
+ description: "Analyze Strokes",
+ event: this.analyzeStrokes,
+ icon: "paint-brush"
});
ContextMenu.Instance.addItem({
description: "Import document", icon: "upload", event: () => {
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 14f0b5a5a..39574db0f 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -43,6 +43,7 @@ import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-i
import { DocumentDecorations } from '../DocumentDecorations';
import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
import DictationManager from '../../util/DictationManager';
+import { CollectionViewType } from '../collections/CollectionBaseView';
const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
library.add(fa.faTrash);
@@ -152,10 +153,6 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
set templates(templates: List<string>) { this.props.Document.templates = templates; }
screenRect = (): ClientRect | DOMRect => this._mainCont.current ? this._mainCont.current.getBoundingClientRect() : new DOMRect();
- constructor(props: DocumentViewProps) {
- super(props);
- }
-
_animateToIconDisposer?: IReactionDisposer;
_reactionDisposer?: IReactionDisposer;
@action
@@ -414,7 +411,10 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
deleteClicked = (): void => { SelectionManager.DeselectAll(); this.props.removeDocument && this.props.removeDocument(this.props.Document); }
@undoBatch
- fieldsClicked = (): void => { let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }); this.props.addDocTab(kvp, this.dataDoc, "onRight"); }
+ fieldsClicked = (): void => {
+ let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 });
+ this.props.addDocTab(kvp, this.dataDoc, "onRight");
+ }
@undoBatch
makeBtnClicked = (): void => {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-10 10:39:00 +0000