aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--package.json1
-rw-r--r--src/client/util/DictationManager.ts88
-rw-r--r--src/client/views/GlobalKeyHandler.ts8
-rw-r--r--src/client/views/MainView.tsx2
-rw-r--r--src/client/views/collections/CollectionBaseView.tsx18
-rw-r--r--src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx92
-rw-r--r--src/client/views/nodes/DocumentView.tsx48
7 files changed, 217 insertions, 40 deletions
diff --git a/package.json b/package.json
index 37052fde3..ef2d18163 100644
--- a/package.json
+++ b/package.json
@@ -200,6 +200,7 @@
"typescript-collections": "^1.3.2",
"url-loader": "^1.1.2",
"uuid": "^3.3.2",
+ "words-to-numbers": "^1.5.1",
"xoauth2": "^1.2.0",
"youtube": "^0.1.0"
}
diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
index b58bdb6c7..60b25afc5 100644
--- a/src/client/util/DictationManager.ts
+++ b/src/client/util/DictationManager.ts
@@ -1,3 +1,10 @@
+import { string } from "prop-types";
+import { observable, action } from "mobx";
+import { SelectionManager } from "./SelectionManager";
+import { DocumentView } from "../views/nodes/DocumentView";
+import { UndoManager } from "./UndoManager";
+import * as converter from "words-to-numbers";
+
namespace CORE {
export interface IWindow extends Window {
webkitSpeechRecognition: any;
@@ -5,9 +12,14 @@ namespace CORE {
}
const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+export type Action = (target: DocumentView) => any | Promise<any>;
+export type DynamicAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise<any>;
+export type RegexEntry = { key: RegExp, value: DynamicAction };
export default class DictationManager {
public static Instance = new DictationManager();
+ private registeredCommands = new Map<string, Action>();
+ private registeredRegexes: RegexEntry[] = [];
private isListening = false;
private recognizer: any;
@@ -17,8 +29,16 @@ export default class DictationManager {
this.recognizer.continuous = true;
}
+ @observable public current = "";
+
+ @action
finish = (handler: any, data: any) => {
+ this.current = data;
handler(data);
+ this.stop();
+ }
+
+ stop = () => {
this.isListening = false;
this.recognizer.stop();
}
@@ -36,4 +56,72 @@ export default class DictationManager {
}
+ private sanitize = (title: string) => {
+ return title.replace("...", "").toLowerCase().trim();
+ }
+
+ public registerStatic = (keys: Array<string>, action: Action, overwrite = false) => {
+ let success = true;
+ keys.forEach(key => {
+ key = this.sanitize(key);
+ let existing = this.registeredCommands.get(key);
+ if (!existing || overwrite) {
+ this.registeredCommands.set(key, action);
+ } else {
+ success = false;
+ }
+ });
+ return success;
+ }
+
+ public interpretNumber = (number: string) => {
+ let initial = parseInt(number);
+ if (!isNaN(initial)) {
+ return initial;
+ }
+ let converted = converter.wordsToNumbers(number, { fuzzy: true });
+ if (converted === null) {
+ return NaN;
+ }
+ return typeof converted === "string" ? parseInt(converted) : converted;
+ }
+
+ public registerDynamic = (dynamicKey: RegExp, action: DynamicAction) => {
+ this.registeredRegexes.push({
+ key: dynamicKey,
+ value: action
+ });
+ }
+
+ public execute = async (phrase: string) => {
+ let target = SelectionManager.SelectedDocuments()[0];
+ if (!target) {
+ return;
+ }
+ let batch = UndoManager.StartBatch("Dictation Action");
+ phrase = this.sanitize(phrase);
+
+ let registeredAction = this.registeredCommands.get(phrase);
+ if (registeredAction) {
+ await registeredAction(target);
+ return true;
+ }
+
+ let success = false;
+ for (let entry of this.registeredRegexes) {
+ let regex = entry.key;
+ let registeredDynamicAction = entry.value;
+ let matches = regex.exec(phrase);
+ regex.lastIndex = 0;
+ if (matches !== null) {
+ await registeredDynamicAction(target, matches);
+ success = true;
+ break;
+ }
+ }
+ batch.end();
+
+ return success;
+ }
+
} \ No newline at end of file
diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts
index 373584b4e..1d3c77ec7 100644
--- a/src/client/views/GlobalKeyHandler.ts
+++ b/src/client/views/GlobalKeyHandler.ts
@@ -73,6 +73,7 @@ export default class KeyManager {
}
MainView.Instance.toggleColorPicker(true);
SelectionManager.DeselectAll();
+ DictationManager.Instance.stop();
break;
case "delete":
case "backspace":
@@ -106,10 +107,11 @@ export default class KeyManager {
switch (keyname) {
case " ":
- let transcript = await DictationManager.Instance.listen();
+ console.log("Listening...");
+ let analyzer = DictationManager.Instance;
+ let transcript = await analyzer.listen();
console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`);
- let command: ContextMenuProps | undefined;
- transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event();
+ transcript && analyzer.execute(transcript);
}
return {
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 91c8fe57c..36ac96907 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -39,6 +39,7 @@ import { FilterBox } from './search/FilterBox';
import { CollectionTreeView } from './collections/CollectionTreeView';
import { ClientUtils } from '../util/ClientUtils';
import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField';
+import DictationManager from '../util/DictationManager';
@observer
export class MainView extends React.Component {
@@ -459,6 +460,7 @@ export class MainView extends React.Component {
render() {
return (
<div id="main-div">
+ <h1>{DictationManager.Instance.current}</h1>
<DocumentDecorations />
{this.mainContent}
<PreviewCursor />
diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx
index c595a4c56..3f88ed98c 100644
--- a/src/client/views/collections/CollectionBaseView.tsx
+++ b/src/client/views/collections/CollectionBaseView.tsx
@@ -22,6 +22,24 @@ export enum CollectionViewType {
Masonry
}
+export namespace CollectionViewType {
+
+ const stringMapping = new Map<string, CollectionViewType>([
+ ["invalid", CollectionViewType.Invalid],
+ ["freeform", CollectionViewType.Freeform],
+ ["schema", CollectionViewType.Schema],
+ ["docking", CollectionViewType.Docking],
+ ["tree", CollectionViewType.Tree],
+ ["stacking", CollectionViewType.Stacking],
+ ["masonry", CollectionViewType.Masonry]
+ ]);
+
+ export const ValueOf = (value: string) => {
+ return stringMapping.get(value.toLowerCase());
+ };
+
+}
+
export interface CollectionRenderProps {
addDocument: (document: Doc, allowDuplicates?: boolean) => boolean;
removeDocument: (document: Doc) => boolean;
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 8dac785e1..7856f3718 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -20,7 +20,7 @@ import { DocumentContentsView } from "../../nodes/DocumentContentsView";
import { DocumentViewProps, positionSchema } from "../../nodes/DocumentView";
import { pageSchema } from "../../nodes/ImageBox";
import PDFMenu from "../../pdf/PDFMenu";
-import { CollectionSubView } from "../CollectionSubView";
+import { CollectionSubView, SubCollectionViewProps } from "../CollectionSubView";
import { CollectionFreeFormLinksView } from "./CollectionFreeFormLinksView";
import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCursors";
import "./CollectionFreeFormView.scss";
@@ -38,6 +38,7 @@ import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsA
import { undo } from "prosemirror-history";
import { number } from "prop-types";
import { ContextMenu } from "../../ContextMenu";
+import DictationManager from "../../../util/DictationManager";
library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass);
@@ -121,11 +122,18 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
});
}
+ constructor(props: SubCollectionViewProps) {
+ super(props);
+ let fixed = DictationManager.Instance.registerStatic;
+ fixed(["Unset Fit To Container", "Set Fit To Container"], this.fitToContainer);
+ fixed(["Arrange contents in grid"], this.arrangeContents);
+ fixed(["Analyze Strokes"], this.analyzeStrokes);
+ }
+
@computed get fieldExtensionDoc() {
return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true");
}
-
@undoBatch
@action
drop = (e: Event, de: DragManager.DropEvent) => {
@@ -516,50 +524,62 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
}
+ fitToContainer = async () => this.props.Document.fitToBox = !this.fitToBox;
+
+ arrangeContents = async () => {
+ const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
+ UndoManager.RunInBatch(() => {
+ if (docs) {
+ let startX = this.Document.panX || 0;
+ let x = startX;
+ let y = this.Document.panY || 0;
+ let i = 0;
+ const width = Math.max(...docs.map(doc => NumCast(doc.width)));
+ const height = Math.max(...docs.map(doc => NumCast(doc.height)));
+ for (const doc of docs) {
+ doc.x = x;
+ doc.y = y;
+ x += width + 20;
+ if (++i === 6) {
+ i = 0;
+ x = startX;
+ y += height + 20;
+ }
+ }
+ }
+ }, "arrange contents");
+ }
+
+ analyzeStrokes = async () => {
+ let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
+ if (!data) {
+ return;
+ }
+ let relevantKeys = ["inkAnalysis", "handwriting"];
+ CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
+ }
+
onContextMenu = () => {
let layoutItems: ContextMenuProps[] = [];
layoutItems.push({
description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
- event: async () => this.props.Document.fitToBox = !this.fitToBox,
+ event: this.fitToContainer,
icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt"
});
layoutItems.push({
description: "Arrange contents in grid",
- icon: "table",
- event: async () => {
- const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
- UndoManager.RunInBatch(() => {
- if (docs) {
- let startX = this.Document.panX || 0;
- let x = startX;
- let y = this.Document.panY || 0;
- let i = 0;
- const width = Math.max(...docs.map(doc => NumCast(doc.width)));
- const height = Math.max(...docs.map(doc => NumCast(doc.height)));
- for (const doc of docs) {
- doc.x = x;
- doc.y = y;
- x += width + 20;
- if (++i === 6) {
- i = 0;
- x = startX;
- y += height + 20;
- }
- }
- }
- }, "arrange contents");
- }
+ event: this.arrangeContents,
+ icon: "table"
});
- ContextMenu.Instance.addItem({ description: "Layout...", subitems: layoutItems, icon: "compass" });
ContextMenu.Instance.addItem({
- description: "Analyze Strokes", event: async () => {
- let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
- if (!data) {
- return;
- }
- let relevantKeys = ["inkAnalysis", "handwriting"];
- CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
- }, icon: "paint-brush"
+ description: "Layout...",
+ subitems: layoutItems,
+ icon: "compass"
+ });
+ ContextMenu.Instance.addItem({
+ description: "Analyze Strokes",
+ event: this.analyzeStrokes,
+ icon: "paint-brush"
});
}
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index dc56c1c8f..00416ca42 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -43,6 +43,7 @@ import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-i
import { DocumentDecorations } from '../DocumentDecorations';
import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
import DictationManager from '../../util/DictationManager';
+import { CollectionViewType } from '../collections/CollectionBaseView';
const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
library.add(fa.faTrash);
@@ -153,6 +154,43 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
constructor(props: DocumentViewProps) {
super(props);
+ let fixed = DictationManager.Instance.registerStatic;
+ let dynamic = DictationManager.Instance.registerDynamic;
+ fixed(["Open Fields"], DocumentView.OpenFieldsDictation);
+ fixed(["Clear"], DocumentView.ClearChildren);
+ dynamic(/create (\w+) documents of type (image|nested collection)/g, DocumentView.BuildLayout);
+ dynamic(/view as (freeform|stacking|masonry|schema|tree)/g, DocumentView.SetViewMode);
+ }
+
+ public static ClearChildren = (target: DocumentView) => {
+ Doc.GetProto(target.props.Document).data = new List();
+ }
+
+ public static BuildLayout = (target: DocumentView, matches: RegExpExecArray) => {
+ let count = DictationManager.Instance.interpretNumber(matches[1]);
+ let what = matches[2];
+ if (!("viewType" in target.props.Document)) {
+ return;
+ }
+ let dataDoc = Doc.GetProto(target.props.Document);
+ let fieldKey = "data";
+ for (let i = 0; i < count; i++) {
+ let created: Doc | undefined;
+ switch (what) {
+ case "image":
+ created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg");
+ break;
+ case "nested collection":
+ created = Docs.Create.FreeformDocument([], {});
+ break;
+ }
+ created && Doc.AddDocToList(dataDoc, fieldKey, created);
+ }
+ }
+
+ public static SetViewMode = (target: DocumentView, matches: RegExpExecArray) => {
+ let mode = CollectionViewType.ValueOf(matches[1]);
+ mode && (target.props.Document.viewType = mode);
}
_animateToIconDisposer?: IReactionDisposer;
@@ -413,7 +451,15 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
deleteClicked = (): void => { SelectionManager.DeselectAll(); this.props.removeDocument && this.props.removeDocument(this.props.Document); }
@undoBatch
- fieldsClicked = (): void => { let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }); this.props.addDocTab(kvp, this.dataDoc, "onRight"); }
+ fieldsClicked = (): void => {
+ let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 });
+ this.props.addDocTab(kvp, this.dataDoc, "onRight");
+ }
+
+ public static OpenFieldsDictation = (target: DocumentView) => {
+ let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 });
+ target.props.addDocTab(kvp, target.dataDoc, "onRight");
+ }
@undoBatch
makeBtnClicked = (): void => {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-10 05:18:20 +0000