From e0316c21838613df0fbf43df6a9ca5d696c52f47 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 20:19:58 -0400 Subject: more interesting speech commands and command manager pattern for DictationManager --- src/client/util/DictationManager.ts | 88 +++++++++++++++++++++ src/client/views/GlobalKeyHandler.ts | 8 +- src/client/views/MainView.tsx | 2 + .../views/collections/CollectionBaseView.tsx | 18 +++++ .../collectionFreeForm/CollectionFreeFormView.tsx | 92 +++++++++++++--------- src/client/views/nodes/DocumentView.tsx | 48 ++++++++++- 6 files changed, 216 insertions(+), 40 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b58bdb6c7..60b25afc5 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,3 +1,10 @@ +import { string } from "prop-types"; +import { observable, action } from "mobx"; +import { SelectionManager } from "./SelectionManager"; +import { DocumentView } from "../views/nodes/DocumentView"; +import { UndoManager } from "./UndoManager"; +import * as converter from "words-to-numbers"; + namespace CORE { export interface IWindow extends Window { webkitSpeechRecognition: any; @@ -5,9 +12,14 @@ namespace CORE { } const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; +export type Action = (target: DocumentView) => any | Promise; +export type DynamicAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; +export type RegexEntry = { key: RegExp, value: DynamicAction }; export default class DictationManager { public static Instance = new DictationManager(); + private registeredCommands = new Map(); + private registeredRegexes: RegexEntry[] = []; private isListening = false; private recognizer: any; @@ -17,8 +29,16 @@ export default class DictationManager { this.recognizer.continuous = true; } + @observable public current = ""; + + @action finish = (handler: any, data: any) => { + this.current = data; handler(data); + this.stop(); + } + + stop = () => { this.isListening = false; this.recognizer.stop(); } @@ -36,4 +56,72 @@ export default class DictationManager { } + private sanitize = (title: string) => { + return title.replace("...", "").toLowerCase().trim(); + } + + public registerStatic = (keys: Array, action: Action, overwrite = false) => { + let success = true; + keys.forEach(key => { + key = this.sanitize(key); + let existing = this.registeredCommands.get(key); + if (!existing || overwrite) { + this.registeredCommands.set(key, action); + } else { + success = false; + } + }); + return success; + } + + public interpretNumber = (number: string) => { + let initial = parseInt(number); + if (!isNaN(initial)) { + return initial; + } + let converted = converter.wordsToNumbers(number, { fuzzy: true }); + if (converted === null) { + return NaN; + } + return typeof converted === "string" ? parseInt(converted) : converted; + } + + public registerDynamic = (dynamicKey: RegExp, action: DynamicAction) => { + this.registeredRegexes.push({ + key: dynamicKey, + value: action + }); + } + + public execute = async (phrase: string) => { + let target = SelectionManager.SelectedDocuments()[0]; + if (!target) { + return; + } + let batch = UndoManager.StartBatch("Dictation Action"); + phrase = this.sanitize(phrase); + + let registeredAction = this.registeredCommands.get(phrase); + if (registeredAction) { + await registeredAction(target); + return true; + } + + let success = false; + for (let entry of this.registeredRegexes) { + let regex = entry.key; + let registeredDynamicAction = entry.value; + let matches = regex.exec(phrase); + regex.lastIndex = 0; + if (matches !== null) { + await registeredDynamicAction(target, matches); + success = true; + break; + } + } + batch.end(); + + return success; + } + } \ No newline at end of file diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 373584b4e..1d3c77ec7 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -73,6 +73,7 @@ export default class KeyManager { } MainView.Instance.toggleColorPicker(true); SelectionManager.DeselectAll(); + DictationManager.Instance.stop(); break; case "delete": case "backspace": @@ -106,10 +107,11 @@ export default class KeyManager { switch (keyname) { case " ": - let transcript = await DictationManager.Instance.listen(); + console.log("Listening..."); + let analyzer = DictationManager.Instance; + let transcript = await analyzer.listen(); console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`); - let command: ContextMenuProps | undefined; - transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event(); + transcript && analyzer.execute(transcript); } return { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 91c8fe57c..36ac96907 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,6 +39,7 @@ import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; +import DictationManager from '../util/DictationManager'; @observer export class MainView extends React.Component { @@ -459,6 +460,7 @@ export class MainView extends React.Component { render() { return (
+

{DictationManager.Instance.current}

{this.mainContent} diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx index c595a4c56..3f88ed98c 100644 --- a/src/client/views/collections/CollectionBaseView.tsx +++ b/src/client/views/collections/CollectionBaseView.tsx @@ -22,6 +22,24 @@ export enum CollectionViewType { Masonry } +export namespace CollectionViewType { + + const stringMapping = new Map([ + ["invalid", CollectionViewType.Invalid], + ["freeform", CollectionViewType.Freeform], + ["schema", CollectionViewType.Schema], + ["docking", CollectionViewType.Docking], + ["tree", CollectionViewType.Tree], + ["stacking", CollectionViewType.Stacking], + ["masonry", CollectionViewType.Masonry] + ]); + + export const ValueOf = (value: string) => { + return stringMapping.get(value.toLowerCase()); + }; + +} + export interface CollectionRenderProps { addDocument: (document: Doc, allowDuplicates?: boolean) => boolean; removeDocument: (document: Doc) => boolean; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 8dac785e1..7856f3718 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -20,7 +20,7 @@ import { DocumentContentsView } from "../../nodes/DocumentContentsView"; import { DocumentViewProps, positionSchema } from "../../nodes/DocumentView"; import { pageSchema } from "../../nodes/ImageBox"; import PDFMenu from "../../pdf/PDFMenu"; -import { CollectionSubView } from "../CollectionSubView"; +import { CollectionSubView, SubCollectionViewProps } from "../CollectionSubView"; import { CollectionFreeFormLinksView } from "./CollectionFreeFormLinksView"; import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCursors"; import "./CollectionFreeFormView.scss"; @@ -38,6 +38,7 @@ import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsA import { undo } from "prosemirror-history"; import { number } from "prop-types"; import { ContextMenu } from "../../ContextMenu"; +import DictationManager from "../../../util/DictationManager"; library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass); @@ -121,11 +122,18 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }); } + constructor(props: SubCollectionViewProps) { + super(props); + let fixed = DictationManager.Instance.registerStatic; + fixed(["Unset Fit To Container", "Set Fit To Container"], this.fitToContainer); + fixed(["Arrange contents in grid"], this.arrangeContents); + fixed(["Analyze Strokes"], this.analyzeStrokes); + } + @computed get fieldExtensionDoc() { return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true"); } - @undoBatch @action drop = (e: Event, de: DragManager.DropEvent) => { @@ -516,50 +524,62 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } + fitToContainer = async () => this.props.Document.fitToBox = !this.fitToBox; + + arrangeContents = async () => { + const docs = await DocListCastAsync(this.Document[this.props.fieldKey]); + UndoManager.RunInBatch(() => { + if (docs) { + let startX = this.Document.panX || 0; + let x = startX; + let y = this.Document.panY || 0; + let i = 0; + const width = Math.max(...docs.map(doc => NumCast(doc.width))); + const height = Math.max(...docs.map(doc => NumCast(doc.height))); + for (const doc of docs) { + doc.x = x; + doc.y = y; + x += width + 20; + if (++i === 6) { + i = 0; + x = startX; + y += height + 20; + } + } + } + }, "arrange contents"); + } + + analyzeStrokes = async () => { + let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); + if (!data) { + return; + } + let relevantKeys = ["inkAnalysis", "handwriting"]; + CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); + } + onContextMenu = () => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, - event: async () => this.props.Document.fitToBox = !this.fitToBox, + event: this.fitToContainer, icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt" }); layoutItems.push({ description: "Arrange contents in grid", - icon: "table", - event: async () => { - const docs = await DocListCastAsync(this.Document[this.props.fieldKey]); - UndoManager.RunInBatch(() => { - if (docs) { - let startX = this.Document.panX || 0; - let x = startX; - let y = this.Document.panY || 0; - let i = 0; - const width = Math.max(...docs.map(doc => NumCast(doc.width))); - const height = Math.max(...docs.map(doc => NumCast(doc.height))); - for (const doc of docs) { - doc.x = x; - doc.y = y; - x += width + 20; - if (++i === 6) { - i = 0; - x = startX; - y += height + 20; - } - } - } - }, "arrange contents"); - } + event: this.arrangeContents, + icon: "table" }); - ContextMenu.Instance.addItem({ description: "Layout...", subitems: layoutItems, icon: "compass" }); ContextMenu.Instance.addItem({ - description: "Analyze Strokes", event: async () => { - let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); - if (!data) { - return; - } - let relevantKeys = ["inkAnalysis", "handwriting"]; - CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); - }, icon: "paint-brush" + description: "Layout...", + subitems: layoutItems, + icon: "compass" + }); + ContextMenu.Instance.addItem({ + description: "Analyze Strokes", + event: this.analyzeStrokes, + icon: "paint-brush" }); } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index dc56c1c8f..00416ca42 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -43,6 +43,7 @@ import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-i import { DocumentDecorations } from '../DocumentDecorations'; import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; import DictationManager from '../../util/DictationManager'; +import { CollectionViewType } from '../collections/CollectionBaseView'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -153,6 +154,43 @@ export class DocumentView extends DocComponent(Docu constructor(props: DocumentViewProps) { super(props); + let fixed = DictationManager.Instance.registerStatic; + let dynamic = DictationManager.Instance.registerDynamic; + fixed(["Open Fields"], DocumentView.OpenFieldsDictation); + fixed(["Clear"], DocumentView.ClearChildren); + dynamic(/create (\w+) documents of type (image|nested collection)/g, DocumentView.BuildLayout); + dynamic(/view as (freeform|stacking|masonry|schema|tree)/g, DocumentView.SetViewMode); + } + + public static ClearChildren = (target: DocumentView) => { + Doc.GetProto(target.props.Document).data = new List(); + } + + public static BuildLayout = (target: DocumentView, matches: RegExpExecArray) => { + let count = DictationManager.Instance.interpretNumber(matches[1]); + let what = matches[2]; + if (!("viewType" in target.props.Document)) { + return; + } + let dataDoc = Doc.GetProto(target.props.Document); + let fieldKey = "data"; + for (let i = 0; i < count; i++) { + let created: Doc | undefined; + switch (what) { + case "image": + created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); + break; + case "nested collection": + created = Docs.Create.FreeformDocument([], {}); + break; + } + created && Doc.AddDocToList(dataDoc, fieldKey, created); + } + } + + public static SetViewMode = (target: DocumentView, matches: RegExpExecArray) => { + let mode = CollectionViewType.ValueOf(matches[1]); + mode && (target.props.Document.viewType = mode); } _animateToIconDisposer?: IReactionDisposer; @@ -413,7 +451,15 @@ export class DocumentView extends DocComponent(Docu deleteClicked = (): void => { SelectionManager.DeselectAll(); this.props.removeDocument && this.props.removeDocument(this.props.Document); } @undoBatch - fieldsClicked = (): void => { let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }); this.props.addDocTab(kvp, this.dataDoc, "onRight"); } + fieldsClicked = (): void => { + let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }); + this.props.addDocTab(kvp, this.dataDoc, "onRight"); + } + + public static OpenFieldsDictation = (target: DocumentView) => { + let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); + target.props.addDocTab(kvp, target.dataDoc, "onRight"); + } @undoBatch makeBtnClicked = (): void => { -- cgit v1.2.3-70-g09d2 From 31a8bb32e696d58816329b66cfc5b92907494d1b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 23:29:23 -0400 Subject: refactor --- src/client/util/DictationManager.ts | 88 +++++++++++++++++++++++++++------ src/client/views/nodes/DocumentView.tsx | 46 ----------------- 2 files changed, 72 insertions(+), 62 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 60b25afc5..6d67f6d6d 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -4,6 +4,10 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; import { UndoManager } from "./UndoManager"; import * as converter from "words-to-numbers"; +import { Doc } from "../../new_fields/Doc"; +import { List } from "../../new_fields/List"; +import { Docs } from "../documents/Documents"; +import { CollectionViewType } from "../views/collections/CollectionBaseView"; namespace CORE { export interface IWindow extends Window { @@ -12,14 +16,12 @@ namespace CORE { } const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; -export type Action = (target: DocumentView) => any | Promise; -export type DynamicAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; -export type RegexEntry = { key: RegExp, value: DynamicAction }; +export type IndependentAction = (target: DocumentView) => any | Promise; +export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; +export type RegexEntry = { key: RegExp, value: DependentAction }; export default class DictationManager { public static Instance = new DictationManager(); - private registeredCommands = new Map(); - private registeredRegexes: RegexEntry[] = []; private isListening = false; private recognizer: any; @@ -60,13 +62,13 @@ export default class DictationManager { return title.replace("...", "").toLowerCase().trim(); } - public registerStatic = (keys: Array, action: Action, overwrite = false) => { + public registerStatic = (keys: Array, action: IndependentAction, overwrite = false) => { let success = true; keys.forEach(key => { key = this.sanitize(key); - let existing = this.registeredCommands.get(key); + let existing = RegisteredCommands.Independent.get(key); if (!existing || overwrite) { - this.registeredCommands.set(key, action); + RegisteredCommands.Independent.set(key, action); } else { success = false; } @@ -86,8 +88,8 @@ export default class DictationManager { return typeof converted === "string" ? parseInt(converted) : converted; } - public registerDynamic = (dynamicKey: RegExp, action: DynamicAction) => { - this.registeredRegexes.push({ + public registerDynamic = (dynamicKey: RegExp, action: DependentAction) => { + RegisteredCommands.Dependent.push({ key: dynamicKey, value: action }); @@ -101,20 +103,20 @@ export default class DictationManager { let batch = UndoManager.StartBatch("Dictation Action"); phrase = this.sanitize(phrase); - let registeredAction = this.registeredCommands.get(phrase); - if (registeredAction) { - await registeredAction(target); + let independentAction = RegisteredCommands.Independent.get(phrase); + if (independentAction) { + await independentAction(target); return true; } let success = false; - for (let entry of this.registeredRegexes) { + for (let entry of RegisteredCommands.Dependent) { let regex = entry.key; - let registeredDynamicAction = entry.value; + let dependentAction = entry.value; let matches = regex.exec(phrase); regex.lastIndex = 0; if (matches !== null) { - await registeredDynamicAction(target, matches); + await dependentAction(target, matches); success = true; break; } @@ -124,4 +126,58 @@ export default class DictationManager { return success; } +} + +export namespace RegisteredCommands { + + export const Independent = new Map([ + + ["clear", (target: DocumentView) => { + Doc.GetProto(target.props.Document).data = new List(); + }], + + ["open fields", (target: DocumentView) => { + let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); + target.props.addDocTab(kvp, target.dataDoc, "onRight"); + }] + + ]); + + export const Dependent = new Array( + + { + key: /create (\w+) documents of type (image|nested collection)/g, + value: (target: DocumentView, matches: RegExpExecArray) => { + let count = DictationManager.Instance.interpretNumber(matches[1]); + let what = matches[2]; + if (!("viewType" in target.props.Document)) { + return; + } + let dataDoc = Doc.GetProto(target.props.Document); + let fieldKey = "data"; + for (let i = 0; i < count; i++) { + let created: Doc | undefined; + switch (what) { + case "image": + created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); + break; + case "nested collection": + created = Docs.Create.FreeformDocument([], {}); + break; + } + created && Doc.AddDocToList(dataDoc, fieldKey, created); + } + } + }, + + { + key: /view as (freeform|stacking|masonry|schema|tree)/g, + value: (target: DocumentView, matches: RegExpExecArray) => { + let mode = CollectionViewType.ValueOf(matches[1]); + mode && (target.props.Document.viewType = mode); + } + } + + ); + } \ No newline at end of file diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 00416ca42..db8203167 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -152,47 +152,6 @@ export class DocumentView extends DocComponent(Docu set templates(templates: List) { this.props.Document.templates = templates; } screenRect = (): ClientRect | DOMRect => this._mainCont.current ? this._mainCont.current.getBoundingClientRect() : new DOMRect(); - constructor(props: DocumentViewProps) { - super(props); - let fixed = DictationManager.Instance.registerStatic; - let dynamic = DictationManager.Instance.registerDynamic; - fixed(["Open Fields"], DocumentView.OpenFieldsDictation); - fixed(["Clear"], DocumentView.ClearChildren); - dynamic(/create (\w+) documents of type (image|nested collection)/g, DocumentView.BuildLayout); - dynamic(/view as (freeform|stacking|masonry|schema|tree)/g, DocumentView.SetViewMode); - } - - public static ClearChildren = (target: DocumentView) => { - Doc.GetProto(target.props.Document).data = new List(); - } - - public static BuildLayout = (target: DocumentView, matches: RegExpExecArray) => { - let count = DictationManager.Instance.interpretNumber(matches[1]); - let what = matches[2]; - if (!("viewType" in target.props.Document)) { - return; - } - let dataDoc = Doc.GetProto(target.props.Document); - let fieldKey = "data"; - for (let i = 0; i < count; i++) { - let created: Doc | undefined; - switch (what) { - case "image": - created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); - break; - case "nested collection": - created = Docs.Create.FreeformDocument([], {}); - break; - } - created && Doc.AddDocToList(dataDoc, fieldKey, created); - } - } - - public static SetViewMode = (target: DocumentView, matches: RegExpExecArray) => { - let mode = CollectionViewType.ValueOf(matches[1]); - mode && (target.props.Document.viewType = mode); - } - _animateToIconDisposer?: IReactionDisposer; _reactionDisposer?: IReactionDisposer; @action @@ -456,11 +415,6 @@ export class DocumentView extends DocComponent(Docu this.props.addDocTab(kvp, this.dataDoc, "onRight"); } - public static OpenFieldsDictation = (target: DocumentView) => { - let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); - target.props.addDocTab(kvp, target.dataDoc, "onRight"); - } - @undoBatch makeBtnClicked = (): void => { let doc = Doc.GetProto(this.props.Document); -- cgit v1.2.3-70-g09d2 From 8a87f7110b56ca96b3960f6fb3917c7ed8c7a814 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 03:21:59 -0400 Subject: overlay completed --- src/client/util/DictationManager.ts | 12 +++---- src/client/views/GlobalKeyHandler.ts | 24 +++++++++---- src/client/views/Main.scss | 68 +++++++++++++++++++++++++++--------- src/client/views/MainView.tsx | 55 +++++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 6d67f6d6d..80efe12cd 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,5 +1,5 @@ import { string } from "prop-types"; -import { observable, action } from "mobx"; +import { observable, action, autorun } from "mobx"; import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; import { UndoManager } from "./UndoManager"; @@ -8,6 +8,7 @@ import { Doc } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; +import { MainView } from "../views/MainView"; namespace CORE { export interface IWindow extends Window { @@ -22,8 +23,8 @@ export type RegexEntry = { key: RegExp, value: DependentAction }; export default class DictationManager { public static Instance = new DictationManager(); - private isListening = false; private recognizer: any; + private isListening = false; constructor() { this.recognizer = new webkitSpeechRecognition(); @@ -31,18 +32,14 @@ export default class DictationManager { this.recognizer.continuous = true; } - @observable public current = ""; - - @action finish = (handler: any, data: any) => { - this.current = data; handler(data); this.stop(); } stop = () => { - this.isListening = false; this.recognizer.stop(); + this.isListening = false; } listen = () => { @@ -55,7 +52,6 @@ export default class DictationManager { this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript); this.recognizer.onerror = (e: any) => this.finish(reject, e); }); - } private sanitize = (title: string) => { diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 1d3c77ec7..a6020bd3f 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -3,7 +3,7 @@ import { SelectionManager } from "../util/SelectionManager"; import { CollectionDockingView } from "./collections/CollectionDockingView"; import { MainView } from "./MainView"; import { DragManager } from "../util/DragManager"; -import { action } from "mobx"; +import { action, runInAction } from "mobx"; import { Doc } from "../../new_fields/Doc"; import { CognitiveServices } from "../cognitive_services/CognitiveServices"; import DictationManager from "../util/DictationManager"; @@ -74,6 +74,7 @@ export default class KeyManager { MainView.Instance.toggleColorPicker(true); SelectionManager.DeselectAll(); DictationManager.Instance.stop(); + MainView.Instance.dictationOverlayVisible = false; break; case "delete": case "backspace": @@ -107,11 +108,22 @@ export default class KeyManager { switch (keyname) { case " ": - console.log("Listening..."); - let analyzer = DictationManager.Instance; - let transcript = await analyzer.listen(); - console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`); - transcript && analyzer.execute(transcript); + let main = MainView.Instance; + main.dictationOverlayVisible = true; + main.isListening = true; + let manager = DictationManager.Instance; + let command = await manager.listen(); + main.isListening = false; + if (!command) { + break; + } + command = command.toLowerCase(); + main.dictatedPhrase = command; + main.dictationSuccess = await manager.execute(command); + setTimeout(() => { + main.dictationOverlayVisible = false; + main.dictationSuccess = undefined; + }, 3000); } return { diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss index a16123476..8e57b88c3 100644 --- a/src/client/views/Main.scss +++ b/src/client/views/Main.scss @@ -24,7 +24,7 @@ div { .jsx-parser { width: 100%; - height:100%; + height: 100%; pointer-events: none; border-radius: inherit; } @@ -119,6 +119,7 @@ button:hover { margin-bottom: 10px; } } + .toolbar-color-picker { background-color: $light-color; border-radius: 5px; @@ -128,6 +129,7 @@ button:hover { left: -3px; box-shadow: $intermediate-color 0.2vw 0.2vw 0.8vw; } + .toolbar-color-button { border-radius: 11px; width: 22px; @@ -146,7 +148,7 @@ button:hover { bottom: 22px; left: 250px; - > label { + >label { background: $dark-color; color: $light-color; display: inline-block; @@ -168,15 +170,15 @@ button:hover { transform: scale(1.15); } - > input { + >input { display: none; } - > input:not(:checked)~#add-options-content { + >input:not(:checked)~#add-options-content { display: none; } - > input:checked~label { + >input:checked~label { transform: rotate(45deg); transition: transform 0.5s; cursor: pointer; @@ -221,7 +223,7 @@ ul#add-options-list { list-style: none; padding: 5 0 0 0; - > li { + >li { display: inline-block; padding: 0; } @@ -231,7 +233,7 @@ ul#add-options-list { height: 100%; position: absolute; display: flex; - flex-direction:column; + flex-direction: column; } .mainView-libraryHandle { @@ -243,21 +245,55 @@ ul#add-options-list { position: absolute; z-index: 1; } + .svg-inline--fa { vertical-align: unset; } + .mainView-workspace { - height:200px; - position:relative; - display:flex; + height: 200px; + position: relative; + display: flex; } + .mainView-library { - height:75%; - position:relative; - display:flex; + height: 75%; + position: relative; + display: flex; } + .mainView-recentlyClosed { - height:25%; - position:relative; - display:flex; + height: 25%; + position: relative; + display: flex; +} + +.dictation-prompt { + position: absolute; + z-index: 1000; + text-align: center; + justify-content: center; + align-self: center; + align-content: center; + padding: 20px; + background: gainsboro; + border-radius: 10px; + border: 3px solid black; + box-shadow: #00000044 5px 5px 10px; + transform: translate(-50%, -50%); + top: 50%; + font-style: italic; + left: 50%; + transition: 0.5s all ease; + pointer-events: none; +} + +.dictation-prompt-overlay { + width: 100%; + height: 100%; + position: absolute; + background: darkslategray; + z-index: 999; + transition: 0.5s all ease; + pointer-events: none; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 36ac96907..d6449cffc 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,7 +39,6 @@ import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; -import DictationManager from '../util/DictationManager'; @observer export class MainView extends React.Component { @@ -48,6 +47,12 @@ export class MainView extends React.Component { @observable private _workspacesShown: boolean = false; @observable public pwidth: number = 0; @observable public pheight: number = 0; + + @observable private dictationState = "Listening..."; + @observable private dictationSuccessState: boolean | undefined = undefined; + @observable private dictationDisplayState = false; + @observable private dictationListeningState = false; + @computed private get mainContainer(): Opt { return FieldValue(Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc)); } @@ -65,6 +70,38 @@ export class MainView extends React.Component { } } + public get dictatedPhrase() { + return this.dictationState; + } + + public set dictatedPhrase(value: string) { + runInAction(() => this.dictationState = value); + } + + public get dictationSuccess() { + return this.dictationSuccessState; + } + + public set dictationSuccess(value: boolean | undefined) { + runInAction(() => this.dictationSuccessState = value); + } + + public get dictationOverlayVisible() { + return this.dictationDisplayState; + } + + public set dictationOverlayVisible(value: boolean) { + runInAction(() => this.dictationDisplayState = value); + } + + public get isListening() { + return this.dictationListeningState; + } + + public set isListening(value: boolean) { + runInAction(() => this.dictationListeningState = value); + } + componentWillMount() { var tag = document.createElement('script'); @@ -458,9 +495,23 @@ export class MainView extends React.Component { } render() { + let display = this.dictationOverlayVisible; + let success = this.dictationSuccess; + let result = this.isListening ? "Listening..." : `"${this.dictatedPhrase}"`; return (
-

{DictationManager.Instance.current}

+
{result}
+
{this.mainContent} -- cgit v1.2.3-70-g09d2 From b2c52f3978d00f759489b38544844ed3206db611 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 03:47:25 -0400 Subject: undo batching error fixed --- src/client/util/DictationManager.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 80efe12cd..f0fd7cbde 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -102,6 +102,7 @@ export default class DictationManager { let independentAction = RegisteredCommands.Independent.get(phrase); if (independentAction) { await independentAction(target); + batch.end(); return true; } -- cgit v1.2.3-70-g09d2 From df444357c153abf8daf87d9f80f4fe2bcfba00b9 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 13:40:47 -0400 Subject: filtering --- src/client/util/DictationManager.ts | 51 ++++++++++++++++++++++++++++--------- src/new_fields/Types.ts | 8 +++--- 2 files changed, 44 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index f0fd7cbde..9c539e86a 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -4,11 +4,14 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; import { UndoManager } from "./UndoManager"; import * as converter from "words-to-numbers"; -import { Doc } from "../../new_fields/Doc"; +import { Doc, Field } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; import { MainView } from "../views/MainView"; +import { listSpec } from "../../new_fields/Schema"; +import { Cast, ToConstructor, ListSpec, CastCtor } from "../../new_fields/Types"; +import { ImageField } from "../../new_fields/URLField"; namespace CORE { export interface IWindow extends Window { @@ -20,6 +23,7 @@ const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; export type IndependentAction = (target: DocumentView) => any | Promise; export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; export type RegexEntry = { key: RegExp, value: DependentAction }; +export type RegistrationUnit = { filter: Predicate, action: T }; export default class DictationManager { public static Instance = new DictationManager(); @@ -58,13 +62,17 @@ export default class DictationManager { return title.replace("...", "").toLowerCase().trim(); } - public registerStatic = (keys: Array, action: IndependentAction, overwrite = false) => { + public registerStatic = (keys: Array, action: IndependentAction, filter: Predicate, overwrite = false) => { let success = true; keys.forEach(key => { key = this.sanitize(key); let existing = RegisteredCommands.Independent.get(key); if (!existing || overwrite) { - RegisteredCommands.Independent.set(key, action); + let unit = { + filter: filter, + action: action + }; + RegisteredCommands.Independent.set(key, unit); } else { success = false; } @@ -99,9 +107,9 @@ export default class DictationManager { let batch = UndoManager.StartBatch("Dictation Action"); phrase = this.sanitize(phrase); - let independentAction = RegisteredCommands.Independent.get(phrase); - if (independentAction) { - await independentAction(target); + let unit = RegisteredCommands.Independent.get(phrase); + if (unit && unit.filter(target)) { + await unit.action(target); batch.end(); return true; } @@ -127,15 +135,21 @@ export default class DictationManager { export namespace RegisteredCommands { - export const Independent = new Map([ + export const Independent = new Map>([ - ["clear", (target: DocumentView) => { - Doc.GetProto(target.props.Document).data = new List(); + ["clear", { + action: (target: DocumentView) => { + Doc.GetProto(target.props.Document).data = new List(); + }, + filter: Filters.isCollectionView }], - ["open fields", (target: DocumentView) => { - let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); - target.props.addDocTab(kvp, target.dataDoc, "onRight"); + ["open fields", { + action: (target: DocumentView) => { + let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); + target.props.addDocTab(kvp, target.dataDoc, "onRight"); + }, + filter: Filters.isImageView }] ]); @@ -177,4 +191,17 @@ export namespace RegisteredCommands { ); +} + +export type Predicate = (target: DocumentView) => boolean; + +export namespace Filters { + + const tryCast = (view: DocumentView, ctor: T) => Cast(Doc.GetProto(view.props.Document).data, ctor); + + export const isCollectionView: Predicate = (target: DocumentView) => tryCast(target, listSpec(Doc)) !== undefined; + + export const isImageView: Predicate = (target: DocumentView) => tryCast(target, ImageField) !== undefined; + + } \ No newline at end of file diff --git a/src/new_fields/Types.ts b/src/new_fields/Types.ts index 565ae2ee3..09cbff25e 100644 --- a/src/new_fields/Types.ts +++ b/src/new_fields/Types.ts @@ -48,9 +48,11 @@ export interface Interface { } export type WithoutRefField = T extends RefField ? never : T; -export function Cast | ListSpec>(field: FieldResult, ctor: T): FieldResult>; -export function Cast | ListSpec>(field: FieldResult, ctor: T, defaultVal: WithoutList>> | null): WithoutList>; -export function Cast | ListSpec>(field: FieldResult, ctor: T, defaultVal?: ToType | null): FieldResult> | undefined { +export type CastCtor = ToConstructor | ListSpec; + +export function Cast(field: FieldResult, ctor: T): FieldResult>; +export function Cast(field: FieldResult, ctor: T, defaultVal: WithoutList>> | null): WithoutList>; +export function Cast(field: FieldResult, ctor: T, defaultVal?: ToType | null): FieldResult> | undefined { if (field instanceof Promise) { return defaultVal === undefined ? field.then(f => Cast(f, ctor) as any) as any : defaultVal === null ? undefined : defaultVal; } -- cgit v1.2.3-70-g09d2 From 2b8c2ddcfefebdce9fdcc3e65476f6abc8b7e832 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 14:49:15 -0400 Subject: none-filter --- src/client/util/DictationManager.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 9c539e86a..e14000555 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -23,7 +23,7 @@ const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; export type IndependentAction = (target: DocumentView) => any | Promise; export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; export type RegexEntry = { key: RegExp, value: DependentAction }; -export type RegistrationUnit = { filter: Predicate, action: T }; +export type RegistrationUnit = { action: T, filter?: Predicate }; export default class DictationManager { public static Instance = new DictationManager(); @@ -104,31 +104,32 @@ export default class DictationManager { if (!target) { return; } - let batch = UndoManager.StartBatch("Dictation Action"); phrase = this.sanitize(phrase); let unit = RegisteredCommands.Independent.get(phrase); - if (unit && unit.filter(target)) { - await unit.action(target); - batch.end(); - return true; + if (unit) { + if (!unit.filter || unit.filter(target)) { + let batch = UndoManager.StartBatch("Dictation Independent Action"); + await unit.action(target); + batch.end(); + return true; + } } - let success = false; for (let entry of RegisteredCommands.Dependent) { let regex = entry.key; let dependentAction = entry.value; let matches = regex.exec(phrase); regex.lastIndex = 0; if (matches !== null) { + let batch = UndoManager.StartBatch("Dictation Dependent Action"); await dependentAction(target, matches); - success = true; - break; + batch.end(); + return true; } } - batch.end(); - return success; + return false; } } @@ -203,5 +204,4 @@ export namespace Filters { export const isImageView: Predicate = (target: DocumentView) => tryCast(target, ImageField) !== undefined; - } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 0eaaf47ec89a32aafb25add36a3d36bb5a2a1370 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 14:59:51 -0400 Subject: clean up --- src/client/util/DictationManager.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index e14000555..88b79be8a 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -2,7 +2,7 @@ import { string } from "prop-types"; import { observable, action, autorun } from "mobx"; import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; -import { UndoManager } from "./UndoManager"; +import { UndoManager, undoBatch } from "./UndoManager"; import * as converter from "words-to-numbers"; import { Doc, Field } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; @@ -23,7 +23,7 @@ const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; export type IndependentAction = (target: DocumentView) => any | Promise; export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; export type RegexEntry = { key: RegExp, value: DependentAction }; -export type RegistrationUnit = { action: T, filter?: Predicate }; +export type RegistrationUnit = { action: T, validate?: Predicate }; export default class DictationManager { public static Instance = new DictationManager(); @@ -99,6 +99,7 @@ export default class DictationManager { }); } + @undoBatch public execute = async (phrase: string) => { let target = SelectionManager.SelectedDocuments()[0]; if (!target) { @@ -108,10 +109,8 @@ export default class DictationManager { let unit = RegisteredCommands.Independent.get(phrase); if (unit) { - if (!unit.filter || unit.filter(target)) { - let batch = UndoManager.StartBatch("Dictation Independent Action"); + if (!unit.validate || unit.validate(target)) { await unit.action(target); - batch.end(); return true; } } @@ -122,9 +121,7 @@ export default class DictationManager { let matches = regex.exec(phrase); regex.lastIndex = 0; if (matches !== null) { - let batch = UndoManager.StartBatch("Dictation Dependent Action"); await dependentAction(target, matches); - batch.end(); return true; } } @@ -142,7 +139,7 @@ export namespace RegisteredCommands { action: (target: DocumentView) => { Doc.GetProto(target.props.Document).data = new List(); }, - filter: Filters.isCollectionView + validate: Filters.isCollectionView }], ["open fields", { @@ -150,7 +147,7 @@ export namespace RegisteredCommands { let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); target.props.addDocTab(kvp, target.dataDoc, "onRight"); }, - filter: Filters.isImageView + validate: Filters.isImageView }] ]); -- cgit v1.2.3-70-g09d2 From 7611bc1ec475dd398e172ba5dbdb9c48b26663c2 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 18:11:03 -0400 Subject: cleanup and validation of dependent commands --- src/client/util/DictationManager.ts | 59 +++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 88b79be8a..80fa63226 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -22,8 +22,8 @@ namespace CORE { const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; export type IndependentAction = (target: DocumentView) => any | Promise; export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; -export type RegexEntry = { key: RegExp, value: DependentAction }; -export type RegistrationUnit = { action: T, validate?: Predicate }; +export type RegistrationEntry = { action: IndependentAction, validate?: ActionPredicate }; +export type RegexEntry = { expression: RegExp, action: DependentAction, validate?: ActionPredicate }; export default class DictationManager { public static Instance = new DictationManager(); @@ -62,7 +62,7 @@ export default class DictationManager { return title.replace("...", "").toLowerCase().trim(); } - public registerStatic = (keys: Array, action: IndependentAction, filter: Predicate, overwrite = false) => { + public registerStatic = (keys: Array, action: IndependentAction, filter: ActionPredicate, overwrite = false) => { let success = true; keys.forEach(key => { key = this.sanitize(key); @@ -94,8 +94,8 @@ export default class DictationManager { public registerDynamic = (dynamicKey: RegExp, action: DependentAction) => { RegisteredCommands.Dependent.push({ - key: dynamicKey, - value: action + expression: dynamicKey, + action: action }); } @@ -107,21 +107,18 @@ export default class DictationManager { } phrase = this.sanitize(phrase); - let unit = RegisteredCommands.Independent.get(phrase); - if (unit) { - if (!unit.validate || unit.validate(target)) { - await unit.action(target); - return true; - } + let entry = RegisteredCommands.Independent.get(phrase); + if (entry && (!entry.validate || entry.validate(target))) { + await entry.action(target); + return true; } for (let entry of RegisteredCommands.Dependent) { - let regex = entry.key; - let dependentAction = entry.value; + let regex = entry.expression; let matches = regex.exec(phrase); regex.lastIndex = 0; - if (matches !== null) { - await dependentAction(target, matches); + if (matches !== null && (!entry.validate || entry.validate(target))) { + await entry.action(target, matches); return true; } } @@ -133,21 +130,20 @@ export default class DictationManager { export namespace RegisteredCommands { - export const Independent = new Map>([ + export const Independent = new Map([ ["clear", { action: (target: DocumentView) => { Doc.GetProto(target.props.Document).data = new List(); }, - validate: Filters.isCollectionView + validate: Validators.isCollectionView }], ["open fields", { action: (target: DocumentView) => { let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); target.props.addDocTab(kvp, target.dataDoc, "onRight"); - }, - validate: Filters.isImageView + } }] ]); @@ -155,13 +151,10 @@ export namespace RegisteredCommands { export const Dependent = new Array( { - key: /create (\w+) documents of type (image|nested collection)/g, - value: (target: DocumentView, matches: RegExpExecArray) => { + expression: /create (\w+) documents of type (image|nested collection)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { let count = DictationManager.Instance.interpretNumber(matches[1]); let what = matches[2]; - if (!("viewType" in target.props.Document)) { - return; - } let dataDoc = Doc.GetProto(target.props.Document); let fieldKey = "data"; for (let i = 0; i < count; i++) { @@ -176,29 +169,31 @@ export namespace RegisteredCommands { } created && Doc.AddDocToList(dataDoc, fieldKey, created); } - } + }, + validate: Validators.isCollectionView }, { - key: /view as (freeform|stacking|masonry|schema|tree)/g, - value: (target: DocumentView, matches: RegExpExecArray) => { + expression: /view as (freeform|stacking|masonry|schema|tree)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { let mode = CollectionViewType.ValueOf(matches[1]); mode && (target.props.Document.viewType = mode); - } + }, + validate: Validators.isCollectionView } ); } -export type Predicate = (target: DocumentView) => boolean; +export type ActionPredicate = (target: DocumentView) => boolean; -export namespace Filters { +export namespace Validators { const tryCast = (view: DocumentView, ctor: T) => Cast(Doc.GetProto(view.props.Document).data, ctor); - export const isCollectionView: Predicate = (target: DocumentView) => tryCast(target, listSpec(Doc)) !== undefined; + export const isCollectionView: ActionPredicate = (target: DocumentView) => tryCast(target, listSpec(Doc)) !== undefined; - export const isImageView: Predicate = (target: DocumentView) => tryCast(target, ImageField) !== undefined; + export const isImageView: ActionPredicate = (target: DocumentView) => tryCast(target, ImageField) !== undefined; } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From bbad2e964edb8ae58a5b611003dcba5ba170503d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 18:21:52 -0400 Subject: compiles --- src/client/util/DictationManager.ts | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 80fa63226..5c318fc9b 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -19,10 +19,21 @@ namespace CORE { } } +export namespace Validators { + + const tryCast = (view: DocumentView, ctor: T) => Cast(Doc.GetProto(view.props.Document).data, ctor); + + export const isCollectionView: ActionPredicate = (target: DocumentView) => tryCast(target, listSpec(Doc)) !== undefined; + + export const isImageView: ActionPredicate = (target: DocumentView) => tryCast(target, ImageField) !== undefined; + +} + const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; export type IndependentAction = (target: DocumentView) => any | Promise; export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; export type RegistrationEntry = { action: IndependentAction, validate?: ActionPredicate }; +export type ActionPredicate = (target: DocumentView) => boolean; export type RegexEntry = { expression: RegExp, action: DependentAction, validate?: ActionPredicate }; export default class DictationManager { @@ -62,15 +73,15 @@ export default class DictationManager { return title.replace("...", "").toLowerCase().trim(); } - public registerStatic = (keys: Array, action: IndependentAction, filter: ActionPredicate, overwrite = false) => { + public registerStatic = (keys: Array, action: IndependentAction, filter?: ActionPredicate) => { let success = true; keys.forEach(key => { key = this.sanitize(key); let existing = RegisteredCommands.Independent.get(key); - if (!existing || overwrite) { + if (!existing) { let unit = { - filter: filter, - action: action + action: action, + filter: filter }; RegisteredCommands.Independent.set(key, unit); } else { @@ -184,16 +195,4 @@ export namespace RegisteredCommands { ); -} - -export type ActionPredicate = (target: DocumentView) => boolean; - -export namespace Validators { - - const tryCast = (view: DocumentView, ctor: T) => Cast(Doc.GetProto(view.props.Document).data, ctor); - - export const isCollectionView: ActionPredicate = (target: DocumentView) => tryCast(target, listSpec(Doc)) !== undefined; - - export const isImageView: ActionPredicate = (target: DocumentView) => tryCast(target, ImageField) !== undefined; - } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 3b628238e9cf155941d949086de44320ceb8e18d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 18:23:45 -0400 Subject: try cast fix --- src/client/util/DictationManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 5c318fc9b..3e130f984 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -21,11 +21,11 @@ namespace CORE { export namespace Validators { - const tryCast = (view: DocumentView, ctor: T) => Cast(Doc.GetProto(view.props.Document).data, ctor); + const tryCast = (view: DocumentView, ctor: T) => Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; - export const isCollectionView: ActionPredicate = (target: DocumentView) => tryCast(target, listSpec(Doc)) !== undefined; + export const isCollectionView: ActionPredicate = (target: DocumentView) => tryCast(target, listSpec(Doc)); - export const isImageView: ActionPredicate = (target: DocumentView) => tryCast(target, ImageField) !== undefined; + export const isImageView: ActionPredicate = (target: DocumentView) => tryCast(target, ImageField); } -- cgit v1.2.3-70-g09d2 From 570081063e27498e3de46d90ae49aa4dc3fc5833 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 23:29:30 -0400 Subject: refactored mapping and validation to make more extensible --- src/client/util/DictationManager.ts | 59 ++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 3e130f984..add3f2dd3 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,17 +1,15 @@ -import { string } from "prop-types"; -import { observable, action, autorun } from "mobx"; import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; -import { UndoManager, undoBatch } from "./UndoManager"; +import { undoBatch } from "./UndoManager"; import * as converter from "words-to-numbers"; -import { Doc, Field } from "../../new_fields/Doc"; +import { Doc } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; -import { Docs } from "../documents/Documents"; +import { Docs, DocumentType } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; -import { MainView } from "../views/MainView"; import { listSpec } from "../../new_fields/Schema"; -import { Cast, ToConstructor, ListSpec, CastCtor } from "../../new_fields/Types"; -import { ImageField } from "../../new_fields/URLField"; +import { Cast, CastCtor } from "../../new_fields/Types"; +import { ImageField, AudioField } from "../../new_fields/URLField"; +import { HistogramField } from "../northstar/dash-fields/HistogramField"; namespace CORE { export interface IWindow extends Window { @@ -19,22 +17,37 @@ namespace CORE { } } -export namespace Validators { - - const tryCast = (view: DocumentView, ctor: T) => Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; - - export const isCollectionView: ActionPredicate = (target: DocumentView) => tryCast(target, listSpec(Doc)); - - export const isImageView: ActionPredicate = (target: DocumentView) => tryCast(target, ImageField); +const Mapping = new Map([ + [DocumentType.COL, listSpec(Doc)], + [DocumentType.AUDIO, AudioField], + [DocumentType.IMG, ImageField], + [DocumentType.HIST, HistogramField], + [DocumentType.IMPORT, listSpec(Doc)] +]); + +const tryCast = (view: DocumentView, type: DocumentType) => { + let ctor = Mapping.get(type); + if (!ctor) { + return false; + } + return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; +}; -} +const validate = (target: DocumentView, types: DocumentType[]) => { + for (let type of types) { + if (tryCast(target, type)) { + return true; + } + } + return false; +}; const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; export type IndependentAction = (target: DocumentView) => any | Promise; export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; -export type RegistrationEntry = { action: IndependentAction, validate?: ActionPredicate }; +export type RegistrationEntry = { action: IndependentAction, valid?: DocumentType[] }; export type ActionPredicate = (target: DocumentView) => boolean; -export type RegexEntry = { expression: RegExp, action: DependentAction, validate?: ActionPredicate }; +export type RegexEntry = { expression: RegExp, action: DependentAction, valid?: DocumentType[] }; export default class DictationManager { public static Instance = new DictationManager(); @@ -119,7 +132,7 @@ export default class DictationManager { phrase = this.sanitize(phrase); let entry = RegisteredCommands.Independent.get(phrase); - if (entry && (!entry.validate || entry.validate(target))) { + if (entry && (!entry.valid || validate(target, entry.valid))) { await entry.action(target); return true; } @@ -128,7 +141,7 @@ export default class DictationManager { let regex = entry.expression; let matches = regex.exec(phrase); regex.lastIndex = 0; - if (matches !== null && (!entry.validate || entry.validate(target))) { + if (matches !== null && (!entry.valid || validate(target, entry.valid))) { await entry.action(target, matches); return true; } @@ -147,7 +160,7 @@ export namespace RegisteredCommands { action: (target: DocumentView) => { Doc.GetProto(target.props.Document).data = new List(); }, - validate: Validators.isCollectionView + valid: [DocumentType.COL] }], ["open fields", { @@ -181,7 +194,7 @@ export namespace RegisteredCommands { created && Doc.AddDocToList(dataDoc, fieldKey, created); } }, - validate: Validators.isCollectionView + valid: [DocumentType.COL] }, { @@ -190,7 +203,7 @@ export namespace RegisteredCommands { let mode = CollectionViewType.ValueOf(matches[1]); mode && (target.props.Document.viewType = mode); }, - validate: Validators.isCollectionView + valid: [DocumentType.COL] } ); -- cgit v1.2.3-70-g09d2 From 5db091ba03d5dd13ffb984e85257c16a314d4c42 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 23:31:46 -0400 Subject: naming tweak --- src/client/util/DictationManager.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index add3f2dd3..49afe5371 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -45,9 +45,9 @@ const validate = (target: DocumentView, types: DocumentType[]) => { const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; export type IndependentAction = (target: DocumentView) => any | Promise; export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; -export type RegistrationEntry = { action: IndependentAction, valid?: DocumentType[] }; +export type RegistrationEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; export type ActionPredicate = (target: DocumentView) => boolean; -export type RegexEntry = { expression: RegExp, action: DependentAction, valid?: DocumentType[] }; +export type RegexEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] }; export default class DictationManager { public static Instance = new DictationManager(); @@ -132,7 +132,7 @@ export default class DictationManager { phrase = this.sanitize(phrase); let entry = RegisteredCommands.Independent.get(phrase); - if (entry && (!entry.valid || validate(target, entry.valid))) { + if (entry && (!entry.restrictTo || validate(target, entry.restrictTo))) { await entry.action(target); return true; } @@ -141,7 +141,7 @@ export default class DictationManager { let regex = entry.expression; let matches = regex.exec(phrase); regex.lastIndex = 0; - if (matches !== null && (!entry.valid || validate(target, entry.valid))) { + if (matches !== null && (!entry.restrictTo || validate(target, entry.restrictTo))) { await entry.action(target, matches); return true; } @@ -160,7 +160,7 @@ export namespace RegisteredCommands { action: (target: DocumentView) => { Doc.GetProto(target.props.Document).data = new List(); }, - valid: [DocumentType.COL] + restrictTo: [DocumentType.COL] }], ["open fields", { @@ -194,7 +194,7 @@ export namespace RegisteredCommands { created && Doc.AddDocToList(dataDoc, fieldKey, created); } }, - valid: [DocumentType.COL] + restrictTo: [DocumentType.COL] }, { @@ -203,7 +203,7 @@ export namespace RegisteredCommands { let mode = CollectionViewType.ValueOf(matches[1]); mode && (target.props.Document.viewType = mode); }, - valid: [DocumentType.COL] + restrictTo: [DocumentType.COL] } ); -- cgit v1.2.3-70-g09d2 From ec224416fc454c7fdbb62943408226c973d8c751 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 1 Aug 2019 02:30:47 -0400 Subject: fixed import ordering issue, allow for multiple targets and valid document types --- src/client/documents/Documents.ts | 45 ++++++++------- src/client/util/DictationManager.ts | 66 ++++++++-------------- .../collectionFreeForm/CollectionFreeFormView.tsx | 8 --- src/client/views/nodes/DocumentView.tsx | 1 - src/client/views/search/FilterBox.tsx | 2 +- src/client/views/search/SearchItem.tsx | 2 +- src/client/views/search/ToggleBar.tsx | 2 +- 7 files changed, 49 insertions(+), 77 deletions(-) (limited to 'src') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 2a1f63d59..3c7de17c8 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -1,3 +1,24 @@ +export enum DocumentType { + NONE = "none", + TEXT = "text", + HIST = "histogram", + IMG = "image", + WEB = "web", + COL = "collection", + KVP = "kvp", + VID = "video", + AUDIO = "audio", + PDF = "pdf", + ICON = "icon", + IMPORT = "import", + LINK = "link", + LINKDOC = "linkdoc", + BUTTON = "button", + TEMPLATE = "template", + EXTENSION = "extension", + YOUTUBE = "youtube", +} + import { HistogramField } from "../northstar/dash-fields/HistogramField"; import { HistogramBox } from "../northstar/dash-nodes/HistogramBox"; import { HistogramOperation } from "../northstar/operations/HistogramOperation"; @@ -25,14 +46,13 @@ import { OmitKeys, JSONUtils } from "../../Utils"; import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField } from "../../new_fields/URLField"; import { HtmlField } from "../../new_fields/HtmlField"; import { List } from "../../new_fields/List"; -import { Cast, NumCast, StrCast, ToConstructor, InterfaceValue, FieldValue } from "../../new_fields/Types"; +import { Cast, NumCast } from "../../new_fields/Types"; import { IconField } from "../../new_fields/IconField"; import { listSpec } from "../../new_fields/Schema"; import { DocServer } from "../DocServer"; import { dropActionType } from "../util/DragManager"; import { DateField } from "../../new_fields/DateField"; import { UndoManager } from "../util/UndoManager"; -import { RouteStore } from "../../server/RouteStore"; import { YoutubeBox } from "../apis/youtube/YoutubeBox"; import { CollectionDockingView } from "../views/collections/CollectionDockingView"; import { LinkManager } from "../util/LinkManager"; @@ -44,27 +64,6 @@ import { SchemaHeaderField, RandomPastel } from "../../new_fields/SchemaHeaderFi var requestImageSize = require('../util/request-image-size'); var path = require('path'); -export enum DocumentType { - NONE = "none", - TEXT = "text", - HIST = "histogram", - IMG = "image", - WEB = "web", - COL = "collection", - KVP = "kvp", - VID = "video", - AUDIO = "audio", - PDF = "pdf", - ICON = "icon", - IMPORT = "import", - LINK = "link", - LINKDOC = "linkdoc", - BUTTON = "button", - TEMPLATE = "template", - EXTENSION = "extension", - YOUTUBE = "youtube", -} - export interface DocumentOptions { x?: number; y?: number; diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 49afe5371..b0866a826 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -6,9 +6,9 @@ import { Doc } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs, DocumentType } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; -import { listSpec } from "../../new_fields/Schema"; import { Cast, CastCtor } from "../../new_fields/Types"; -import { ImageField, AudioField } from "../../new_fields/URLField"; +import { listSpec } from "../../new_fields/Schema"; +import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; namespace CORE { @@ -17,7 +17,7 @@ namespace CORE { } } -const Mapping = new Map([ +const ConstructorMap = new Map([ [DocumentType.COL, listSpec(Doc)], [DocumentType.AUDIO, AudioField], [DocumentType.IMG, ImageField], @@ -26,7 +26,7 @@ const Mapping = new Map([ ]); const tryCast = (view: DocumentView, type: DocumentType) => { - let ctor = Mapping.get(type); + let ctor = ConstructorMap.get(type); if (!ctor) { return false; } @@ -82,28 +82,6 @@ export default class DictationManager { }); } - private sanitize = (title: string) => { - return title.replace("...", "").toLowerCase().trim(); - } - - public registerStatic = (keys: Array, action: IndependentAction, filter?: ActionPredicate) => { - let success = true; - keys.forEach(key => { - key = this.sanitize(key); - let existing = RegisteredCommands.Independent.get(key); - if (!existing) { - let unit = { - action: action, - filter: filter - }; - RegisteredCommands.Independent.set(key, unit); - } else { - success = false; - } - }); - return success; - } - public interpretNumber = (number: string) => { let initial = parseInt(number); if (!isNaN(initial)) { @@ -116,34 +94,38 @@ export default class DictationManager { return typeof converted === "string" ? parseInt(converted) : converted; } - public registerDynamic = (dynamicKey: RegExp, action: DependentAction) => { - RegisteredCommands.Dependent.push({ - expression: dynamicKey, - action: action - }); - } - @undoBatch public execute = async (phrase: string) => { - let target = SelectionManager.SelectedDocuments()[0]; - if (!target) { + let targets = SelectionManager.SelectedDocuments(); + if (!targets || !targets.length) { return; } - phrase = this.sanitize(phrase); let entry = RegisteredCommands.Independent.get(phrase); - if (entry && (!entry.restrictTo || validate(target, entry.restrictTo))) { - await entry.action(target); - return true; + if (entry) { + let success = false; + for (let target of targets) { + if (!entry.restrictTo || validate(target, entry.restrictTo)) { + await entry.action(target); + success = true; + } + } + return success; } for (let entry of RegisteredCommands.Dependent) { let regex = entry.expression; let matches = regex.exec(phrase); regex.lastIndex = 0; - if (matches !== null && (!entry.restrictTo || validate(target, entry.restrictTo))) { - await entry.action(target, matches); - return true; + if (matches !== null) { + let success = false; + for (let target of targets) { + if (!entry.restrictTo || validate(target, entry.restrictTo)) { + await entry.action(target, matches); + success = true; + } + } + return success; } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 59c77f1c9..fa49e7175 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -123,14 +123,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }); } - constructor(props: SubCollectionViewProps) { - super(props); - let fixed = DictationManager.Instance.registerStatic; - fixed(["Unset Fit To Container", "Set Fit To Container"], this.fitToContainer); - fixed(["Arrange contents in grid"], this.arrangeContents); - fixed(["Analyze Strokes"], this.analyzeStrokes); - } - @computed get fieldExtensionDoc() { return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true"); } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 39574db0f..51661d1ae 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -41,7 +41,6 @@ import { ClientUtils } from '../../util/ClientUtils'; import { EditableView } from '../EditableView'; import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons'; import { DocumentDecorations } from '../DocumentDecorations'; -import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; import DictationManager from '../../util/DictationManager'; import { CollectionViewType } from '../collections/CollectionBaseView'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? diff --git a/src/client/views/search/FilterBox.tsx b/src/client/views/search/FilterBox.tsx index 995ddd5c3..3e8582d61 100644 --- a/src/client/views/search/FilterBox.tsx +++ b/src/client/views/search/FilterBox.tsx @@ -384,7 +384,7 @@ export class FilterBox extends React.Component {
Collection Filters Active
: undefined}
- ) + ); } // Useful queries: diff --git a/src/client/views/search/SearchItem.tsx b/src/client/views/search/SearchItem.tsx index 0390359b3..6fbc92007 100644 --- a/src/client/views/search/SearchItem.tsx +++ b/src/client/views/search/SearchItem.tsx @@ -134,7 +134,7 @@ export class LinkContextMenu extends React.Component {
- ) + ); } } diff --git a/src/client/views/search/ToggleBar.tsx b/src/client/views/search/ToggleBar.tsx index a30104089..ed5ecd3ba 100644 --- a/src/client/views/search/ToggleBar.tsx +++ b/src/client/views/search/ToggleBar.tsx @@ -59,7 +59,7 @@ export class ToggleBar extends React.Component{ this._forwardTimeline.play(); this._forwardTimeline.reverse(); this.props.handleChange(); - console.log(this.props.getStatus()) + console.log(this.props.getStatus()); } @action.bound -- cgit v1.2.3-70-g09d2 From 810b7898b279069fb8d7cc666333d02e71e23fd2 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 1 Aug 2019 02:39:33 -0400 Subject: naming tweak --- src/client/views/GlobalKeyHandler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 59d120974..99345f04e 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -114,15 +114,15 @@ export default class KeyManager { let main = MainView.Instance; main.dictationOverlayVisible = true; main.isListening = true; - let manager = DictationManager.Instance; - let command = await manager.listen(); + let dictation = DictationManager.Instance; + let command = await dictation.listen(); main.isListening = false; if (!command) { break; } command = command.toLowerCase(); main.dictatedPhrase = command; - main.dictationSuccess = await manager.execute(command); + main.dictationSuccess = await dictation.execute(command); main.overlayTimeout = setTimeout(() => { main.dictationOverlayVisible = false; main.dictationSuccess = undefined; -- cgit v1.2.3-70-g09d2 From c48345b74fadc558062ced96591f041eb4f2729d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 1 Aug 2019 15:55:07 -0400 Subject: cognitive services --- src/client/cognitive_services/CognitiveServices.ts | 32 ++++++++++++++-------- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 8 ++++-- 3 files changed, 27 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index c118d91d3..08fcb4883 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -7,9 +7,9 @@ import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; -type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; +type APIManager = { converter: BodyConverter, requester: RequestExecutor }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; -type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => any; +type AnalysisApplier = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any; type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; @@ -38,7 +38,7 @@ export enum Confidence { */ export namespace CognitiveServices { - const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise> => { + const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { @@ -46,7 +46,7 @@ export namespace CognitiveServices { return undefined; } - let results: Opt; + let results: any; try { results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); } catch { @@ -99,7 +99,11 @@ export namespace CognitiveServices { return request.post(options); }, - analyzer: async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { + }; + + export namespace Appliers { + + export const ProcessImage: AnalysisApplier = async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { let batch = UndoManager.StartBatch("Image Analysis"); let storageKey = keys[0]; @@ -107,7 +111,7 @@ export namespace CognitiveServices { return; } let toStore: any; - let results = await ExecuteQuery(service, Manager, url); + let results = await ExecuteQuery(service, Manager, url); if (!results) { toStore = "Cognitive Services could not process the given image URL."; } else { @@ -120,9 +124,9 @@ export namespace CognitiveServices { target[storageKey] = toStore; batch.end(); - } + }; - }; + } export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; @@ -179,10 +183,14 @@ export namespace CognitiveServices { return new Promise(promisified); }, - analyzer: async (target: Doc, keys: string[], inkData: InkData) => { + }; + + export namespace Appliers { + + export const ConcatenateHandwriting: AnalysisApplier = async (target: Doc, keys: string[], inkData: InkData) => { let batch = UndoManager.StartBatch("Ink Analysis"); - let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); + let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); @@ -192,9 +200,9 @@ export namespace CognitiveServices { } batch.end(); - } + }; - }; + } export interface AzureStrokeData { id: number; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index fa49e7175..9535a6f78 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -555,7 +555,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return; } let relevantKeys = ["inkAnalysis", "handwriting"]; - CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); + CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.fieldExtensionDoc, relevantKeys, data.inkData); } onContextMenu = (e: React.MouseEvent) => { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 9a0615d68..879a91fa1 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -237,7 +237,9 @@ export class ImageBox extends DocComponent(ImageD results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); return faceDocs; }; - CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + if (this.url) { + CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + } } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { @@ -256,7 +258,9 @@ export class ImageBox extends DocComponent(ImageD tagDoc.confidence = threshold; return tagDoc; }; - CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + if (this.url) { + CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + } } @action -- cgit v1.2.3-70-g09d2 From 982961c71fdd9d5dcd02ea33a2b631076a6a1f4b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 13:31:12 -0400 Subject: refactor --- src/client/util/DictationManager.ts | 307 +++++++++++---------- src/client/views/GlobalKeyHandler.ts | 14 +- .../views/collections/CollectionBaseView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 4 +- 4 files changed, 165 insertions(+), 162 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b0866a826..b51d309a1 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -11,183 +11,190 @@ import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; -namespace CORE { - export interface IWindow extends Window { - webkitSpeechRecognition: any; - } -} - -const ConstructorMap = new Map([ - [DocumentType.COL, listSpec(Doc)], - [DocumentType.AUDIO, AudioField], - [DocumentType.IMG, ImageField], - [DocumentType.HIST, HistogramField], - [DocumentType.IMPORT, listSpec(Doc)] -]); - -const tryCast = (view: DocumentView, type: DocumentType) => { - let ctor = ConstructorMap.get(type); - if (!ctor) { - return false; - } - return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; -}; +export namespace DictationManager { -const validate = (target: DocumentView, types: DocumentType[]) => { - for (let type of types) { - if (tryCast(target, type)) { - return true; + namespace CORE { + export interface IWindow extends Window { + webkitSpeechRecognition: any; } } - return false; -}; - -const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; -export type IndependentAction = (target: DocumentView) => any | Promise; -export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; -export type RegistrationEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; -export type ActionPredicate = (target: DocumentView) => boolean; -export type RegexEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] }; - -export default class DictationManager { - public static Instance = new DictationManager(); - private recognizer: any; - private isListening = false; - - constructor() { - this.recognizer = new webkitSpeechRecognition(); - this.recognizer.interimResults = false; - this.recognizer.continuous = true; - } - finish = (handler: any, data: any) => { - handler(data); - this.stop(); - } + const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; - stop = () => { - this.recognizer.stop(); - this.isListening = false; - } + let isListening = false; + const recognizer = (() => { + let initialized = new webkitSpeechRecognition(); + initialized.interimResults = false; + initialized.continuous = true; + return initialized; + })(); - listen = () => { - if (this.isListening) { - return undefined; - } - this.isListening = true; - this.recognizer.start(); - return new Promise((resolve, reject) => { - this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript); - this.recognizer.onerror = (e: any) => this.finish(reject, e); - }); - } + export namespace Controls { + + export const listen = () => { + if (isListening) { + return undefined; + } + isListening = true; + recognizer.start(); + return new Promise((resolve, reject) => { + recognizer.onresult = (e: any) => { + resolve(e.results[0][0].transcript); + stop(); + }; + recognizer.onerror = (e: any) => { + reject(e); + stop(); + }; + }); + }; + + export const stop = () => { + recognizer.stop(); + isListening = false; + }; - public interpretNumber = (number: string) => { - let initial = parseInt(number); - if (!isNaN(initial)) { - return initial; - } - let converted = converter.wordsToNumbers(number, { fuzzy: true }); - if (converted === null) { - return NaN; - } - return typeof converted === "string" ? parseInt(converted) : converted; } - @undoBatch - public execute = async (phrase: string) => { - let targets = SelectionManager.SelectedDocuments(); - if (!targets || !targets.length) { - return; - } + export namespace Commands { - let entry = RegisteredCommands.Independent.get(phrase); - if (entry) { - let success = false; - for (let target of targets) { - if (!entry.restrictTo || validate(target, entry.restrictTo)) { - await entry.action(target); - success = true; - } + export type IndependentAction = (target: DocumentView) => any | Promise; + export type IndependentEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; + + export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; + export type DependentEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] }; + + export const RegisterIndependent = (key: string, value: IndependentEntry) => Independent.set(key, value); + export const RegisterDependent = (entry: DependentEntry) => Dependent.push(entry); + + export const execute = async (phrase: string) => { + let targets = SelectionManager.SelectedDocuments(); + if (!targets || !targets.length) { + return; } - return success; - } - for (let entry of RegisteredCommands.Dependent) { - let regex = entry.expression; - let matches = regex.exec(phrase); - regex.lastIndex = 0; - if (matches !== null) { + let entry = Independent.get(phrase); + if (entry) { let success = false; + let restrictTo = entry.restrictTo; for (let target of targets) { - if (!entry.restrictTo || validate(target, entry.restrictTo)) { - await entry.action(target, matches); + if (!restrictTo || validate(target, restrictTo)) { + await entry.action(target); success = true; } } return success; } - } - return false; - } + for (let entry of Dependent) { + let regex = entry.expression; + let matches = regex.exec(phrase); + regex.lastIndex = 0; + if (matches !== null) { + let success = false; + let restrictTo = entry.restrictTo; + for (let target of targets) { + if (!restrictTo || validate(target, restrictTo)) { + await entry.action(target, matches); + success = true; + } + } + return success; + } + } -} + return false; + }; + + const ConstructorMap = new Map([ + [DocumentType.COL, listSpec(Doc)], + [DocumentType.AUDIO, AudioField], + [DocumentType.IMG, ImageField], + [DocumentType.HIST, HistogramField], + [DocumentType.IMPORT, listSpec(Doc)] + ]); + + const tryCast = (view: DocumentView, type: DocumentType) => { + let ctor = ConstructorMap.get(type); + if (!ctor) { + return false; + } + return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; + }; -export namespace RegisteredCommands { + const validate = (target: DocumentView, types: DocumentType[]) => { + for (let type of types) { + if (tryCast(target, type)) { + return true; + } + } + return false; + }; - export const Independent = new Map([ + const interpretNumber = (number: string) => { + let initial = parseInt(number); + if (!isNaN(initial)) { + return initial; + } + let converted = converter.wordsToNumbers(number, { fuzzy: true }); + if (converted === null) { + return NaN; + } + return typeof converted === "string" ? parseInt(converted) : converted; + }; - ["clear", { - action: (target: DocumentView) => { - Doc.GetProto(target.props.Document).data = new List(); - }, - restrictTo: [DocumentType.COL] - }], + const Independent = new Map([ - ["open fields", { - action: (target: DocumentView) => { - let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); - target.props.addDocTab(kvp, target.dataDoc, "onRight"); - } - }] - - ]); - - export const Dependent = new Array( - - { - expression: /create (\w+) documents of type (image|nested collection)/g, - action: (target: DocumentView, matches: RegExpExecArray) => { - let count = DictationManager.Instance.interpretNumber(matches[1]); - let what = matches[2]; - let dataDoc = Doc.GetProto(target.props.Document); - let fieldKey = "data"; - for (let i = 0; i < count; i++) { - let created: Doc | undefined; - switch (what) { - case "image": - created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); - break; - case "nested collection": - created = Docs.Create.FreeformDocument([], {}); - break; - } - created && Doc.AddDocToList(dataDoc, fieldKey, created); + ["clear", { + action: (target: DocumentView) => Doc.GetProto(target.props.Document).data = new List(), + restrictTo: [DocumentType.COL] + }], + + ["open fields", { + action: (target: DocumentView) => { + let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); + target.props.addDocTab(kvp, target.dataDoc, "onRight"); } + }] + + ]); + + const Dependent = new Array( + + { + expression: /create (\w+) documents of type (image|nested collection)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { + let count = interpretNumber(matches[1]); + let what = matches[2]; + let dataDoc = Doc.GetProto(target.props.Document); + let fieldKey = "data"; + for (let i = 0; i < count; i++) { + let created: Doc | undefined; + switch (what) { + case "image": + created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); + break; + case "nested collection": + created = Docs.Create.FreeformDocument([], {}); + break; + } + created && Doc.AddDocToList(dataDoc, fieldKey, created); + } + }, + restrictTo: [DocumentType.COL] }, - restrictTo: [DocumentType.COL] - }, - - { - expression: /view as (freeform|stacking|masonry|schema|tree)/g, - action: (target: DocumentView, matches: RegExpExecArray) => { - let mode = CollectionViewType.ValueOf(matches[1]); - mode && (target.props.Document.viewType = mode); - }, - restrictTo: [DocumentType.COL] - } - ); + { + expression: /view as (freeform|stacking|masonry|schema|tree)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { + let mode = CollectionViewType.valueOf(matches[1]); + mode && (target.props.Document.viewType = mode); + }, + restrictTo: [DocumentType.COL] + } + + ); + + } } \ No newline at end of file diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 99345f04e..609136bb5 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -5,10 +5,7 @@ import { MainView } from "./MainView"; import { DragManager } from "../util/DragManager"; import { action, runInAction } from "mobx"; import { Doc } from "../../new_fields/Doc"; -import { CognitiveServices } from "../cognitive_services/CognitiveServices"; -import DictationManager from "../util/DictationManager"; -import { ContextMenu } from "./ContextMenu"; -import { ContextMenuProps } from "./ContextMenuItem"; +import { DictationManager } from "../util/DictationManager"; const modifiers = ["control", "meta", "shift", "alt"]; type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise; @@ -74,7 +71,7 @@ export default class KeyManager { } main.toggleColorPicker(true); SelectionManager.DeselectAll(); - DictationManager.Instance.stop(); + DictationManager.Controls.stop(); main.dictationOverlayVisible = false; main.dictationSuccess = undefined; main.overlayTimeout && clearTimeout(main.overlayTimeout); @@ -114,19 +111,18 @@ export default class KeyManager { let main = MainView.Instance; main.dictationOverlayVisible = true; main.isListening = true; - let dictation = DictationManager.Instance; - let command = await dictation.listen(); + let command = await DictationManager.Controls.listen(); main.isListening = false; if (!command) { break; } command = command.toLowerCase(); main.dictatedPhrase = command; - main.dictationSuccess = await dictation.execute(command); + main.dictationSuccess = await DictationManager.Commands.execute(command); main.overlayTimeout = setTimeout(() => { main.dictationOverlayVisible = false; main.dictationSuccess = undefined; - }, 3000); + }, 2000); stopPropagation = true; preventDefault = true; } diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx index 3f88ed98c..24604c812 100644 --- a/src/client/views/collections/CollectionBaseView.tsx +++ b/src/client/views/collections/CollectionBaseView.tsx @@ -34,7 +34,7 @@ export namespace CollectionViewType { ["masonry", CollectionViewType.Masonry] ]); - export const ValueOf = (value: string) => { + export const valueOf = (value: string) => { return stringMapping.get(value.toLowerCase()); }; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 51661d1ae..c5d526a5a 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -41,7 +41,7 @@ import { ClientUtils } from '../../util/ClientUtils'; import { EditableView } from '../EditableView'; import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons'; import { DocumentDecorations } from '../DocumentDecorations'; -import DictationManager from '../../util/DictationManager'; +import { DictationManager } from '../../util/DictationManager'; import { CollectionViewType } from '../collections/CollectionBaseView'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? @@ -539,7 +539,7 @@ export class DocumentView extends DocComponent(Docu } listen = async () => { - let transcript = await DictationManager.Instance.listen(); + let transcript = await DictationManager.Controls.listen(); transcript && (Doc.GetProto(this.props.Document).transcript = transcript); } -- cgit v1.2.3-70-g09d2 From 30444bd06dbd00c948749fa739e7a42cef87ec85 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 14:05:33 -0400 Subject: added final or interim listning --- src/client/util/DictationManager.ts | 35 +++++++++++++++++++++++++++-------- src/client/views/GlobalKeyHandler.ts | 2 +- 2 files changed, 28 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b51d309a1..2ed74a729 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -2,7 +2,7 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; import { undoBatch } from "./UndoManager"; import * as converter from "words-to-numbers"; -import { Doc } from "../../new_fields/Doc"; +import { Doc, Opt } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs, DocumentType } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; @@ -18,40 +18,59 @@ export namespace DictationManager { webkitSpeechRecognition: any; } } - const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; let isListening = false; const recognizer = (() => { let initialized = new webkitSpeechRecognition(); - initialized.interimResults = false; initialized.continuous = true; return initialized; })(); export namespace Controls { - export const listen = () => { + export type InterimResultHandler = (results: any) => any; + + export const listen = (handler: Opt = undefined) => { if (isListening) { return undefined; } isListening = true; + + recognizer.interimResults = handler !== undefined; recognizer.start(); + return new Promise((resolve, reject) => { - recognizer.onresult = (e: any) => { - resolve(e.results[0][0].transcript); - stop(); - }; recognizer.onerror = (e: any) => { reject(e); stop(); }; + if (handler) { + let newestResult: string; + recognizer.onresult = (e: any) => { + newestResult = e.results[0][0].transcript; + handler(newestResult); + }; + recognizer.onend = (e: any) => { + resolve(newestResult); + stop(); + }; + } else { + recognizer.onresult = (e: any) => { + let finalResult = e.results[0][0].transcript; + resolve(finalResult); + stop(); + }; + } }); }; export const stop = () => { recognizer.stop(); isListening = false; + recognizer.onresult = undefined; + recognizer.onend = undefined; + recognizer.onerror = undefined; }; } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 609136bb5..b1ea92bb8 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -111,7 +111,7 @@ export default class KeyManager { let main = MainView.Instance; main.dictationOverlayVisible = true; main.isListening = true; - let command = await DictationManager.Controls.listen(); + let command = await DictationManager.Controls.listen((results: any) => console.log(results)); main.isListening = false; if (!command) { break; -- cgit v1.2.3-70-g09d2 From 813476d753f0286d546a0b1c9f1b1774481604e0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 17:17:43 -0400 Subject: final refactor and commenting --- src/client/util/DictationManager.ts | 29 +++++++++++++++++++++++++++-- src/client/views/GlobalKeyHandler.ts | 14 +++++++------- src/client/views/MainView.tsx | 16 ++++++++++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 2ed74a729..b799238ba 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,6 +1,6 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; -import { undoBatch } from "./UndoManager"; +import { UndoManager } from "./UndoManager"; import * as converter from "words-to-numbers"; import { Doc, Opt } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; @@ -11,8 +11,26 @@ import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; +/** + * This namespace provides a singleton instance of a manager that + * handles the listening and text-conversion of user speech. + * + * The basic manager functionality can be attained by the DictationManager.Controls namespace, which provide + * a simple recording operation that returns the interpreted text as a string. + * + * Additionally, however, the DictationManager also exposes the ability to execute voice commands within Dash. + * It stores a default library of registered commands that can be triggered by listen()'ing for a phrase and then + * passing the results into the execute() function. + * + * In addition to compile-time default commands, you can invoke DictationManager.Commands.Register(Independent|Dependent) + * to add new commands as classes or components are constructed. + */ export namespace DictationManager { + /** + * Some type maneuvering to access Webkit's built-in + * speech recognizer. + */ namespace CORE { export interface IWindow extends Window { webkitSpeechRecognition: any; @@ -24,6 +42,7 @@ export namespace DictationManager { const recognizer = (() => { let initialized = new webkitSpeechRecognition(); initialized.continuous = true; + initialized.language = "en-US"; return initialized; })(); @@ -77,6 +96,8 @@ export namespace DictationManager { export namespace Commands { + export const dictationFadeDuration = 2000; + export type IndependentAction = (target: DocumentView) => any | Promise; export type IndependentEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; @@ -91,14 +112,16 @@ export namespace DictationManager { if (!targets || !targets.length) { return; } - + phrase = phrase.toLowerCase(); let entry = Independent.get(phrase); if (entry) { let success = false; let restrictTo = entry.restrictTo; for (let target of targets) { if (!restrictTo || validate(target, restrictTo)) { + let batch = UndoManager.StartBatch("Independent Command"); await entry.action(target); + batch.end(); success = true; } } @@ -114,7 +137,9 @@ export namespace DictationManager { let restrictTo = entry.restrictTo; for (let target of targets) { if (!restrictTo || validate(target, restrictTo)) { + let batch = UndoManager.StartBatch("Dependent Command"); await entry.action(target, matches); + batch.end(); success = true; } } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index b1ea92bb8..82289c249 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -74,7 +74,7 @@ export default class KeyManager { DictationManager.Controls.stop(); main.dictationOverlayVisible = false; main.dictationSuccess = undefined; - main.overlayTimeout && clearTimeout(main.overlayTimeout); + main.cancelDictationFade(); break; case "delete": case "backspace": @@ -110,19 +110,19 @@ export default class KeyManager { case " ": let main = MainView.Instance; main.dictationOverlayVisible = true; + main.isListening = true; let command = await DictationManager.Controls.listen((results: any) => console.log(results)); main.isListening = false; + if (!command) { break; } - command = command.toLowerCase(); - main.dictatedPhrase = command; + + main.dictatedPhrase = command = command.toLowerCase(); main.dictationSuccess = await DictationManager.Commands.execute(command); - main.overlayTimeout = setTimeout(() => { - main.dictationOverlayVisible = false; - main.dictationSuccess = undefined; - }, 2000); + main.initiateDictationFade(); + stopPropagation = true; preventDefault = true; } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 4a5e4a3d1..748e1e634 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,6 +39,7 @@ import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; +import { DictationManager } from '../util/DictationManager'; @observer export class MainView extends React.Component { @@ -55,6 +56,21 @@ export class MainView extends React.Component { public overlayTimeout: NodeJS.Timeout | undefined; + public initiateDictationFade = () => { + let duration = DictationManager.Commands.dictationFadeDuration; + this.overlayTimeout = setTimeout(() => { + this.dictationOverlayVisible = false; + this.dictationSuccess = undefined; + }, duration); + } + + public cancelDictationFade = () => { + if (this.overlayTimeout) { + clearTimeout(this.overlayTimeout); + this.overlayTimeout = undefined; + } + } + @computed private get mainContainer(): Opt { return FieldValue(Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc)); } -- cgit v1.2.3-70-g09d2 From 6588455dfa091a1e925b4eeee5d018e23731b491 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 18:23:18 -0400 Subject: undoable, name change --- src/client/util/DictationManager.ts | 62 ++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b799238ba..b02a5ecbe 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,7 +1,7 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; -import { UndoManager } from "./UndoManager"; -import * as converter from "words-to-numbers"; +import { UndoManager, undoBatch } from "./UndoManager"; +import * as interpreter from "words-to-numbers"; import { Doc, Opt } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs, DocumentType } from "../documents/Documents"; @@ -108,46 +108,46 @@ export namespace DictationManager { export const RegisterDependent = (entry: DependentEntry) => Dependent.push(entry); export const execute = async (phrase: string) => { - let targets = SelectionManager.SelectedDocuments(); - if (!targets || !targets.length) { - return; - } - phrase = phrase.toLowerCase(); - let entry = Independent.get(phrase); - if (entry) { - let success = false; - let restrictTo = entry.restrictTo; - for (let target of targets) { - if (!restrictTo || validate(target, restrictTo)) { - let batch = UndoManager.StartBatch("Independent Command"); - await entry.action(target); - batch.end(); - success = true; - } + return UndoManager.RunInBatch(async () => { + let targets = SelectionManager.SelectedDocuments(); + if (!targets || !targets.length) { + return; } - return success; - } - for (let entry of Dependent) { - let regex = entry.expression; - let matches = regex.exec(phrase); - regex.lastIndex = 0; - if (matches !== null) { + phrase = phrase.toLowerCase(); + let entry = Independent.get(phrase); + + if (entry) { let success = false; let restrictTo = entry.restrictTo; for (let target of targets) { if (!restrictTo || validate(target, restrictTo)) { - let batch = UndoManager.StartBatch("Dependent Command"); - await entry.action(target, matches); - batch.end(); + await entry.action(target); success = true; } } return success; } - } - return false; + for (let entry of Dependent) { + let regex = entry.expression; + let matches = regex.exec(phrase); + regex.lastIndex = 0; + if (matches !== null) { + let success = false; + let restrictTo = entry.restrictTo; + for (let target of targets) { + if (!restrictTo || validate(target, restrictTo)) { + await entry.action(target, matches); + success = true; + } + } + return success; + } + } + + return false; + }, "Execute Command"); }; const ConstructorMap = new Map([ @@ -180,7 +180,7 @@ export namespace DictationManager { if (!isNaN(initial)) { return initial; } - let converted = converter.wordsToNumbers(number, { fuzzy: true }); + let converted = interpreter.wordsToNumbers(number, { fuzzy: true }); if (converted === null) { return NaN; } -- cgit v1.2.3-70-g09d2 From d6fda11588f1a117e8acc30ea5600d34ff22e01b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 21:29:56 -0400 Subject: now differentiate between continuous, indefinite and, separately, interim vs final only dictation results --- src/client/util/DictationManager.ts | 81 +++++++++++++++++++++------------ src/client/views/GlobalKeyHandler.ts | 3 +- src/client/views/MainView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 3 +- src/new_fields/Doc.ts | 2 +- 5 files changed, 59 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b02a5ecbe..89797f101 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -39,57 +39,82 @@ export namespace DictationManager { const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; let isListening = false; - const recognizer = (() => { - let initialized = new webkitSpeechRecognition(); - initialized.continuous = true; - initialized.language = "en-US"; - return initialized; - })(); + let isManuallyStopped = false; + const recognizer: SpeechRecognition = new webkitSpeechRecognition(); export namespace Controls { + let newestResult: string; export type InterimResultHandler = (results: any) => any; + export type ContinuityArgs = { indefinite: boolean } | false; + export interface ListeningOptions { + language: string; + continuous: ContinuityArgs; + interimHandler: InterimResultHandler; + delimiter: string; + } - export const listen = (handler: Opt = undefined) => { + export const listen = (options?: Partial) => { if (isListening) { return undefined; } isListening = true; + let handler = options ? options.interimHandler : undefined; + let continuous = options ? options.continuous : undefined; + let language = options ? options.language : undefined; + let delimiter = options ? options.delimiter : undefined; + recognizer.interimResults = handler !== undefined; + recognizer.continuous = continuous === undefined ? false : continuous !== false; + recognizer.lang = language === undefined ? "en-US" : language; + recognizer.start(); return new Promise((resolve, reject) => { + recognizer.onerror = (e: any) => { reject(e); stop(); }; - if (handler) { - let newestResult: string; - recognizer.onresult = (e: any) => { - newestResult = e.results[0][0].transcript; - handler(newestResult); - }; - recognizer.onend = (e: any) => { + + recognizer.onresult = (e: SpeechRecognitionEvent) => { + newestResult = synthesize(e, delimiter); + handler && handler(newestResult); + }; + + recognizer.onend = (e: Event) => { + if (continuous && continuous.indefinite && !isManuallyStopped) { + recognizer.start(); + } else { resolve(newestResult); - stop(); - }; - } else { - recognizer.onresult = (e: any) => { - let finalResult = e.results[0][0].transcript; - resolve(finalResult); - stop(); - }; - } + reset(); + } + }; + }); }; - export const stop = () => { - recognizer.stop(); + export const stop = (saveCumulative = true) => { + saveCumulative ? recognizer.stop() : recognizer.abort(); + reset(); + }; + + const reset = () => { isListening = false; - recognizer.onresult = undefined; - recognizer.onend = undefined; - recognizer.onerror = undefined; + isManuallyStopped = false; + recognizer.onresult = null; + recognizer.onend = null; + recognizer.onerror = null; + }; + + const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { + let results = e.results; + let transcripts: string[] = []; + for (let i = 0; i < results.length; i++) { + transcripts.push(results.item(i).item(0).transcript.trim()); + } + return transcripts.join(delimiter || "..."); }; } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 82289c249..98df43a1e 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -112,7 +112,8 @@ export default class KeyManager { main.dictationOverlayVisible = true; main.isListening = true; - let command = await DictationManager.Controls.listen((results: any) => console.log(results)); + // let printer = (results: any) => console.log(results); + let command = await DictationManager.Controls.listen(); main.isListening = false; if (!command) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 748e1e634..5cec34293 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -49,7 +49,7 @@ export class MainView extends React.Component { @observable public pwidth: number = 0; @observable public pheight: number = 0; - @observable private dictationState = "Listening..."; + @observable private dictationState = ""; @observable private dictationSuccessState: boolean | undefined = undefined; @observable private dictationDisplayState = false; @observable private dictationListeningState = false; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index c5d526a5a..bd87bf21b 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -539,7 +539,8 @@ export class DocumentView extends DocComponent(Docu } listen = async () => { - let transcript = await DictationManager.Controls.listen(); + let options = { continuous: { indefinite: true }, delimiter: " " }; + let transcript = await DictationManager.Controls.listen(options); transcript && (Doc.GetProto(this.props.Document).transcript = transcript); } diff --git a/src/new_fields/Doc.ts b/src/new_fields/Doc.ts index 84b8589dd..979574f16 100644 --- a/src/new_fields/Doc.ts +++ b/src/new_fields/Doc.ts @@ -531,7 +531,7 @@ export namespace Doc { d.layout = detailLayout; d.nativeWidth = d.nativeHeight = undefined; if (detailLayout instanceof Doc) { - let delegDetailLayout = Doc.MakeDelegate(detailLayout) as Doc; + let delegDetailLayout = Doc.MakeDelegate(detailLayout); d.layout = delegDetailLayout; let subDetailLayout1 = await PromiseValue(delegDetailLayout.detailedLayout); let subDetailLayout = await PromiseValue(delegDetailLayout.detailedLayout); -- cgit v1.2.3-70-g09d2 From 133d890f74e6ad927dda366015fd3a4ddb7497c8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 21:37:53 -0400 Subject: improved error handling --- src/client/util/DictationManager.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 89797f101..d022fea2e 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -54,7 +54,18 @@ export namespace DictationManager { delimiter: string; } - export const listen = (options?: Partial) => { + export const listen = async (options?: Partial) => { + let results: any; + try { + results = await listenImpl(options); + } catch (e) { + results = "Dication Error: "; + results += e instanceof SpeechRecognitionError ? e.error : "unknown error"; + } + return results; + }; + + const listenImpl = (options?: Partial) => { if (isListening) { return undefined; } -- cgit v1.2.3-70-g09d2 From 30e3964bd24ffcd3b0fa77b917a2a68bd34be6b5 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 21:39:44 -0400 Subject: clean up --- src/client/util/DictationManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index d022fea2e..fd5ee25a1 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,8 +1,8 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; -import { UndoManager, undoBatch } from "./UndoManager"; +import { UndoManager } from "./UndoManager"; import * as interpreter from "words-to-numbers"; -import { Doc, Opt } from "../../new_fields/Doc"; +import { Doc } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs, DocumentType } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; @@ -114,9 +114,9 @@ export namespace DictationManager { const reset = () => { isListening = false; isManuallyStopped = false; + recognizer.onerror = null; recognizer.onresult = null; recognizer.onend = null; - recognizer.onerror = null; }; const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { -- cgit v1.2.3-70-g09d2 From 056a03812f202c9588c25d543ffa316060b97b84 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 22:07:56 -0400 Subject: tweaks to indefinite transcription --- src/client/util/DictationManager.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index fd5ee25a1..fa34ca9ad 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -40,6 +40,7 @@ export namespace DictationManager { let isListening = false; let isManuallyStopped = false; + const results: string[] = []; const recognizer: SpeechRecognition = new webkitSpeechRecognition(); export namespace Controls { @@ -59,8 +60,8 @@ export namespace DictationManager { try { results = await listenImpl(options); } catch (e) { - results = "Dication Error: "; - results += e instanceof SpeechRecognitionError ? e.error : "unknown error"; + results = "Dictation Error: "; + results += "error" in e ? e.error : "unknown error"; } return results; }; @@ -91,14 +92,16 @@ export namespace DictationManager { recognizer.onresult = (e: SpeechRecognitionEvent) => { newestResult = synthesize(e, delimiter); + continuous && continuous.indefinite && results.push(newestResult); handler && handler(newestResult); }; recognizer.onend = (e: Event) => { - if (continuous && continuous.indefinite && !isManuallyStopped) { + let indefinite = continuous && continuous.indefinite; + if (indefinite && !isManuallyStopped) { recognizer.start(); } else { - resolve(newestResult); + resolve(indefinite ? newestResult : results.join(delimiter)); reset(); } }; -- cgit v1.2.3-70-g09d2 From 62e541e21b5675039283a515d9b1bba02b62e432 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 03:18:51 -0400 Subject: handled all edge cases for continuous recording --- src/client/util/DictationManager.ts | 70 +++++++++++++++++++++++------------- src/client/views/ContextMenuItem.tsx | 2 +- src/client/views/GlobalKeyHandler.ts | 8 +++-- 3 files changed, 52 insertions(+), 28 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index fa34ca9ad..5f443f99e 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -38,14 +38,17 @@ export namespace DictationManager { } const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; - let isListening = false; - let isManuallyStopped = false; - const results: string[] = []; - const recognizer: SpeechRecognition = new webkitSpeechRecognition(); - export namespace Controls { - let newestResult: string; + const defaultDelimiter = "..."; + let isListening = false; + let isManuallyStopped = false; + let sessionResults: string[] = []; + + const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition(); + recognizer.onstart = () => console.log("initiating speech recognition session..."); + + let current: string | undefined = undefined; export type InterimResultHandler = (results: any) => any; export type ContinuityArgs = { indefinite: boolean } | false; export interface ListeningOptions { @@ -60,8 +63,7 @@ export namespace DictationManager { try { results = await listenImpl(options); } catch (e) { - results = "Dictation Error: "; - results += "error" in e ? e.error : "unknown error"; + results = `Dictation Error: ${"error" in e ? e.error : "unknown error"}`; } return results; }; @@ -74,6 +76,7 @@ export namespace DictationManager { let handler = options ? options.interimHandler : undefined; let continuous = options ? options.continuous : undefined; + let indefinite = continuous && continuous.indefinite; let language = options ? options.language : undefined; let delimiter = options ? options.delimiter : undefined; @@ -85,41 +88,56 @@ export namespace DictationManager { return new Promise((resolve, reject) => { - recognizer.onerror = (e: any) => { - reject(e); - stop(); + recognizer.onerror = (e: SpeechRecognitionError) => { + if (!(indefinite && e.error === "no-speech")) { + stop(true); + reject(e); + } }; recognizer.onresult = (e: SpeechRecognitionEvent) => { - newestResult = synthesize(e, delimiter); - continuous && continuous.indefinite && results.push(newestResult); - handler && handler(newestResult); + current = synthesize(e, delimiter); + handler && handler(current); + isManuallyStopped && complete(); }; recognizer.onend = (e: Event) => { - let indefinite = continuous && continuous.indefinite; - if (indefinite && !isManuallyStopped) { - recognizer.start(); + if (!indefinite || isManuallyStopped) { + return complete(); + } + + if (current) { + sessionResults.push(current); + current = undefined; + } + recognizer.start(); + }; + + let complete = () => { + if (indefinite) { + current && sessionResults.push(current); + resolve(connect(sessionResults, delimiter)); } else { - resolve(indefinite ? newestResult : results.join(delimiter)); - reset(); + resolve(current); } + reset(); }; }); }; - export const stop = (saveCumulative = true) => { - saveCumulative ? recognizer.stop() : recognizer.abort(); - reset(); + export const stop = (errorTriggered = false) => { + !errorTriggered && (isManuallyStopped = true); + recognizer.stop(); }; const reset = () => { isListening = false; isManuallyStopped = false; - recognizer.onerror = null; recognizer.onresult = null; + recognizer.onerror = null; recognizer.onend = null; + sessionResults = []; }; const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { @@ -128,7 +146,11 @@ export namespace DictationManager { for (let i = 0; i < results.length; i++) { transcripts.push(results.item(i).item(0).transcript.trim()); } - return transcripts.join(delimiter || "..."); + return transcripts.join(delimiter || defaultDelimiter); + }; + + const connect = (sessions: string[], delimiter?: string) => { + return sessions.map(text => `(${text})`).join(delimiter || defaultDelimiter); }; } diff --git a/src/client/views/ContextMenuItem.tsx b/src/client/views/ContextMenuItem.tsx index a1787e78f..90f7be33f 100644 --- a/src/client/views/ContextMenuItem.tsx +++ b/src/client/views/ContextMenuItem.tsx @@ -39,13 +39,13 @@ export class ContextMenuItem extends React.Component) => { if ("event" in this.props) { + this.props.closeMenu && this.props.closeMenu(); let batch: UndoManager.Batch | undefined; if (this.props.undoable !== false) { batch = UndoManager.StartBatch(`Context menu event: ${this.props.description}`); } await this.props.event(); batch && batch.end(); - this.props.closeMenu && this.props.closeMenu(); } } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 98df43a1e..c3e6ae6c8 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -72,9 +72,11 @@ export default class KeyManager { main.toggleColorPicker(true); SelectionManager.DeselectAll(); DictationManager.Controls.stop(); - main.dictationOverlayVisible = false; - main.dictationSuccess = undefined; - main.cancelDictationFade(); + if (main.dictationOverlayVisible) { + main.dictationOverlayVisible = false; + main.dictationSuccess = undefined; + main.cancelDictationFade(); + } break; case "delete": case "backspace": -- cgit v1.2.3-70-g09d2 From 8d6620dd6818c82363ec0c4d49a92b0e2c0c1036 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 03:30:41 -0400 Subject: clean up --- src/client/util/DictationManager.ts | 46 +++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 5f443f99e..b6f871713 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -40,22 +40,27 @@ export namespace DictationManager { export namespace Controls { - const defaultDelimiter = "..."; + const intraSession = ". "; + const interSession = " ... "; + let isListening = false; let isManuallyStopped = false; + + let current: string | undefined = undefined; let sessionResults: string[] = []; const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition(); recognizer.onstart = () => console.log("initiating speech recognition session..."); - let current: string | undefined = undefined; export type InterimResultHandler = (results: any) => any; export type ContinuityArgs = { indefinite: boolean } | false; + export type DelimiterArgs = { inter: string, intra: string }; + export interface ListeningOptions { language: string; continuous: ContinuityArgs; + delimiters: DelimiterArgs; interimHandler: InterimResultHandler; - delimiter: string; } export const listen = async (options?: Partial) => { @@ -78,7 +83,8 @@ export namespace DictationManager { let continuous = options ? options.continuous : undefined; let indefinite = continuous && continuous.indefinite; let language = options ? options.language : undefined; - let delimiter = options ? options.delimiter : undefined; + let intra = options && options.delimiters ? options.delimiters.intra : undefined; + let inter = options && options.delimiters ? options.delimiters.inter : undefined; recognizer.interimResults = handler !== undefined; recognizer.continuous = continuous === undefined ? false : continuous !== false; @@ -90,13 +96,13 @@ export namespace DictationManager { recognizer.onerror = (e: SpeechRecognitionError) => { if (!(indefinite && e.error === "no-speech")) { - stop(true); + recognizer.stop(); reject(e); } }; recognizer.onresult = (e: SpeechRecognitionEvent) => { - current = synthesize(e, delimiter); + current = synthesize(e, intra); handler && handler(current); isManuallyStopped && complete(); }; @@ -116,7 +122,7 @@ export namespace DictationManager { let complete = () => { if (indefinite) { current && sessionResults.push(current); - resolve(connect(sessionResults, delimiter)); + resolve(sessionResults.join(inter || interSession)); } else { resolve(current); } @@ -126,18 +132,9 @@ export namespace DictationManager { }); }; - export const stop = (errorTriggered = false) => { - !errorTriggered && (isManuallyStopped = true); - recognizer.stop(); - }; - - const reset = () => { - isListening = false; - isManuallyStopped = false; - recognizer.onresult = null; - recognizer.onerror = null; - recognizer.onend = null; - sessionResults = []; + export const stop = (salvageSession = true) => { + isManuallyStopped = true; + salvageSession ? recognizer.stop() : recognizer.abort(); }; const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { @@ -146,11 +143,16 @@ export namespace DictationManager { for (let i = 0; i < results.length; i++) { transcripts.push(results.item(i).item(0).transcript.trim()); } - return transcripts.join(delimiter || defaultDelimiter); + return transcripts.join(delimiter || intraSession); }; - const connect = (sessions: string[], delimiter?: string) => { - return sessions.map(text => `(${text})`).join(delimiter || defaultDelimiter); + const reset = () => { + isListening = false; + isManuallyStopped = false; + recognizer.onresult = null; + recognizer.onerror = null; + recognizer.onend = null; + sessionResults = []; }; } -- cgit v1.2.3-70-g09d2 From 6d718c8a243e68d23199d35592bfded285385c91 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 05:09:56 -0400 Subject: now all listen() calls display recording UI --- src/client/util/DictationManager.ts | 24 ++++++++++++++++++++---- src/client/views/GlobalKeyHandler.ts | 21 ++++----------------- src/client/views/Main.scss | 1 - src/client/views/MainView.tsx | 11 ++++++++--- src/client/views/nodes/DocumentView.tsx | 16 +++++++++++++--- 5 files changed, 45 insertions(+), 28 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b6f871713..a882994c1 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -10,6 +10,7 @@ import { Cast, CastCtor } from "../../new_fields/Types"; import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; +import { MainView } from "../views/MainView"; /** * This namespace provides a singleton instance of a manager that @@ -37,9 +38,11 @@ export namespace DictationManager { } } const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; + export const placeholder = "Listening..."; export namespace Controls { + const infringe = "unable to process: dictation manager still involved in previous session"; const intraSession = ". "; const interSession = " ... "; @@ -52,7 +55,7 @@ export namespace DictationManager { const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition(); recognizer.onstart = () => console.log("initiating speech recognition session..."); - export type InterimResultHandler = (results: any) => any; + export type InterimResultHandler = (results: string) => any; export type ContinuityArgs = { indefinite: boolean } | false; export type DelimiterArgs = { inter: string, intra: string }; @@ -61,21 +64,33 @@ export namespace DictationManager { continuous: ContinuityArgs; delimiters: DelimiterArgs; interimHandler: InterimResultHandler; + tryExecute: boolean; } export const listen = async (options?: Partial) => { - let results: any; + let results: string | undefined; + MainView.Instance.dictationOverlayVisible = true; + MainView.Instance.isListening = true; try { results = await listenImpl(options); + if (results) { + MainView.Instance.isListening = false; + MainView.Instance.dictatedPhrase = results = results.toLowerCase(); + MainView.Instance.dictationSuccess = options && options.tryExecute ? await DictationManager.Commands.execute(results) : true; + } } catch (e) { - results = `Dictation Error: ${"error" in e ? e.error : "unknown error"}`; + MainView.Instance.isListening = false; + MainView.Instance.dictatedPhrase = results = `dictation error: ${"error" in e ? e.error : "unknown error"}`; + MainView.Instance.dictationSuccess = false; + } finally { + MainView.Instance.initiateDictationFade(); } return results; }; const listenImpl = (options?: Partial) => { if (isListening) { - return undefined; + return infringe; } isListening = true; @@ -126,6 +141,7 @@ export namespace DictationManager { } else { resolve(current); } + current = undefined; reset(); }; diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index c3e6ae6c8..0989e8db1 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -73,9 +73,11 @@ export default class KeyManager { SelectionManager.DeselectAll(); DictationManager.Controls.stop(); if (main.dictationOverlayVisible) { + main.cancelDictationFade(); main.dictationOverlayVisible = false; + main.isListening = true; + main.dictatedPhrase = ""; main.dictationSuccess = undefined; - main.cancelDictationFade(); } break; case "delete": @@ -110,22 +112,7 @@ export default class KeyManager { switch (keyname) { case " ": - let main = MainView.Instance; - main.dictationOverlayVisible = true; - - main.isListening = true; - // let printer = (results: any) => console.log(results); - let command = await DictationManager.Controls.listen(); - main.isListening = false; - - if (!command) { - break; - } - - main.dictatedPhrase = command = command.toLowerCase(); - main.dictationSuccess = await DictationManager.Commands.execute(command); - main.initiateDictationFade(); - + DictationManager.Controls.listen({ tryExecute: true }); stopPropagation = true; preventDefault = true; } diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss index 8e57b88c3..f76abaff3 100644 --- a/src/client/views/Main.scss +++ b/src/client/views/Main.scss @@ -292,7 +292,6 @@ ul#add-options-list { width: 100%; height: 100%; position: absolute; - background: darkslategray; z-index: 999; transition: 0.5s all ease; pointer-events: none; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 4443eea6d..383efa1e3 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,7 +1,7 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; import { faArrowDown, faCloudUploadAlt, faArrowUp, faClone, faCheck, faPlay, faPause, faCaretUp, faLongArrowAltRight, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faPortrait, faMusic, faObjectGroup, faPenNib, faRedoAlt, faTable, faThumbtack, faTree, faUndoAlt, faCat, faBolt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, configure, observable, runInAction, reaction, trace } from 'mobx'; +import { action, computed, configure, observable, runInAction, reaction, trace, autorun } from 'mobx'; import { observer } from 'mobx-react'; import "normalize.css"; import * as React from 'react'; @@ -167,6 +167,8 @@ export class MainView extends React.Component { } } + autorun(() => console.log(`this.isListening = ${this.isListening}`)); + library.add(faFont); library.add(faExclamation); library.add(faPortrait); @@ -523,7 +525,7 @@ export class MainView extends React.Component { render() { let display = this.dictationOverlayVisible; let success = this.dictationSuccess; - let result = this.isListening ? "Listening..." : `"${this.dictatedPhrase}"`; + let result = this.isListening ? DictationManager.placeholder : `"${this.dictatedPhrase}"`; return (
{result}
{this.mainContent} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 1d9cb3c80..a415aefda 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -38,6 +38,7 @@ import "./DocumentView.scss"; import { FormattedTextBox } from './FormattedTextBox'; import React = require("react"); import { DictationManager } from '../../util/DictationManager'; +import { MainView } from '../MainView'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -536,9 +537,18 @@ export class DocumentView extends DocComponent(Docu } listen = async () => { - let options = { continuous: { indefinite: true }, delimiter: " " }; - let transcript = await DictationManager.Controls.listen(options); - transcript && (Doc.GetProto(this.props.Document).transcript = transcript); + let dataDoc = Doc.GetProto(this.props.Document); + let options = { + continuous: { indefinite: true }, + delimiter: " ", + interimHandler: (results: string) => { + MainView.Instance.isListening = false; + MainView.Instance.dictationSuccess = true; + MainView.Instance.dictatedPhrase = results; + } + }; + let final = await DictationManager.Controls.listen(options); + final && (dataDoc.transcript = final); } @action -- cgit v1.2.3-70-g09d2 From 1ed381022450bc5c39238c73f732179f4b21daf2 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 05:13:51 -0400 Subject: clean up --- src/client/views/nodes/DocumentView.tsx | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index a415aefda..458fb582d 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -538,16 +538,15 @@ export class DocumentView extends DocComponent(Docu listen = async () => { let dataDoc = Doc.GetProto(this.props.Document); - let options = { - continuous: { indefinite: true }, - delimiter: " ", - interimHandler: (results: string) => { - MainView.Instance.isListening = false; - MainView.Instance.dictationSuccess = true; - MainView.Instance.dictatedPhrase = results; - } + let handler = (results: string) => { + MainView.Instance.isListening = false; + MainView.Instance.dictationSuccess = true; + MainView.Instance.dictatedPhrase = results; }; - let final = await DictationManager.Controls.listen(options); + let final = await DictationManager.Controls.listen({ + continuous: { indefinite: true }, + interimHandler: handler + }); final && (dataDoc.transcript = final); } -- cgit v1.2.3-70-g09d2 From 95ff1649631457449bdc580a5a47937718c00e3b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 05:19:14 -0400 Subject: factored out stop UI code --- src/client/util/DictationManager.ts | 10 ++++++++++ src/client/views/GlobalKeyHandler.ts | 7 ------- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index a882994c1..2af7c53cb 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -149,8 +149,18 @@ export namespace DictationManager { }; export const stop = (salvageSession = true) => { + if (!isListening) { + return; + } isManuallyStopped = true; salvageSession ? recognizer.stop() : recognizer.abort(); + if (MainView.Instance.dictationOverlayVisible) { + MainView.Instance.cancelDictationFade(); + MainView.Instance.dictationOverlayVisible = false; + MainView.Instance.isListening = true; + MainView.Instance.dictatedPhrase = ""; + MainView.Instance.dictationSuccess = undefined; + } }; const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 0989e8db1..e773014e3 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -72,13 +72,6 @@ export default class KeyManager { main.toggleColorPicker(true); SelectionManager.DeselectAll(); DictationManager.Controls.stop(); - if (main.dictationOverlayVisible) { - main.cancelDictationFade(); - main.dictationOverlayVisible = false; - main.isListening = true; - main.dictatedPhrase = ""; - main.dictationSuccess = undefined; - } break; case "delete": case "backspace": -- cgit v1.2.3-70-g09d2 From 0010f88e1002feb14ecfb111c2c6ae56ee34cf2d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 13:32:19 -0400 Subject: clean up and UI fix --- src/client/util/DictationManager.ts | 43 +++++++++++++++++++-------------- src/client/views/MainView.tsx | 11 ++++----- src/client/views/nodes/DocumentView.tsx | 20 ++++++--------- 3 files changed, 38 insertions(+), 36 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 2af7c53cb..ee1f11b4f 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -58,6 +58,7 @@ export namespace DictationManager { export type InterimResultHandler = (results: string) => any; export type ContinuityArgs = { indefinite: boolean } | false; export type DelimiterArgs = { inter: string, intra: string }; + export type ListeningUIStatus = { interim: boolean } | false; export interface ListeningOptions { language: string; @@ -69,22 +70,28 @@ export namespace DictationManager { export const listen = async (options?: Partial) => { let results: string | undefined; - MainView.Instance.dictationOverlayVisible = true; - MainView.Instance.isListening = true; + let main = MainView.Instance; + + main.dictationOverlayVisible = true; + let interim = options !== undefined && options.interimHandler !== undefined; + main.isListening = { interim: interim }; + try { results = await listenImpl(options); if (results) { - MainView.Instance.isListening = false; - MainView.Instance.dictatedPhrase = results = results.toLowerCase(); - MainView.Instance.dictationSuccess = options && options.tryExecute ? await DictationManager.Commands.execute(results) : true; + main.isListening = false; + main.dictatedPhrase = results; + let execute = options && options.tryExecute; + main.dictationSuccess = execute ? await DictationManager.Commands.execute(results) : true; } } catch (e) { - MainView.Instance.isListening = false; - MainView.Instance.dictatedPhrase = results = `dictation error: ${"error" in e ? e.error : "unknown error"}`; - MainView.Instance.dictationSuccess = false; + main.isListening = false; + main.dictatedPhrase = results = `dictation error: ${"error" in e ? e.error : "unknown error"}`; + main.dictationSuccess = false; } finally { - MainView.Instance.initiateDictationFade(); + main.initiateDictationFade(); } + return results; }; @@ -137,11 +144,10 @@ export namespace DictationManager { let complete = () => { if (indefinite) { current && sessionResults.push(current); - resolve(sessionResults.join(inter || interSession)); + sessionResults.length && resolve(sessionResults.join(inter || interSession)); } else { resolve(current); } - current = undefined; reset(); }; @@ -154,12 +160,12 @@ export namespace DictationManager { } isManuallyStopped = true; salvageSession ? recognizer.stop() : recognizer.abort(); - if (MainView.Instance.dictationOverlayVisible) { - MainView.Instance.cancelDictationFade(); - MainView.Instance.dictationOverlayVisible = false; - MainView.Instance.isListening = true; - MainView.Instance.dictatedPhrase = ""; - MainView.Instance.dictationSuccess = undefined; + let main = MainView.Instance; + if (main.dictationOverlayVisible) { + main.cancelDictationFade(); + main.dictationOverlayVisible = false; + main.dictationSuccess = undefined; + setTimeout(() => main.dictatedPhrase = placeholder, 500); } }; @@ -173,12 +179,13 @@ export namespace DictationManager { }; const reset = () => { + current = undefined; + sessionResults = []; isListening = false; isManuallyStopped = false; recognizer.onresult = null; recognizer.onerror = null; recognizer.onend = null; - sessionResults = []; }; } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 383efa1e3..631d24cb1 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -49,10 +49,10 @@ export class MainView extends React.Component { @observable public pwidth: number = 0; @observable public pheight: number = 0; - @observable private dictationState = ""; + @observable private dictationState = DictationManager.placeholder; @observable private dictationSuccessState: boolean | undefined = undefined; @observable private dictationDisplayState = false; - @observable private dictationListeningState = false; + @observable private dictationListeningState: DictationManager.Controls.ListeningUIStatus = false; public overlayTimeout: NodeJS.Timeout | undefined; @@ -61,6 +61,7 @@ export class MainView extends React.Component { this.overlayTimeout = setTimeout(() => { this.dictationOverlayVisible = false; this.dictationSuccess = undefined; + setTimeout(() => this.dictatedPhrase = DictationManager.placeholder, 500); }, duration); } @@ -116,7 +117,7 @@ export class MainView extends React.Component { return this.dictationListeningState; } - public set isListening(value: boolean) { + public set isListening(value: DictationManager.Controls.ListeningUIStatus) { runInAction(() => this.dictationListeningState = value); } @@ -167,8 +168,6 @@ export class MainView extends React.Component { } } - autorun(() => console.log(`this.isListening = ${this.isListening}`)); - library.add(faFont); library.add(faExclamation); library.add(faPortrait); @@ -525,7 +524,7 @@ export class MainView extends React.Component { render() { let display = this.dictationOverlayVisible; let success = this.dictationSuccess; - let result = this.isListening ? DictationManager.placeholder : `"${this.dictatedPhrase}"`; + let result = this.isListening && !this.isListening.interim ? DictationManager.placeholder : `"${this.dictatedPhrase}"`; return (
(Docu this.props.Document.lockedPosition = BoolCast(this.props.Document.lockedPosition) ? undefined : true; } - listen = async () => { - let dataDoc = Doc.GetProto(this.props.Document); - let handler = (results: string) => { - MainView.Instance.isListening = false; - MainView.Instance.dictationSuccess = true; - MainView.Instance.dictatedPhrase = results; - }; - let final = await DictationManager.Controls.listen({ + listen = async () => + Doc.GetProto(this.props.Document).transcript = await DictationManager.Controls.listen({ continuous: { indefinite: true }, - interimHandler: handler - }); - final && (dataDoc.transcript = final); - } + interimHandler: (results: string) => { + MainView.Instance.isListening = { interim: true }; + MainView.Instance.dictationSuccess = true; + MainView.Instance.dictatedPhrase = results; + } + }) @action onContextMenu = async (e: React.MouseEvent): Promise => { -- cgit v1.2.3-70-g09d2 From 857e98b16f693db968b04ede62e3f864f8b2ed35 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 14:04:22 -0400 Subject: final UI tweaks --- src/client/util/DictationManager.ts | 5 ++--- src/client/views/MainView.tsx | 8 ++++---- src/client/views/nodes/DocumentView.tsx | 12 +++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index ee1f11b4f..978889830 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -73,15 +73,14 @@ export namespace DictationManager { let main = MainView.Instance; main.dictationOverlayVisible = true; - let interim = options !== undefined && options.interimHandler !== undefined; - main.isListening = { interim: interim }; + main.isListening = { interim: false }; try { results = await listenImpl(options); if (results) { main.isListening = false; - main.dictatedPhrase = results; let execute = options && options.tryExecute; + main.dictatedPhrase = execute ? results.toLowerCase() : results; main.dictationSuccess = execute ? await DictationManager.Commands.execute(results) : true; } } catch (e) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 631d24cb1..618e83bfa 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -89,7 +89,7 @@ export class MainView extends React.Component { } } - public get dictatedPhrase() { + @computed public get dictatedPhrase() { return this.dictationState; } @@ -97,7 +97,7 @@ export class MainView extends React.Component { runInAction(() => this.dictationState = value); } - public get dictationSuccess() { + @computed public get dictationSuccess() { return this.dictationSuccessState; } @@ -105,7 +105,7 @@ export class MainView extends React.Component { runInAction(() => this.dictationSuccessState = value); } - public get dictationOverlayVisible() { + @computed public get dictationOverlayVisible() { return this.dictationDisplayState; } @@ -113,7 +113,7 @@ export class MainView extends React.Component { runInAction(() => this.dictationDisplayState = value); } - public get isListening() { + @computed public get isListening() { return this.dictationListeningState; } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index d6ae17537..57e66ff1b 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -536,15 +536,17 @@ export class DocumentView extends DocComponent(Docu this.props.Document.lockedPosition = BoolCast(this.props.Document.lockedPosition) ? undefined : true; } - listen = async () => + listen = async () => { Doc.GetProto(this.props.Document).transcript = await DictationManager.Controls.listen({ continuous: { indefinite: true }, interimHandler: (results: string) => { - MainView.Instance.isListening = { interim: true }; - MainView.Instance.dictationSuccess = true; - MainView.Instance.dictatedPhrase = results; + let main = MainView.Instance; + main.dictationSuccess = true; + main.dictatedPhrase = results; + main.isListening = { interim: true }; } - }) + }); + } @action onContextMenu = async (e: React.MouseEvent): Promise => { -- cgit v1.2.3-70-g09d2 From ac6d9fe0b327436729ceeaee2acecf6ea8f2fa3d Mon Sep 17 00:00:00 2001 From: kimdahey Date: Sun, 4 Aug 2019 20:53:50 -0400 Subject: copy to clipboard --- .vscode/launch.json | 3 +++ package.json | 3 ++- src/client/util/DictationManager.ts | 12 +++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/.vscode/launch.json b/.vscode/launch.json index e4196600e..d2c18d6f1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,6 +12,9 @@ "breakOnLoad": true, "url": "http://localhost:1050/login", "webRoot": "${workspaceFolder}", + "runtimeArgs": [ + "--experimental-modules" + ] }, { "type": "firefox", diff --git a/package.json b/package.json index 3d0d5c3d7..97980f0ca 100644 --- a/package.json +++ b/package.json @@ -117,8 +117,8 @@ "bluebird": "^3.5.3", "body-parser": "^1.18.3", "bootstrap": "^4.3.1", - "child_process": "^1.0.2", "canvas": "^2.5.0", + "child_process": "^1.0.2", "class-transformer": "^0.2.0", "connect-flash": "^0.1.1", "connect-mongo": "^2.0.3", @@ -211,6 +211,7 @@ "socket.io-client": "^2.2.0", "solr-node": "^1.2.1", "standard-http-error": "^2.0.1", + "ts-clipboard": "^1.0.17", "typescript-collections": "^1.3.2", "url-loader": "^1.1.2", "uuid": "^3.3.2", diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 978889830..3668ca0d2 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -11,6 +11,7 @@ import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; import { MainView } from "../views/MainView"; +import { Clipboard } from "ts-clipboard"; /** * This namespace provides a singleton instance of a manager that @@ -78,6 +79,7 @@ export namespace DictationManager { try { results = await listenImpl(options); if (results) { + Clipboard.copy(results); main.isListening = false; let execute = options && options.tryExecute; main.dictatedPhrase = execute ? results.toLowerCase() : results; @@ -250,7 +252,8 @@ export namespace DictationManager { [DocumentType.AUDIO, AudioField], [DocumentType.IMG, ImageField], [DocumentType.HIST, HistogramField], - [DocumentType.IMPORT, listSpec(Doc)] + [DocumentType.IMPORT, listSpec(Doc)], + [DocumentType.TEXT, "string"] ]); const tryCast = (view: DocumentView, type: DocumentType) => { @@ -294,6 +297,13 @@ export namespace DictationManager { let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); target.props.addDocTab(kvp, target.dataDoc, "onRight"); } + }], + + ["promote", { + action: (target: DocumentView) => { + console.log(target); + }, + restrictTo: [DocumentType.TEXT] }] ]); -- cgit v1.2.3-70-g09d2 From 2978e805156bffad92e8ac23259843a7cf207fb7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 6 Aug 2019 11:11:26 -0400 Subject: beginning link to doc in context --- src/client/util/DictationManager.ts | 4 ++-- src/client/views/nodes/DocumentView.tsx | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 3668ca0d2..9c61fe125 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -11,7 +11,7 @@ import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; import { MainView } from "../views/MainView"; -import { Clipboard } from "ts-clipboard"; +import { Utils } from "../../Utils"; /** * This namespace provides a singleton instance of a manager that @@ -79,7 +79,7 @@ export namespace DictationManager { try { results = await listenImpl(options); if (results) { - Clipboard.copy(results); + Utils.CopyText(results); main.isListening = false; let execute = options && options.tryExecute; main.dictatedPhrase = execute ? results.toLowerCase() : results; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 57e66ff1b..e5b0b0b52 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -602,6 +602,14 @@ export class DocumentView extends DocComponent(Docu if (!ClientUtils.RELEASE) { let copies: ContextMenuProps[] = []; copies.push({ description: "Copy URL", event: () => Utils.CopyText(Utils.prepend("/doc/" + this.props.Document[Id])), icon: "link" }); + copies.push({ + description: "Copy Context", event: () => { + let parent = this.props.ContainingCollectionView; + if (parent) { + Utils.CopyText(Utils.prepend("/doc/" + parent.props.Document[Id])); + } + }, icon: "link" + }); copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" }); cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" }); } -- cgit v1.2.3-70-g09d2 From 572c4196e0f41ec6bae8cae403812f9b97d5a3c7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 6 Aug 2019 13:27:47 -0400 Subject: post factored out, buxton pivot viewer route --- src/Utils.ts | 11 +++++++++++ src/client/views/nodes/DocumentView.tsx | 1 + src/server/index.ts | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+) (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index 502540eb0..c1737a084 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -3,6 +3,7 @@ import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; import { RouteStore } from './server/RouteStore'; +import requestPromise = require('request-promise'); export class Utils { @@ -173,4 +174,14 @@ export namespace JSONUtils { return results; } +} + +export function PostToServer(relativeRoute: string, body: any) { + let options = { + method: "POST", + uri: Utils.prepend(relativeRoute), + json: true, + body: body + }; + return requestPromise.post(options); } \ No newline at end of file diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 547d858ce..dbbf95479 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -39,6 +39,7 @@ import { FormattedTextBox } from './FormattedTextBox'; import React = require("react"); import { DictationManager } from '../../util/DictationManager'; import { MainView } from '../MainView'; +import requestPromise = require('request-promise'); const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); diff --git a/src/server/index.ts b/src/server/index.ts index 10a84c823..1e811c1b2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -148,6 +148,25 @@ app.get("/pull", (req, res) => res.redirect("/"); })); +app.get("/buxton/:clear", (req, res) => { + if (req.params.clear === "true") { + deleteFields().then(() => upload_buxton_docs(res)); + } else { + upload_buxton_docs(res); + } +}); + +let upload_buxton_docs = (res: Response) => { + let buxton_scraping = path.join(__dirname, '../scraping/buxton'); + exec('python scraper.py', { cwd: buxton_scraping }, (err, stdout, sterr) => { + if (err) { + res.send(err.message); + return; + } + res.redirect("/"); + }); +}; + app.get("/version", (req, res) => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout, stderr) => { if (err) { -- cgit v1.2.3-70-g09d2 From d8c065935a0cb5a89b6d76407b7729f8584424eb Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 6 Aug 2019 22:03:59 -0400 Subject: pivot view --- src/client/views/Main.tsx | 2 + src/client/views/collections/CollectionView.tsx | 1 + .../collectionFreeForm/CollectionFreeFormView.tsx | 32 +++++++++++++++ src/scraping/buxton/scraper.py | 15 +++---- src/scraping/buxton/scripts/initialization.txt | 46 ++++++++++++++++++++++ src/scraping/buxton/scripts/layout.txt | 1 + src/server/index.ts | 29 +++++++++----- src/server/remapUrl.ts | 2 + 8 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 src/scraping/buxton/scripts/initialization.txt create mode 100644 src/scraping/buxton/scripts/layout.txt (limited to 'src') diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 1cf13aa74..daa778ed3 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,6 +7,7 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; +import { PivotView } from "./collections/collectionFreeForm/CollectionFreeFormView"; let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); @@ -33,6 +34,7 @@ let swapDocs = async () => { DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); await CurrentUserUtils.loadUserDocument(info); + await PivotView.loadLayouts(); // updates old user documents to prevent chrome on tree view. (await Cast(CurrentUserUtils.UserDocument.workspaces, Doc))!.chromeStatus = "disabled"; (await Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc))!.chromeStatus = "disabled"; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index f59fee985..59572b7e7 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -97,6 +97,7 @@ export class CollectionView extends React.Component { switch (this.props.Document.viewType) { case CollectionViewType.Freeform: { subItems.push({ description: "Custom", icon: "fingerprint", event: CollectionFreeFormView.AddCustomLayout(this.props.Document, this.props.fieldKey) }); + subItems.push({ description: "Pivot", icon: "fingerprint", event: () => CollectionFreeFormView.SetPivotLayout(this.props.Document) }); break; } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 7ba13e961..17c4e83b0 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -37,6 +37,7 @@ import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); import { DocumentType, Docs } from "../../../documents/Documents"; +import { RouteStore } from "../../../../server/RouteStore"; library.add(faEye as any, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBraille, faChalkboard); @@ -48,6 +49,20 @@ export const panZoomSchema = createSchema({ arrangeInit: ScriptField, }); +export namespace PivotView { + + export let arrangeInit: string; + export let arrangeScript: string; + + export async function loadLayouts() { + let response = await fetch(Utils.prepend("/layoutscripts")); + let scripts = JSON.parse(await response.text()); + arrangeInit = scripts[0]; + arrangeScript = scripts[1]; + } + +} + type PanZoomDocument = makeInterface<[typeof panZoomSchema, typeof positionSchema, typeof pageSchema]>; const PanZoomDocument = makeInterface(panZoomSchema, positionSchema, pageSchema); @@ -787,6 +802,23 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }; } + public static SetPivotLayout = (target: Doc) => { + let setSpecifiedLayoutField = (originalText: string, key: string, params: Record, requiredType?: string) => { + const script = CompileScript(originalText, { + params, + requiredType, + typecheck: false + }); + if (!script.compiled) { + console.log(script.errors.map(error => error.messageText).join("\n")); + return; + } + target[key] = new ScriptField(script); + }; + setSpecifiedLayoutField(PivotView.arrangeInit, "arrangeInit", { collection: "Doc", docs: "Doc[]" }, undefined); + setSpecifiedLayoutField(PivotView.arrangeScript, "arrangeScript", { doc: "Doc", index: "number", collection: "Doc", state: "any", docs: "Doc[]" }, "{x: number, y: number, width?: number, height?: number}"); + } + render() { const easing = () => this.props.Document.panTransformType === "Ease"; Doc.UpdateDocumentExtensionForField(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey); diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index f0f45d8f9..1c19118fd 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -17,6 +17,7 @@ dist = "../../server/public/files" db = MongoClient("localhost", 27017)["Dash"] target_collection = db.newDocuments +target_doc_title = "Workspace 1" schema_guids = [] common_proto_id = "" @@ -69,7 +70,7 @@ def text_doc_map(string_list): return listify(proxify_guids(list(map(guid_map, string_list)))) -def write_schema(parse_results, display_fields, storage_key): +def write_collection(parse_results, display_fields, storage_key, viewType=2): view_guids = parse_results["child_guids"] data_doc = parse_results["schema"] @@ -90,7 +91,7 @@ def write_schema(parse_results, display_fields, storage_key): "zoomBasis": 1, "zIndex": 2, "libraryBrush": False, - "viewType": 2 + "viewType": viewType }, "__type": "Doc" } @@ -237,7 +238,7 @@ def parse_document(file_name: str): copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) copyfile(dir_path + "/" + image, dir_path + - "/" + image.replace(".", "_m.", 1)) + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") def sanitize(line): return re.sub("[\n\t]+", "", line).replace(u"\u00A0", " ").replace( @@ -381,22 +382,22 @@ candidates = 0 for file_name in os.listdir(source): if file_name.endswith('.docx'): candidates += 1 - schema_guids.append(write_schema( + schema_guids.append(write_collection( parse_document(file_name), ["title", "data"], "image_data")) print("writing parent schema...") -parent_guid = write_schema({ +parent_guid = write_collection({ "schema": { "_id": guid(), "fields": {}, "__type": "Doc" }, "child_guids": schema_guids -}, ["title", "short_description", "original_price"], "data") +}, ["title", "short_description", "original_price"], "data", 1) print("appending parent schema to main workspace...\n") target_collection.update_one( - {"fields.title": "WS collection 1"}, + {"fields.title": target_doc_title}, {"$push": {"fields.data.fields": {"fieldId": parent_guid, "__type": "proxy"}}} ) diff --git a/src/scraping/buxton/scripts/initialization.txt b/src/scraping/buxton/scripts/initialization.txt new file mode 100644 index 000000000..53f3f0d53 --- /dev/null +++ b/src/scraping/buxton/scripts/initialization.txt @@ -0,0 +1,46 @@ +const field = collection.pivotField || "title"; +const width = collection.pivotWidth || 200; + +const groups = new Map; + +for (const doc of docs) { + const val = doc[field]; + if (val === undefined) continue; + + const l = groups.get(val); + if (l) { + l.push(doc); + } else { + groups.set(val, [doc]); + } + +} + +let minSize = Infinity; + +groups.forEach((val, key) => { + minSize = Math.min(minSize, val.length); +}); + +const numCols = collection.pivotNumColumns || Math.ceil(Math.sqrt(minSize)); + +const docMap = new Map; +const groupNames = []; + +let x = 0; +groups.forEach((val, key) => { + let y = 0; + let xCount = 0; + groupNames.push({type:"text", text:String(key), x, y:width + 50, width: width * 1.25 * numCols, height:100, fontSize:collection.pivotFontSize}); + for (const doc of val) { + docMap.set(doc, {x: x + xCount * width * 1.25, y:-y, width, height:width}); + xCount++; + if (xCount >= numCols) { + xCount = 0; + y += width * 1.25; + } + } + x += width * 1.25 * (numCols + 1); +}); + +return {state:{ map: docMap}, views:groupNames }; \ No newline at end of file diff --git a/src/scraping/buxton/scripts/layout.txt b/src/scraping/buxton/scripts/layout.txt new file mode 100644 index 000000000..46b6dbaac --- /dev/null +++ b/src/scraping/buxton/scripts/layout.txt @@ -0,0 +1 @@ +return state.map.get(doc) \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 1e811c1b2..d735a6d8c 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -148,24 +148,33 @@ app.get("/pull", (req, res) => res.redirect("/"); })); -app.get("/buxton/:clear", (req, res) => { - if (req.params.clear === "true") { - deleteFields().then(() => upload_buxton_docs(res)); - } else { - upload_buxton_docs(res); - } -}); - -let upload_buxton_docs = (res: Response) => { +app.get("/buxton", (req, res) => { let buxton_scraping = path.join(__dirname, '../scraping/buxton'); exec('python scraper.py', { cwd: buxton_scraping }, (err, stdout, sterr) => { if (err) { res.send(err.message); return; } + console.log(stdout); res.redirect("/"); }); -}; +}); + +app.get('/layoutscripts', (req, res) => { + let scripts: string[] = []; + let handler = (err: NodeJS.ErrnoException | null, data: Buffer) => { + if (err) { + console.log(err.message); + return; + } + scripts.push(data.toString()); + if (scripts.length === 2) { + res.send(JSON.stringify(scripts)); + } + }; + fs.readFile(path.join(__dirname, '../scraping/buxton/scripts/initialization.txt'), handler); + fs.readFile(path.join(__dirname, '../scraping/buxton/scripts/layout.txt'), handler); +}); app.get("/version", (req, res) => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout, stderr) => { diff --git a/src/server/remapUrl.ts b/src/server/remapUrl.ts index 69c766d56..5218a239a 100644 --- a/src/server/remapUrl.ts +++ b/src/server/remapUrl.ts @@ -2,6 +2,8 @@ import { Database } from "./database"; import { Search } from "./Search"; import * as path from 'path'; +//npx ts-node src/server/remapUrl.ts + const suffixMap: { [type: string]: true } = { "video": true, "pdf": true, -- cgit v1.2.3-70-g09d2 From 4d3962cf221cb2421835c7016562433077e7b200 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 7 Aug 2019 01:48:15 -0400 Subject: pivot view script retrieval and context menu behavior improvements --- src/client/views/collections/CollectionView.tsx | 14 ++++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 12 ++--- src/scraping/buxton/source/Silverlight.dmg | Bin 0 -> 15097394 bytes src/server/index.ts | 52 ++++++++++++--------- 4 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 src/scraping/buxton/source/Silverlight.dmg (limited to 'src') diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index 59572b7e7..60ede17e7 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -1,6 +1,6 @@ import { library } from '@fortawesome/fontawesome-svg-core'; import { faEye } from '@fortawesome/free-regular-svg-icons'; -import { faColumns, faEllipsisV, faFingerprint, faImage, faProjectDiagram, faSignature, faSquare, faTh, faThList, faTree } from '@fortawesome/free-solid-svg-icons'; +import { faColumns, faEllipsisV, faFingerprint, faImage, faProjectDiagram, faSignature, faSquare, faTh, faThList, faTree, faCopy } from '@fortawesome/free-solid-svg-icons'; import { action, IReactionDisposer, observable, reaction, runInAction } from 'mobx'; import { observer } from "mobx-react"; import * as React from 'react'; @@ -20,7 +20,7 @@ import { CollectionTreeView } from "./CollectionTreeView"; import { CollectionViewBaseChrome } from './CollectionViewChromes'; export const COLLECTION_BORDER_WIDTH = 2; -library.add(faTh, faTree, faSquare, faProjectDiagram, faSignature, faThList, faFingerprint, faColumns, faEllipsisV, faImage, faEye as any); +library.add(faTh, faTree, faSquare, faProjectDiagram, faSignature, faThList, faFingerprint, faColumns, faEllipsisV, faImage, faEye as any, faCopy); @observer export class CollectionView extends React.Component { @@ -86,7 +86,13 @@ export class CollectionView extends React.Component { onContextMenu = (e: React.MouseEvent): void => { if (!this.isAnnotationOverlay && !e.isPropagationStopped() && this.props.Document[Id] !== CurrentUserUtils.MainDocId) { // need to test this because GoldenLayout causes a parallel hierarchy in the React DOM for its children and the main document view7 let subItems: ContextMenuProps[] = []; - subItems.push({ description: "Freeform", event: () => this.props.Document.viewType = CollectionViewType.Freeform, icon: "signature" }); + subItems.push({ + description: "Freeform", event: () => { + this.props.Document.viewType = CollectionViewType.Freeform; + delete this.props.Document.arrangeInit; + delete this.props.Document.arrangeScript; + }, icon: "signature" + }); if (CollectionBaseView.InSafeMode()) { ContextMenu.Instance.addItem({ description: "Test Freeform", event: () => this.props.Document.viewType = CollectionViewType.Invalid, icon: "project-diagram" }); } @@ -97,7 +103,7 @@ export class CollectionView extends React.Component { switch (this.props.Document.viewType) { case CollectionViewType.Freeform: { subItems.push({ description: "Custom", icon: "fingerprint", event: CollectionFreeFormView.AddCustomLayout(this.props.Document, this.props.fieldKey) }); - subItems.push({ description: "Pivot", icon: "fingerprint", event: () => CollectionFreeFormView.SetPivotLayout(this.props.Document) }); + subItems.push({ description: "Pivot", icon: "copy", event: () => CollectionFreeFormView.SetPivotLayout(this.props.Document) }); break; } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 17c4e83b0..110ac2f25 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -51,14 +51,10 @@ export const panZoomSchema = createSchema({ export namespace PivotView { - export let arrangeInit: string; - export let arrangeScript: string; + export let scripts: { arrangeInit: string, arrangeScript: string }; export async function loadLayouts() { - let response = await fetch(Utils.prepend("/layoutscripts")); - let scripts = JSON.parse(await response.text()); - arrangeInit = scripts[0]; - arrangeScript = scripts[1]; + scripts = JSON.parse(await (await fetch(Utils.prepend("/layoutscripts"))).text()); } } @@ -815,8 +811,8 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } target[key] = new ScriptField(script); }; - setSpecifiedLayoutField(PivotView.arrangeInit, "arrangeInit", { collection: "Doc", docs: "Doc[]" }, undefined); - setSpecifiedLayoutField(PivotView.arrangeScript, "arrangeScript", { doc: "Doc", index: "number", collection: "Doc", state: "any", docs: "Doc[]" }, "{x: number, y: number, width?: number, height?: number}"); + setSpecifiedLayoutField(PivotView.scripts.arrangeInit, "arrangeInit", { collection: "Doc", docs: "Doc[]" }, undefined); + setSpecifiedLayoutField(PivotView.scripts.arrangeScript, "arrangeScript", { doc: "Doc", index: "number", collection: "Doc", state: "any", docs: "Doc[]" }, "{x: number, y: number, width?: number, height?: number}"); } render() { diff --git a/src/scraping/buxton/source/Silverlight.dmg b/src/scraping/buxton/source/Silverlight.dmg new file mode 100644 index 000000000..d88290362 Binary files /dev/null and b/src/scraping/buxton/source/Silverlight.dmg differ diff --git a/src/server/index.ts b/src/server/index.ts index d735a6d8c..a0101e3f8 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,6 +1,6 @@ require('dotenv').config(); import * as bodyParser from 'body-parser'; -import { exec } from 'child_process'; +import { exec, ExecOptions } from 'child_process'; import * as cookieParser from 'cookie-parser'; import * as express from 'express'; import * as session from 'express-session'; @@ -149,31 +149,39 @@ app.get("/pull", (req, res) => })); app.get("/buxton", (req, res) => { - let buxton_scraping = path.join(__dirname, '../scraping/buxton'); - exec('python scraper.py', { cwd: buxton_scraping }, (err, stdout, sterr) => { - if (err) { - res.send(err.message); - return; + let cwd = '../scraping/buxton'; + + let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; + let onRejected = (err: any) => { console.error(err.message); res.send(err); }; + let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); + + command_line('python scraper.py', cwd).then(onResolved, tryPython3); +}); + +const command_line = (command: string, fromDirectory?: string) => { + return new Promise((resolve, reject) => { + let options: ExecOptions = {}; + if (fromDirectory) { + options.cwd = path.join(__dirname, fromDirectory); } - console.log(stdout); - res.redirect("/"); + exec(command, options, (err, stdout) => err ? reject(err) : resolve(stdout)); }); -}); +}; + +const read_text_file = (relativePath: string) => { + let target = path.join(__dirname, relativePath); + return new Promise((resolve, reject) => { + fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString())); + }); +}; app.get('/layoutscripts', (req, res) => { - let scripts: string[] = []; - let handler = (err: NodeJS.ErrnoException | null, data: Buffer) => { - if (err) { - console.log(err.message); - return; - } - scripts.push(data.toString()); - if (scripts.length === 2) { - res.send(JSON.stringify(scripts)); - } - }; - fs.readFile(path.join(__dirname, '../scraping/buxton/scripts/initialization.txt'), handler); - fs.readFile(path.join(__dirname, '../scraping/buxton/scripts/layout.txt'), handler); + let prefix = '../scraping/buxton/scripts/'; + read_text_file(prefix + 'initialization.txt').then(arrangeInit => { + read_text_file(prefix + 'layout.txt').then(arrangeScript => { + res.send(JSON.stringify({ arrangeInit, arrangeScript })); + }); + }); }); app.get("/version", (req, res) => { -- cgit v1.2.3-70-g09d2 From 74cb792cb1ef9fd11b7bd635fb25518903757204 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 7 Aug 2019 01:49:30 -0400 Subject: removed dmg --- src/scraping/buxton/source/Silverlight.dmg | Bin 15097394 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/scraping/buxton/source/Silverlight.dmg (limited to 'src') diff --git a/src/scraping/buxton/source/Silverlight.dmg b/src/scraping/buxton/source/Silverlight.dmg deleted file mode 100644 index d88290362..000000000 Binary files a/src/scraping/buxton/source/Silverlight.dmg and /dev/null differ -- cgit v1.2.3-70-g09d2 From 50d7ffb3303cf06ced84b9788b95578fff2e92da Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 7 Aug 2019 20:14:53 -0400 Subject: pivot viewer native code --- src/client/views/Main.tsx | 2 +- src/client/views/collections/CollectionView.tsx | 5 +- .../views/collections/CollectionViewChromes.tsx | 33 +++- .../collectionFreeForm/CollectionFreeFormView.tsx | 203 +++++++++++++++++---- 4 files changed, 205 insertions(+), 38 deletions(-) (limited to 'src') diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index daa778ed3..8be633f80 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -34,7 +34,7 @@ let swapDocs = async () => { DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); await CurrentUserUtils.loadUserDocument(info); - await PivotView.loadLayouts(); + // await PivotView.loadLayouts(); // updates old user documents to prevent chrome on tree view. (await Cast(CurrentUserUtils.UserDocument.workspaces, Doc))!.chromeStatus = "disabled"; (await Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc))!.chromeStatus = "disabled"; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index 60ede17e7..7a402798e 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -89,8 +89,7 @@ export class CollectionView extends React.Component { subItems.push({ description: "Freeform", event: () => { this.props.Document.viewType = CollectionViewType.Freeform; - delete this.props.Document.arrangeInit; - delete this.props.Document.arrangeScript; + delete this.props.Document.usePivotLayout; }, icon: "signature" }); if (CollectionBaseView.InSafeMode()) { @@ -103,7 +102,7 @@ export class CollectionView extends React.Component { switch (this.props.Document.viewType) { case CollectionViewType.Freeform: { subItems.push({ description: "Custom", icon: "fingerprint", event: CollectionFreeFormView.AddCustomLayout(this.props.Document, this.props.fieldKey) }); - subItems.push({ description: "Pivot", icon: "copy", event: () => CollectionFreeFormView.SetPivotLayout(this.props.Document) }); + subItems.push({ description: "Pivot", icon: "copy", event: () => this.props.Document.usePivotLayout = true }); break; } } diff --git a/src/client/views/collections/CollectionViewChromes.tsx b/src/client/views/collections/CollectionViewChromes.tsx index 1b2561953..e1ac79da6 100644 --- a/src/client/views/collections/CollectionViewChromes.tsx +++ b/src/client/views/collections/CollectionViewChromes.tsx @@ -3,7 +3,7 @@ import { CollectionView } from "./CollectionView"; import "./CollectionViewChromes.scss"; import { CollectionViewType } from "./CollectionBaseView"; import { undoBatch } from "../../util/UndoManager"; -import { action, observable, runInAction, computed, IObservable, IObservableValue } from "mobx"; +import { action, observable, runInAction, computed, IObservable, IObservableValue, reaction, autorun } from "mobx"; import { observer } from "mobx-react"; import { Doc, DocListCast } from "../../../new_fields/Doc"; import { DocLike } from "../MetadataEntryMenu"; @@ -187,6 +187,36 @@ export class CollectionViewBaseChrome extends React.Component { + if (!this.document.usePivotLayout) { + return (null); + } + return () => this.pivotKeyDisplay = e.currentTarget.value)} + onKeyPress={action((e: React.KeyboardEvent) => { + let value = e.currentTarget.value; + if (e.which === 13) { + this.pivotKey = value; + this.pivotKeyDisplay = ""; + } + })} />); + } + render() { return (
@@ -219,6 +249,7 @@ export class CollectionViewBaseChrome extends React.Component { }} onPointerDown={this.openViewSpecs} /> + {this.getPivotInput()}
{ + let collection = target.Document; + const field = StrCast(collection.pivotField) || "title"; + const width = NumCast(collection.pivotWidth) || 200; + + const groups = new Map, Doc[]>(); + + for (const doc of target.childDocs) { + const val = doc[field]; + if (val === undefined) continue; + + const l = groups.get(val); + if (l) { + l.push(doc); + } else { + groups.set(val, [doc]); + } + + } + + let minSize = Infinity; + + groups.forEach((val, key) => { + minSize = Math.min(minSize, val.length); + }); + + const numCols = NumCast(collection.pivotNumColumns) || Math.ceil(Math.sqrt(minSize)); + const fontSize = NumCast(collection.pivotFontSize); + + const docMap = new Map(); + const groupNames: PivotData[] = []; + + let x = 0; + groups.forEach((val, key) => { + let y = 0; + let xCount = 0; + groupNames.push({ + type: "text", + text: String(key), + x, + y: width + 50, + width: width * 1.25 * numCols, + height: 100, fontSize: fontSize + }); + for (const doc of val) { + docMap.set(doc, { + x: x + xCount * width * 1.25, + y: -y, + width, + height: width + }); + xCount++; + if (xCount >= numCols) { + xCount = 0; + y += width * 1.25; + } + } + x += width * 1.25 * (numCols + 1); + }); + + let elements = target.viewDefsToJSX(groupNames); + let curPage = FieldValue(target.Document.curPage, -1); + + let docViews = target.childDocs.filter(doc => doc instanceof Doc).reduce((prev, doc) => { + var page = NumCast(doc.page, -1); + if ((Math.abs(Math.round(page) - Math.round(curPage)) < 3) || page === -1) { + let minim = BoolCast(doc.isMinimized); + if (minim === undefined || !minim) { + let defaultPosition = (): ViewDefBounds => { + return { + x: NumCast(doc.x), + y: NumCast(doc.y), + z: NumCast(doc.z), + width: NumCast(doc.width), + height: NumCast(doc.height) + }; + }; + const pos = docMap.get(doc) || defaultPosition(); + prev.push({ + ele: ( + ), + bounds: { + x: pos.x, + y: pos.y, + z: pos.z, + width: NumCast(pos.width), + height: NumCast(pos.height) + } + }); + } + } + return prev; + }, elements); + + target.resetSelectOnLoaded(); + + return docViews; + }; + } type PanZoomDocument = makeInterface<[typeof panZoomSchema, typeof positionSchema, typeof pageSchema]>; @@ -508,9 +638,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return 1; } - getChildDocumentViewProps(childDocLayout: Doc): DocumentViewProps { - let self = this; let pair = Doc.GetLayoutDataDocPair(this.props.Document, this.props.DataDoc, this.props.fieldKey, childDocLayout); return { DataDoc: pair.data, @@ -569,7 +697,19 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return result.result === undefined ? { x: Cast(doc.x, "number"), y: Cast(doc.y, "number"), z: Cast(doc.z, "number"), width: Cast(doc.width, "number"), height: Cast(doc.height, "number") } : result.result; } - private viewDefToJSX(viewDef: any): { ele: JSX.Element, bounds?: { x: number, y: number, z?: number, width: number, height: number } } | undefined { + viewDefsToJSX = (views: any[]) => { + let elements: ViewDefResult[] = []; + if (Array.isArray(views)) { + elements = views.reduce((prev, ele) => { + const jsx = this.viewDefToJSX(ele); + jsx && prev.push(jsx); + return prev; + }, elements); + } + return elements; + } + + private viewDefToJSX(viewDef: any): Opt { if (viewDef.type === "text") { const text = Cast(viewDef.text, "string"); const x = Cast(viewDef.x, "number"); @@ -598,20 +738,14 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { const script = this.Document.arrangeScript; let state: any = undefined; const docs = this.childDocs; - let elements: { ele: JSX.Element, bounds?: { x: number, y: number, z?: number, width: number, height: number } }[] = []; + let elements: ViewDefResult[] = []; if (initScript) { const initResult = initScript.script.run({ docs, collection: this.Document }); if (initResult.success) { const result = initResult.result; const { state: scriptState, views } = result; state = scriptState; - if (Array.isArray(views)) { - elements = views.reduce((prev, ele) => { - const jsx = this.viewDefToJSX(ele); - jsx && prev.push(jsx); - return prev; - }, elements); - } + elements = this.viewDefsToJSX(views); } } let docviews = docs.filter(doc => doc instanceof Doc).reduce((prev, doc) => { @@ -633,14 +767,17 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return prev; }, elements); - setTimeout(() => this._selectOnLoaded = "", 600);// bcz: surely there must be a better way .... + this.resetSelectOnLoaded(); return docviews; } + resetSelectOnLoaded = () => setTimeout(() => this._selectOnLoaded = "", 600);// bcz: surely there must be a better way .... + @computed.struct get views() { - return this.elements.filter(ele => ele.bounds && !ele.bounds.z).map(ele => ele.ele); + let source = this.Document.usePivotLayout === true ? PivotView.elements(this) : this.elements; + return source.filter(ele => ele.bounds && !ele.bounds.z).map(ele => ele.ele); } @computed.struct get overlayViews() { @@ -798,22 +935,22 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }; } - public static SetPivotLayout = (target: Doc) => { - let setSpecifiedLayoutField = (originalText: string, key: string, params: Record, requiredType?: string) => { - const script = CompileScript(originalText, { - params, - requiredType, - typecheck: false - }); - if (!script.compiled) { - console.log(script.errors.map(error => error.messageText).join("\n")); - return; - } - target[key] = new ScriptField(script); - }; - setSpecifiedLayoutField(PivotView.scripts.arrangeInit, "arrangeInit", { collection: "Doc", docs: "Doc[]" }, undefined); - setSpecifiedLayoutField(PivotView.scripts.arrangeScript, "arrangeScript", { doc: "Doc", index: "number", collection: "Doc", state: "any", docs: "Doc[]" }, "{x: number, y: number, width?: number, height?: number}"); - } + // public static SetPivotLayout = (target: Doc) => { + // let setSpecifiedLayoutField = (originalText: string, key: string, params: Record, requiredType?: string) => { + // const script = CompileScript(originalText, { + // params, + // requiredType, + // typecheck: false + // }); + // if (!script.compiled) { + // console.log(script.errors.map(error => error.messageText).join("\n")); + // return; + // } + // target[key] = new ScriptField(script); + // }; + // setSpecifiedLayoutField(PivotView.scripts.arrangeInit, "arrangeInit", { collection: "Doc", docs: "Doc[]" }, undefined); + // setSpecifiedLayoutField(PivotView.scripts.arrangeScript, "arrangeScript", { doc: "Doc", index: "number", collection: "Doc", state: "any", docs: "Doc[]" }, "{x: number, y: number, width?: number, height?: number}"); + // } render() { const easing = () => this.props.Document.panTransformType === "Ease"; -- cgit v1.2.3-70-g09d2 From 7cf716e4288f09e835fce3613ab007f239a59e8e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 8 Aug 2019 00:14:32 -0400 Subject: updated bill's word docs --- .../source/Bill_Notes_Apple_Mac_Portable.docx | Bin 0 -> 748412 bytes .../buxton/source/Bill_Notes_Bill_Notes_CyKey.docx | Bin 1561425 -> 1675500 bytes src/scraping/buxton/source/Bill_Notes_FrogPad.docx | Bin 679241 -> 840173 bytes .../buxton/source/Bill_Notes_Gavilan_SC.docx | Bin 1531689 -> 1729610 bytes .../source/Bill_Notes_Grandjean_Stenotype.docx | Bin 1933004 -> 2094142 bytes src/scraping/buxton/source/Bill_Notes_NewO.docx | Bin 2150143 -> 2264571 bytes src/scraping/buxton/source/Bill_Notes_PARCkbd.docx | Bin 517484 -> 631959 bytes src/scraping/buxton/source/Bill_Notes_PARCtab.doc | Bin 0 -> 4046250 bytes 8 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/scraping/buxton/source/Bill_Notes_Apple_Mac_Portable.docx create mode 100644 src/scraping/buxton/source/Bill_Notes_PARCtab.doc (limited to 'src') diff --git a/src/scraping/buxton/source/Bill_Notes_Apple_Mac_Portable.docx b/src/scraping/buxton/source/Bill_Notes_Apple_Mac_Portable.docx new file mode 100644 index 000000000..df1aafe9c Binary files /dev/null and b/src/scraping/buxton/source/Bill_Notes_Apple_Mac_Portable.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx b/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx index 649d636e3..06094b4d3 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx and b/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_FrogPad.docx b/src/scraping/buxton/source/Bill_Notes_FrogPad.docx index ba80c1959..d01e1bf5c 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_FrogPad.docx and b/src/scraping/buxton/source/Bill_Notes_FrogPad.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx b/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx index 8558a4e13..b9a30c8a9 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx and b/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx b/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx index 09e17f971..0615c4953 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx and b/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_NewO.docx b/src/scraping/buxton/source/Bill_Notes_NewO.docx index 2f4a04e81..a514926d2 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_NewO.docx and b/src/scraping/buxton/source/Bill_Notes_NewO.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx b/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx index 3038de363..c0cf6ba9a 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx and b/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_PARCtab.doc b/src/scraping/buxton/source/Bill_Notes_PARCtab.doc new file mode 100644 index 000000000..3cdc2d21b Binary files /dev/null and b/src/scraping/buxton/source/Bill_Notes_PARCtab.doc differ -- cgit v1.2.3-70-g09d2