From e0316c21838613df0fbf43df6a9ca5d696c52f47 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 20:19:58 -0400 Subject: more interesting speech commands and command manager pattern for DictationManager --- src/client/views/GlobalKeyHandler.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 373584b4e..1d3c77ec7 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -73,6 +73,7 @@ export default class KeyManager { } MainView.Instance.toggleColorPicker(true); SelectionManager.DeselectAll(); + DictationManager.Instance.stop(); break; case "delete": case "backspace": @@ -106,10 +107,11 @@ export default class KeyManager { switch (keyname) { case " ": - let transcript = await DictationManager.Instance.listen(); + console.log("Listening..."); + let analyzer = DictationManager.Instance; + let transcript = await analyzer.listen(); console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`); - let command: ContextMenuProps | undefined; - transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event(); + transcript && analyzer.execute(transcript); } return { -- cgit v1.2.3-70-g09d2 From 8a87f7110b56ca96b3960f6fb3917c7ed8c7a814 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 31 Jul 2019 03:21:59 -0400 Subject: overlay completed --- src/client/util/DictationManager.ts | 12 +++---- src/client/views/GlobalKeyHandler.ts | 24 +++++++++---- src/client/views/Main.scss | 68 +++++++++++++++++++++++++++--------- src/client/views/MainView.tsx | 55 +++++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 32 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 6d67f6d6d..80efe12cd 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,5 +1,5 @@ import { string } from "prop-types"; -import { observable, action } from "mobx"; +import { observable, action, autorun } from "mobx"; import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; import { UndoManager } from "./UndoManager"; @@ -8,6 +8,7 @@ import { Doc } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; +import { MainView } from "../views/MainView"; namespace CORE { export interface IWindow extends Window { @@ -22,8 +23,8 @@ export type RegexEntry = { key: RegExp, value: DependentAction }; export default class DictationManager { public static Instance = new DictationManager(); - private isListening = false; private recognizer: any; + private isListening = false; constructor() { this.recognizer = new webkitSpeechRecognition(); @@ -31,18 +32,14 @@ export default class DictationManager { this.recognizer.continuous = true; } - @observable public current = ""; - - @action finish = (handler: any, data: any) => { - this.current = data; handler(data); this.stop(); } stop = () => { - this.isListening = false; this.recognizer.stop(); + this.isListening = false; } listen = () => { @@ -55,7 +52,6 @@ export default class DictationManager { this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript); this.recognizer.onerror = (e: any) => this.finish(reject, e); }); - } private sanitize = (title: string) => { diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 1d3c77ec7..a6020bd3f 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -3,7 +3,7 @@ import { SelectionManager } from "../util/SelectionManager"; import { CollectionDockingView } from "./collections/CollectionDockingView"; import { MainView } from "./MainView"; import { DragManager } from "../util/DragManager"; -import { action } from "mobx"; +import { action, runInAction } from "mobx"; import { Doc } from "../../new_fields/Doc"; import { CognitiveServices } from "../cognitive_services/CognitiveServices"; import DictationManager from "../util/DictationManager"; @@ -74,6 +74,7 @@ export default class KeyManager { MainView.Instance.toggleColorPicker(true); SelectionManager.DeselectAll(); DictationManager.Instance.stop(); + MainView.Instance.dictationOverlayVisible = false; break; case "delete": case "backspace": @@ -107,11 +108,22 @@ export default class KeyManager { switch (keyname) { case " ": - console.log("Listening..."); - let analyzer = DictationManager.Instance; - let transcript = await analyzer.listen(); - console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`); - transcript && analyzer.execute(transcript); + let main = MainView.Instance; + main.dictationOverlayVisible = true; + main.isListening = true; + let manager = DictationManager.Instance; + let command = await manager.listen(); + main.isListening = false; + if (!command) { + break; + } + command = command.toLowerCase(); + main.dictatedPhrase = command; + main.dictationSuccess = await manager.execute(command); + setTimeout(() => { + main.dictationOverlayVisible = false; + main.dictationSuccess = undefined; + }, 3000); } return { diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss index a16123476..8e57b88c3 100644 --- a/src/client/views/Main.scss +++ b/src/client/views/Main.scss @@ -24,7 +24,7 @@ div { .jsx-parser { width: 100%; - height:100%; + height: 100%; pointer-events: none; border-radius: inherit; } @@ -119,6 +119,7 @@ button:hover { margin-bottom: 10px; } } + .toolbar-color-picker { background-color: $light-color; border-radius: 5px; @@ -128,6 +129,7 @@ button:hover { left: -3px; box-shadow: $intermediate-color 0.2vw 0.2vw 0.8vw; } + .toolbar-color-button { border-radius: 11px; width: 22px; @@ -146,7 +148,7 @@ button:hover { bottom: 22px; left: 250px; - > label { + >label { background: $dark-color; color: $light-color; display: inline-block; @@ -168,15 +170,15 @@ button:hover { transform: scale(1.15); } - > input { + >input { display: none; } - > input:not(:checked)~#add-options-content { + >input:not(:checked)~#add-options-content { display: none; } - > input:checked~label { + >input:checked~label { transform: rotate(45deg); transition: transform 0.5s; cursor: pointer; @@ -221,7 +223,7 @@ ul#add-options-list { list-style: none; padding: 5 0 0 0; - > li { + >li { display: inline-block; padding: 0; } @@ -231,7 +233,7 @@ ul#add-options-list { height: 100%; position: absolute; display: flex; - flex-direction:column; + flex-direction: column; } .mainView-libraryHandle { @@ -243,21 +245,55 @@ ul#add-options-list { position: absolute; z-index: 1; } + .svg-inline--fa { vertical-align: unset; } + .mainView-workspace { - height:200px; - position:relative; - display:flex; + height: 200px; + position: relative; + display: flex; } + .mainView-library { - height:75%; - position:relative; - display:flex; + height: 75%; + position: relative; + display: flex; } + .mainView-recentlyClosed { - height:25%; - position:relative; - display:flex; + height: 25%; + position: relative; + display: flex; +} + +.dictation-prompt { + position: absolute; + z-index: 1000; + text-align: center; + justify-content: center; + align-self: center; + align-content: center; + padding: 20px; + background: gainsboro; + border-radius: 10px; + border: 3px solid black; + box-shadow: #00000044 5px 5px 10px; + transform: translate(-50%, -50%); + top: 50%; + font-style: italic; + left: 50%; + transition: 0.5s all ease; + pointer-events: none; +} + +.dictation-prompt-overlay { + width: 100%; + height: 100%; + position: absolute; + background: darkslategray; + z-index: 999; + transition: 0.5s all ease; + pointer-events: none; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 36ac96907..d6449cffc 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,7 +39,6 @@ import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; -import DictationManager from '../util/DictationManager'; @observer export class MainView extends React.Component { @@ -48,6 +47,12 @@ export class MainView extends React.Component { @observable private _workspacesShown: boolean = false; @observable public pwidth: number = 0; @observable public pheight: number = 0; + + @observable private dictationState = "Listening..."; + @observable private dictationSuccessState: boolean | undefined = undefined; + @observable private dictationDisplayState = false; + @observable private dictationListeningState = false; + @computed private get mainContainer(): Opt { return FieldValue(Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc)); } @@ -65,6 +70,38 @@ export class MainView extends React.Component { } } + public get dictatedPhrase() { + return this.dictationState; + } + + public set dictatedPhrase(value: string) { + runInAction(() => this.dictationState = value); + } + + public get dictationSuccess() { + return this.dictationSuccessState; + } + + public set dictationSuccess(value: boolean | undefined) { + runInAction(() => this.dictationSuccessState = value); + } + + public get dictationOverlayVisible() { + return this.dictationDisplayState; + } + + public set dictationOverlayVisible(value: boolean) { + runInAction(() => this.dictationDisplayState = value); + } + + public get isListening() { + return this.dictationListeningState; + } + + public set isListening(value: boolean) { + runInAction(() => this.dictationListeningState = value); + } + componentWillMount() { var tag = document.createElement('script'); @@ -458,9 +495,23 @@ export class MainView extends React.Component { } render() { + let display = this.dictationOverlayVisible; + let success = this.dictationSuccess; + let result = this.isListening ? "Listening..." : `"${this.dictatedPhrase}"`; return (
-

{DictationManager.Instance.current}

+
{result}
+
{this.mainContent} -- cgit v1.2.3-70-g09d2 From 810b7898b279069fb8d7cc666333d02e71e23fd2 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 1 Aug 2019 02:39:33 -0400 Subject: naming tweak --- src/client/views/GlobalKeyHandler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 59d120974..99345f04e 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -114,15 +114,15 @@ export default class KeyManager { let main = MainView.Instance; main.dictationOverlayVisible = true; main.isListening = true; - let manager = DictationManager.Instance; - let command = await manager.listen(); + let dictation = DictationManager.Instance; + let command = await dictation.listen(); main.isListening = false; if (!command) { break; } command = command.toLowerCase(); main.dictatedPhrase = command; - main.dictationSuccess = await manager.execute(command); + main.dictationSuccess = await dictation.execute(command); main.overlayTimeout = setTimeout(() => { main.dictationOverlayVisible = false; main.dictationSuccess = undefined; -- cgit v1.2.3-70-g09d2 From 982961c71fdd9d5dcd02ea33a2b631076a6a1f4b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 13:31:12 -0400 Subject: refactor --- src/client/util/DictationManager.ts | 307 +++++++++++---------- src/client/views/GlobalKeyHandler.ts | 14 +- .../views/collections/CollectionBaseView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 4 +- 4 files changed, 165 insertions(+), 162 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b0866a826..b51d309a1 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -11,183 +11,190 @@ import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; -namespace CORE { - export interface IWindow extends Window { - webkitSpeechRecognition: any; - } -} - -const ConstructorMap = new Map([ - [DocumentType.COL, listSpec(Doc)], - [DocumentType.AUDIO, AudioField], - [DocumentType.IMG, ImageField], - [DocumentType.HIST, HistogramField], - [DocumentType.IMPORT, listSpec(Doc)] -]); - -const tryCast = (view: DocumentView, type: DocumentType) => { - let ctor = ConstructorMap.get(type); - if (!ctor) { - return false; - } - return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; -}; +export namespace DictationManager { -const validate = (target: DocumentView, types: DocumentType[]) => { - for (let type of types) { - if (tryCast(target, type)) { - return true; + namespace CORE { + export interface IWindow extends Window { + webkitSpeechRecognition: any; } } - return false; -}; - -const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; -export type IndependentAction = (target: DocumentView) => any | Promise; -export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; -export type RegistrationEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; -export type ActionPredicate = (target: DocumentView) => boolean; -export type RegexEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] }; - -export default class DictationManager { - public static Instance = new DictationManager(); - private recognizer: any; - private isListening = false; - - constructor() { - this.recognizer = new webkitSpeechRecognition(); - this.recognizer.interimResults = false; - this.recognizer.continuous = true; - } - finish = (handler: any, data: any) => { - handler(data); - this.stop(); - } + const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; - stop = () => { - this.recognizer.stop(); - this.isListening = false; - } + let isListening = false; + const recognizer = (() => { + let initialized = new webkitSpeechRecognition(); + initialized.interimResults = false; + initialized.continuous = true; + return initialized; + })(); - listen = () => { - if (this.isListening) { - return undefined; - } - this.isListening = true; - this.recognizer.start(); - return new Promise((resolve, reject) => { - this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript); - this.recognizer.onerror = (e: any) => this.finish(reject, e); - }); - } + export namespace Controls { + + export const listen = () => { + if (isListening) { + return undefined; + } + isListening = true; + recognizer.start(); + return new Promise((resolve, reject) => { + recognizer.onresult = (e: any) => { + resolve(e.results[0][0].transcript); + stop(); + }; + recognizer.onerror = (e: any) => { + reject(e); + stop(); + }; + }); + }; + + export const stop = () => { + recognizer.stop(); + isListening = false; + }; - public interpretNumber = (number: string) => { - let initial = parseInt(number); - if (!isNaN(initial)) { - return initial; - } - let converted = converter.wordsToNumbers(number, { fuzzy: true }); - if (converted === null) { - return NaN; - } - return typeof converted === "string" ? parseInt(converted) : converted; } - @undoBatch - public execute = async (phrase: string) => { - let targets = SelectionManager.SelectedDocuments(); - if (!targets || !targets.length) { - return; - } + export namespace Commands { - let entry = RegisteredCommands.Independent.get(phrase); - if (entry) { - let success = false; - for (let target of targets) { - if (!entry.restrictTo || validate(target, entry.restrictTo)) { - await entry.action(target); - success = true; - } + export type IndependentAction = (target: DocumentView) => any | Promise; + export type IndependentEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; + + export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise; + export type DependentEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] }; + + export const RegisterIndependent = (key: string, value: IndependentEntry) => Independent.set(key, value); + export const RegisterDependent = (entry: DependentEntry) => Dependent.push(entry); + + export const execute = async (phrase: string) => { + let targets = SelectionManager.SelectedDocuments(); + if (!targets || !targets.length) { + return; } - return success; - } - for (let entry of RegisteredCommands.Dependent) { - let regex = entry.expression; - let matches = regex.exec(phrase); - regex.lastIndex = 0; - if (matches !== null) { + let entry = Independent.get(phrase); + if (entry) { let success = false; + let restrictTo = entry.restrictTo; for (let target of targets) { - if (!entry.restrictTo || validate(target, entry.restrictTo)) { - await entry.action(target, matches); + if (!restrictTo || validate(target, restrictTo)) { + await entry.action(target); success = true; } } return success; } - } - return false; - } + for (let entry of Dependent) { + let regex = entry.expression; + let matches = regex.exec(phrase); + regex.lastIndex = 0; + if (matches !== null) { + let success = false; + let restrictTo = entry.restrictTo; + for (let target of targets) { + if (!restrictTo || validate(target, restrictTo)) { + await entry.action(target, matches); + success = true; + } + } + return success; + } + } -} + return false; + }; + + const ConstructorMap = new Map([ + [DocumentType.COL, listSpec(Doc)], + [DocumentType.AUDIO, AudioField], + [DocumentType.IMG, ImageField], + [DocumentType.HIST, HistogramField], + [DocumentType.IMPORT, listSpec(Doc)] + ]); + + const tryCast = (view: DocumentView, type: DocumentType) => { + let ctor = ConstructorMap.get(type); + if (!ctor) { + return false; + } + return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; + }; -export namespace RegisteredCommands { + const validate = (target: DocumentView, types: DocumentType[]) => { + for (let type of types) { + if (tryCast(target, type)) { + return true; + } + } + return false; + }; - export const Independent = new Map([ + const interpretNumber = (number: string) => { + let initial = parseInt(number); + if (!isNaN(initial)) { + return initial; + } + let converted = converter.wordsToNumbers(number, { fuzzy: true }); + if (converted === null) { + return NaN; + } + return typeof converted === "string" ? parseInt(converted) : converted; + }; - ["clear", { - action: (target: DocumentView) => { - Doc.GetProto(target.props.Document).data = new List(); - }, - restrictTo: [DocumentType.COL] - }], + const Independent = new Map([ - ["open fields", { - action: (target: DocumentView) => { - let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); - target.props.addDocTab(kvp, target.dataDoc, "onRight"); - } - }] - - ]); - - export const Dependent = new Array( - - { - expression: /create (\w+) documents of type (image|nested collection)/g, - action: (target: DocumentView, matches: RegExpExecArray) => { - let count = DictationManager.Instance.interpretNumber(matches[1]); - let what = matches[2]; - let dataDoc = Doc.GetProto(target.props.Document); - let fieldKey = "data"; - for (let i = 0; i < count; i++) { - let created: Doc | undefined; - switch (what) { - case "image": - created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); - break; - case "nested collection": - created = Docs.Create.FreeformDocument([], {}); - break; - } - created && Doc.AddDocToList(dataDoc, fieldKey, created); + ["clear", { + action: (target: DocumentView) => Doc.GetProto(target.props.Document).data = new List(), + restrictTo: [DocumentType.COL] + }], + + ["open fields", { + action: (target: DocumentView) => { + let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); + target.props.addDocTab(kvp, target.dataDoc, "onRight"); } + }] + + ]); + + const Dependent = new Array( + + { + expression: /create (\w+) documents of type (image|nested collection)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { + let count = interpretNumber(matches[1]); + let what = matches[2]; + let dataDoc = Doc.GetProto(target.props.Document); + let fieldKey = "data"; + for (let i = 0; i < count; i++) { + let created: Doc | undefined; + switch (what) { + case "image": + created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); + break; + case "nested collection": + created = Docs.Create.FreeformDocument([], {}); + break; + } + created && Doc.AddDocToList(dataDoc, fieldKey, created); + } + }, + restrictTo: [DocumentType.COL] }, - restrictTo: [DocumentType.COL] - }, - - { - expression: /view as (freeform|stacking|masonry|schema|tree)/g, - action: (target: DocumentView, matches: RegExpExecArray) => { - let mode = CollectionViewType.ValueOf(matches[1]); - mode && (target.props.Document.viewType = mode); - }, - restrictTo: [DocumentType.COL] - } - ); + { + expression: /view as (freeform|stacking|masonry|schema|tree)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { + let mode = CollectionViewType.valueOf(matches[1]); + mode && (target.props.Document.viewType = mode); + }, + restrictTo: [DocumentType.COL] + } + + ); + + } } \ No newline at end of file diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 99345f04e..609136bb5 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -5,10 +5,7 @@ import { MainView } from "./MainView"; import { DragManager } from "../util/DragManager"; import { action, runInAction } from "mobx"; import { Doc } from "../../new_fields/Doc"; -import { CognitiveServices } from "../cognitive_services/CognitiveServices"; -import DictationManager from "../util/DictationManager"; -import { ContextMenu } from "./ContextMenu"; -import { ContextMenuProps } from "./ContextMenuItem"; +import { DictationManager } from "../util/DictationManager"; const modifiers = ["control", "meta", "shift", "alt"]; type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise; @@ -74,7 +71,7 @@ export default class KeyManager { } main.toggleColorPicker(true); SelectionManager.DeselectAll(); - DictationManager.Instance.stop(); + DictationManager.Controls.stop(); main.dictationOverlayVisible = false; main.dictationSuccess = undefined; main.overlayTimeout && clearTimeout(main.overlayTimeout); @@ -114,19 +111,18 @@ export default class KeyManager { let main = MainView.Instance; main.dictationOverlayVisible = true; main.isListening = true; - let dictation = DictationManager.Instance; - let command = await dictation.listen(); + let command = await DictationManager.Controls.listen(); main.isListening = false; if (!command) { break; } command = command.toLowerCase(); main.dictatedPhrase = command; - main.dictationSuccess = await dictation.execute(command); + main.dictationSuccess = await DictationManager.Commands.execute(command); main.overlayTimeout = setTimeout(() => { main.dictationOverlayVisible = false; main.dictationSuccess = undefined; - }, 3000); + }, 2000); stopPropagation = true; preventDefault = true; } diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx index 3f88ed98c..24604c812 100644 --- a/src/client/views/collections/CollectionBaseView.tsx +++ b/src/client/views/collections/CollectionBaseView.tsx @@ -34,7 +34,7 @@ export namespace CollectionViewType { ["masonry", CollectionViewType.Masonry] ]); - export const ValueOf = (value: string) => { + export const valueOf = (value: string) => { return stringMapping.get(value.toLowerCase()); }; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 51661d1ae..c5d526a5a 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -41,7 +41,7 @@ import { ClientUtils } from '../../util/ClientUtils'; import { EditableView } from '../EditableView'; import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons'; import { DocumentDecorations } from '../DocumentDecorations'; -import DictationManager from '../../util/DictationManager'; +import { DictationManager } from '../../util/DictationManager'; import { CollectionViewType } from '../collections/CollectionBaseView'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? @@ -539,7 +539,7 @@ export class DocumentView extends DocComponent(Docu } listen = async () => { - let transcript = await DictationManager.Instance.listen(); + let transcript = await DictationManager.Controls.listen(); transcript && (Doc.GetProto(this.props.Document).transcript = transcript); } -- cgit v1.2.3-70-g09d2 From 30444bd06dbd00c948749fa739e7a42cef87ec85 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 14:05:33 -0400 Subject: added final or interim listning --- src/client/util/DictationManager.ts | 35 +++++++++++++++++++++++++++-------- src/client/views/GlobalKeyHandler.ts | 2 +- 2 files changed, 28 insertions(+), 9 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b51d309a1..2ed74a729 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -2,7 +2,7 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; import { undoBatch } from "./UndoManager"; import * as converter from "words-to-numbers"; -import { Doc } from "../../new_fields/Doc"; +import { Doc, Opt } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs, DocumentType } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; @@ -18,40 +18,59 @@ export namespace DictationManager { webkitSpeechRecognition: any; } } - const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; let isListening = false; const recognizer = (() => { let initialized = new webkitSpeechRecognition(); - initialized.interimResults = false; initialized.continuous = true; return initialized; })(); export namespace Controls { - export const listen = () => { + export type InterimResultHandler = (results: any) => any; + + export const listen = (handler: Opt = undefined) => { if (isListening) { return undefined; } isListening = true; + + recognizer.interimResults = handler !== undefined; recognizer.start(); + return new Promise((resolve, reject) => { - recognizer.onresult = (e: any) => { - resolve(e.results[0][0].transcript); - stop(); - }; recognizer.onerror = (e: any) => { reject(e); stop(); }; + if (handler) { + let newestResult: string; + recognizer.onresult = (e: any) => { + newestResult = e.results[0][0].transcript; + handler(newestResult); + }; + recognizer.onend = (e: any) => { + resolve(newestResult); + stop(); + }; + } else { + recognizer.onresult = (e: any) => { + let finalResult = e.results[0][0].transcript; + resolve(finalResult); + stop(); + }; + } }); }; export const stop = () => { recognizer.stop(); isListening = false; + recognizer.onresult = undefined; + recognizer.onend = undefined; + recognizer.onerror = undefined; }; } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 609136bb5..b1ea92bb8 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -111,7 +111,7 @@ export default class KeyManager { let main = MainView.Instance; main.dictationOverlayVisible = true; main.isListening = true; - let command = await DictationManager.Controls.listen(); + let command = await DictationManager.Controls.listen((results: any) => console.log(results)); main.isListening = false; if (!command) { break; -- cgit v1.2.3-70-g09d2 From 813476d753f0286d546a0b1c9f1b1774481604e0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 17:17:43 -0400 Subject: final refactor and commenting --- src/client/util/DictationManager.ts | 29 +++++++++++++++++++++++++++-- src/client/views/GlobalKeyHandler.ts | 14 +++++++------- src/client/views/MainView.tsx | 16 ++++++++++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index 2ed74a729..b799238ba 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,6 +1,6 @@ import { SelectionManager } from "./SelectionManager"; import { DocumentView } from "../views/nodes/DocumentView"; -import { undoBatch } from "./UndoManager"; +import { UndoManager } from "./UndoManager"; import * as converter from "words-to-numbers"; import { Doc, Opt } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; @@ -11,8 +11,26 @@ import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; +/** + * This namespace provides a singleton instance of a manager that + * handles the listening and text-conversion of user speech. + * + * The basic manager functionality can be attained by the DictationManager.Controls namespace, which provide + * a simple recording operation that returns the interpreted text as a string. + * + * Additionally, however, the DictationManager also exposes the ability to execute voice commands within Dash. + * It stores a default library of registered commands that can be triggered by listen()'ing for a phrase and then + * passing the results into the execute() function. + * + * In addition to compile-time default commands, you can invoke DictationManager.Commands.Register(Independent|Dependent) + * to add new commands as classes or components are constructed. + */ export namespace DictationManager { + /** + * Some type maneuvering to access Webkit's built-in + * speech recognizer. + */ namespace CORE { export interface IWindow extends Window { webkitSpeechRecognition: any; @@ -24,6 +42,7 @@ export namespace DictationManager { const recognizer = (() => { let initialized = new webkitSpeechRecognition(); initialized.continuous = true; + initialized.language = "en-US"; return initialized; })(); @@ -77,6 +96,8 @@ export namespace DictationManager { export namespace Commands { + export const dictationFadeDuration = 2000; + export type IndependentAction = (target: DocumentView) => any | Promise; export type IndependentEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; @@ -91,14 +112,16 @@ export namespace DictationManager { if (!targets || !targets.length) { return; } - + phrase = phrase.toLowerCase(); let entry = Independent.get(phrase); if (entry) { let success = false; let restrictTo = entry.restrictTo; for (let target of targets) { if (!restrictTo || validate(target, restrictTo)) { + let batch = UndoManager.StartBatch("Independent Command"); await entry.action(target); + batch.end(); success = true; } } @@ -114,7 +137,9 @@ export namespace DictationManager { let restrictTo = entry.restrictTo; for (let target of targets) { if (!restrictTo || validate(target, restrictTo)) { + let batch = UndoManager.StartBatch("Dependent Command"); await entry.action(target, matches); + batch.end(); success = true; } } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index b1ea92bb8..82289c249 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -74,7 +74,7 @@ export default class KeyManager { DictationManager.Controls.stop(); main.dictationOverlayVisible = false; main.dictationSuccess = undefined; - main.overlayTimeout && clearTimeout(main.overlayTimeout); + main.cancelDictationFade(); break; case "delete": case "backspace": @@ -110,19 +110,19 @@ export default class KeyManager { case " ": let main = MainView.Instance; main.dictationOverlayVisible = true; + main.isListening = true; let command = await DictationManager.Controls.listen((results: any) => console.log(results)); main.isListening = false; + if (!command) { break; } - command = command.toLowerCase(); - main.dictatedPhrase = command; + + main.dictatedPhrase = command = command.toLowerCase(); main.dictationSuccess = await DictationManager.Commands.execute(command); - main.overlayTimeout = setTimeout(() => { - main.dictationOverlayVisible = false; - main.dictationSuccess = undefined; - }, 2000); + main.initiateDictationFade(); + stopPropagation = true; preventDefault = true; } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 4a5e4a3d1..748e1e634 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,6 +39,7 @@ import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; +import { DictationManager } from '../util/DictationManager'; @observer export class MainView extends React.Component { @@ -55,6 +56,21 @@ export class MainView extends React.Component { public overlayTimeout: NodeJS.Timeout | undefined; + public initiateDictationFade = () => { + let duration = DictationManager.Commands.dictationFadeDuration; + this.overlayTimeout = setTimeout(() => { + this.dictationOverlayVisible = false; + this.dictationSuccess = undefined; + }, duration); + } + + public cancelDictationFade = () => { + if (this.overlayTimeout) { + clearTimeout(this.overlayTimeout); + this.overlayTimeout = undefined; + } + } + @computed private get mainContainer(): Opt { return FieldValue(Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc)); } -- cgit v1.2.3-70-g09d2 From d6fda11588f1a117e8acc30ea5600d34ff22e01b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 3 Aug 2019 21:29:56 -0400 Subject: now differentiate between continuous, indefinite and, separately, interim vs final only dictation results --- src/client/util/DictationManager.ts | 81 +++++++++++++++++++++------------ src/client/views/GlobalKeyHandler.ts | 3 +- src/client/views/MainView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 3 +- src/new_fields/Doc.ts | 2 +- 5 files changed, 59 insertions(+), 32 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b02a5ecbe..89797f101 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -39,57 +39,82 @@ export namespace DictationManager { const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; let isListening = false; - const recognizer = (() => { - let initialized = new webkitSpeechRecognition(); - initialized.continuous = true; - initialized.language = "en-US"; - return initialized; - })(); + let isManuallyStopped = false; + const recognizer: SpeechRecognition = new webkitSpeechRecognition(); export namespace Controls { + let newestResult: string; export type InterimResultHandler = (results: any) => any; + export type ContinuityArgs = { indefinite: boolean } | false; + export interface ListeningOptions { + language: string; + continuous: ContinuityArgs; + interimHandler: InterimResultHandler; + delimiter: string; + } - export const listen = (handler: Opt = undefined) => { + export const listen = (options?: Partial) => { if (isListening) { return undefined; } isListening = true; + let handler = options ? options.interimHandler : undefined; + let continuous = options ? options.continuous : undefined; + let language = options ? options.language : undefined; + let delimiter = options ? options.delimiter : undefined; + recognizer.interimResults = handler !== undefined; + recognizer.continuous = continuous === undefined ? false : continuous !== false; + recognizer.lang = language === undefined ? "en-US" : language; + recognizer.start(); return new Promise((resolve, reject) => { + recognizer.onerror = (e: any) => { reject(e); stop(); }; - if (handler) { - let newestResult: string; - recognizer.onresult = (e: any) => { - newestResult = e.results[0][0].transcript; - handler(newestResult); - }; - recognizer.onend = (e: any) => { + + recognizer.onresult = (e: SpeechRecognitionEvent) => { + newestResult = synthesize(e, delimiter); + handler && handler(newestResult); + }; + + recognizer.onend = (e: Event) => { + if (continuous && continuous.indefinite && !isManuallyStopped) { + recognizer.start(); + } else { resolve(newestResult); - stop(); - }; - } else { - recognizer.onresult = (e: any) => { - let finalResult = e.results[0][0].transcript; - resolve(finalResult); - stop(); - }; - } + reset(); + } + }; + }); }; - export const stop = () => { - recognizer.stop(); + export const stop = (saveCumulative = true) => { + saveCumulative ? recognizer.stop() : recognizer.abort(); + reset(); + }; + + const reset = () => { isListening = false; - recognizer.onresult = undefined; - recognizer.onend = undefined; - recognizer.onerror = undefined; + isManuallyStopped = false; + recognizer.onresult = null; + recognizer.onend = null; + recognizer.onerror = null; + }; + + const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { + let results = e.results; + let transcripts: string[] = []; + for (let i = 0; i < results.length; i++) { + transcripts.push(results.item(i).item(0).transcript.trim()); + } + return transcripts.join(delimiter || "..."); }; } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 82289c249..98df43a1e 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -112,7 +112,8 @@ export default class KeyManager { main.dictationOverlayVisible = true; main.isListening = true; - let command = await DictationManager.Controls.listen((results: any) => console.log(results)); + // let printer = (results: any) => console.log(results); + let command = await DictationManager.Controls.listen(); main.isListening = false; if (!command) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 748e1e634..5cec34293 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -49,7 +49,7 @@ export class MainView extends React.Component { @observable public pwidth: number = 0; @observable public pheight: number = 0; - @observable private dictationState = "Listening..."; + @observable private dictationState = ""; @observable private dictationSuccessState: boolean | undefined = undefined; @observable private dictationDisplayState = false; @observable private dictationListeningState = false; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index c5d526a5a..bd87bf21b 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -539,7 +539,8 @@ export class DocumentView extends DocComponent(Docu } listen = async () => { - let transcript = await DictationManager.Controls.listen(); + let options = { continuous: { indefinite: true }, delimiter: " " }; + let transcript = await DictationManager.Controls.listen(options); transcript && (Doc.GetProto(this.props.Document).transcript = transcript); } diff --git a/src/new_fields/Doc.ts b/src/new_fields/Doc.ts index 84b8589dd..979574f16 100644 --- a/src/new_fields/Doc.ts +++ b/src/new_fields/Doc.ts @@ -531,7 +531,7 @@ export namespace Doc { d.layout = detailLayout; d.nativeWidth = d.nativeHeight = undefined; if (detailLayout instanceof Doc) { - let delegDetailLayout = Doc.MakeDelegate(detailLayout) as Doc; + let delegDetailLayout = Doc.MakeDelegate(detailLayout); d.layout = delegDetailLayout; let subDetailLayout1 = await PromiseValue(delegDetailLayout.detailedLayout); let subDetailLayout = await PromiseValue(delegDetailLayout.detailedLayout); -- cgit v1.2.3-70-g09d2 From 62e541e21b5675039283a515d9b1bba02b62e432 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 03:18:51 -0400 Subject: handled all edge cases for continuous recording --- src/client/util/DictationManager.ts | 70 +++++++++++++++++++++++------------- src/client/views/ContextMenuItem.tsx | 2 +- src/client/views/GlobalKeyHandler.ts | 8 +++-- 3 files changed, 52 insertions(+), 28 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index fa34ca9ad..5f443f99e 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -38,14 +38,17 @@ export namespace DictationManager { } const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; - let isListening = false; - let isManuallyStopped = false; - const results: string[] = []; - const recognizer: SpeechRecognition = new webkitSpeechRecognition(); - export namespace Controls { - let newestResult: string; + const defaultDelimiter = "..."; + let isListening = false; + let isManuallyStopped = false; + let sessionResults: string[] = []; + + const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition(); + recognizer.onstart = () => console.log("initiating speech recognition session..."); + + let current: string | undefined = undefined; export type InterimResultHandler = (results: any) => any; export type ContinuityArgs = { indefinite: boolean } | false; export interface ListeningOptions { @@ -60,8 +63,7 @@ export namespace DictationManager { try { results = await listenImpl(options); } catch (e) { - results = "Dictation Error: "; - results += "error" in e ? e.error : "unknown error"; + results = `Dictation Error: ${"error" in e ? e.error : "unknown error"}`; } return results; }; @@ -74,6 +76,7 @@ export namespace DictationManager { let handler = options ? options.interimHandler : undefined; let continuous = options ? options.continuous : undefined; + let indefinite = continuous && continuous.indefinite; let language = options ? options.language : undefined; let delimiter = options ? options.delimiter : undefined; @@ -85,41 +88,56 @@ export namespace DictationManager { return new Promise((resolve, reject) => { - recognizer.onerror = (e: any) => { - reject(e); - stop(); + recognizer.onerror = (e: SpeechRecognitionError) => { + if (!(indefinite && e.error === "no-speech")) { + stop(true); + reject(e); + } }; recognizer.onresult = (e: SpeechRecognitionEvent) => { - newestResult = synthesize(e, delimiter); - continuous && continuous.indefinite && results.push(newestResult); - handler && handler(newestResult); + current = synthesize(e, delimiter); + handler && handler(current); + isManuallyStopped && complete(); }; recognizer.onend = (e: Event) => { - let indefinite = continuous && continuous.indefinite; - if (indefinite && !isManuallyStopped) { - recognizer.start(); + if (!indefinite || isManuallyStopped) { + return complete(); + } + + if (current) { + sessionResults.push(current); + current = undefined; + } + recognizer.start(); + }; + + let complete = () => { + if (indefinite) { + current && sessionResults.push(current); + resolve(connect(sessionResults, delimiter)); } else { - resolve(indefinite ? newestResult : results.join(delimiter)); - reset(); + resolve(current); } + reset(); }; }); }; - export const stop = (saveCumulative = true) => { - saveCumulative ? recognizer.stop() : recognizer.abort(); - reset(); + export const stop = (errorTriggered = false) => { + !errorTriggered && (isManuallyStopped = true); + recognizer.stop(); }; const reset = () => { isListening = false; isManuallyStopped = false; - recognizer.onerror = null; recognizer.onresult = null; + recognizer.onerror = null; recognizer.onend = null; + sessionResults = []; }; const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { @@ -128,7 +146,11 @@ export namespace DictationManager { for (let i = 0; i < results.length; i++) { transcripts.push(results.item(i).item(0).transcript.trim()); } - return transcripts.join(delimiter || "..."); + return transcripts.join(delimiter || defaultDelimiter); + }; + + const connect = (sessions: string[], delimiter?: string) => { + return sessions.map(text => `(${text})`).join(delimiter || defaultDelimiter); }; } diff --git a/src/client/views/ContextMenuItem.tsx b/src/client/views/ContextMenuItem.tsx index a1787e78f..90f7be33f 100644 --- a/src/client/views/ContextMenuItem.tsx +++ b/src/client/views/ContextMenuItem.tsx @@ -39,13 +39,13 @@ export class ContextMenuItem extends React.Component) => { if ("event" in this.props) { + this.props.closeMenu && this.props.closeMenu(); let batch: UndoManager.Batch | undefined; if (this.props.undoable !== false) { batch = UndoManager.StartBatch(`Context menu event: ${this.props.description}`); } await this.props.event(); batch && batch.end(); - this.props.closeMenu && this.props.closeMenu(); } } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 98df43a1e..c3e6ae6c8 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -72,9 +72,11 @@ export default class KeyManager { main.toggleColorPicker(true); SelectionManager.DeselectAll(); DictationManager.Controls.stop(); - main.dictationOverlayVisible = false; - main.dictationSuccess = undefined; - main.cancelDictationFade(); + if (main.dictationOverlayVisible) { + main.dictationOverlayVisible = false; + main.dictationSuccess = undefined; + main.cancelDictationFade(); + } break; case "delete": case "backspace": -- cgit v1.2.3-70-g09d2 From 6d718c8a243e68d23199d35592bfded285385c91 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 05:09:56 -0400 Subject: now all listen() calls display recording UI --- src/client/util/DictationManager.ts | 24 ++++++++++++++++++++---- src/client/views/GlobalKeyHandler.ts | 21 ++++----------------- src/client/views/Main.scss | 1 - src/client/views/MainView.tsx | 11 ++++++++--- src/client/views/nodes/DocumentView.tsx | 16 +++++++++++++--- 5 files changed, 45 insertions(+), 28 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b6f871713..a882994c1 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -10,6 +10,7 @@ import { Cast, CastCtor } from "../../new_fields/Types"; import { listSpec } from "../../new_fields/Schema"; import { AudioField, ImageField } from "../../new_fields/URLField"; import { HistogramField } from "../northstar/dash-fields/HistogramField"; +import { MainView } from "../views/MainView"; /** * This namespace provides a singleton instance of a manager that @@ -37,9 +38,11 @@ export namespace DictationManager { } } const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; + export const placeholder = "Listening..."; export namespace Controls { + const infringe = "unable to process: dictation manager still involved in previous session"; const intraSession = ". "; const interSession = " ... "; @@ -52,7 +55,7 @@ export namespace DictationManager { const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition(); recognizer.onstart = () => console.log("initiating speech recognition session..."); - export type InterimResultHandler = (results: any) => any; + export type InterimResultHandler = (results: string) => any; export type ContinuityArgs = { indefinite: boolean } | false; export type DelimiterArgs = { inter: string, intra: string }; @@ -61,21 +64,33 @@ export namespace DictationManager { continuous: ContinuityArgs; delimiters: DelimiterArgs; interimHandler: InterimResultHandler; + tryExecute: boolean; } export const listen = async (options?: Partial) => { - let results: any; + let results: string | undefined; + MainView.Instance.dictationOverlayVisible = true; + MainView.Instance.isListening = true; try { results = await listenImpl(options); + if (results) { + MainView.Instance.isListening = false; + MainView.Instance.dictatedPhrase = results = results.toLowerCase(); + MainView.Instance.dictationSuccess = options && options.tryExecute ? await DictationManager.Commands.execute(results) : true; + } } catch (e) { - results = `Dictation Error: ${"error" in e ? e.error : "unknown error"}`; + MainView.Instance.isListening = false; + MainView.Instance.dictatedPhrase = results = `dictation error: ${"error" in e ? e.error : "unknown error"}`; + MainView.Instance.dictationSuccess = false; + } finally { + MainView.Instance.initiateDictationFade(); } return results; }; const listenImpl = (options?: Partial) => { if (isListening) { - return undefined; + return infringe; } isListening = true; @@ -126,6 +141,7 @@ export namespace DictationManager { } else { resolve(current); } + current = undefined; reset(); }; diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index c3e6ae6c8..0989e8db1 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -73,9 +73,11 @@ export default class KeyManager { SelectionManager.DeselectAll(); DictationManager.Controls.stop(); if (main.dictationOverlayVisible) { + main.cancelDictationFade(); main.dictationOverlayVisible = false; + main.isListening = true; + main.dictatedPhrase = ""; main.dictationSuccess = undefined; - main.cancelDictationFade(); } break; case "delete": @@ -110,22 +112,7 @@ export default class KeyManager { switch (keyname) { case " ": - let main = MainView.Instance; - main.dictationOverlayVisible = true; - - main.isListening = true; - // let printer = (results: any) => console.log(results); - let command = await DictationManager.Controls.listen(); - main.isListening = false; - - if (!command) { - break; - } - - main.dictatedPhrase = command = command.toLowerCase(); - main.dictationSuccess = await DictationManager.Commands.execute(command); - main.initiateDictationFade(); - + DictationManager.Controls.listen({ tryExecute: true }); stopPropagation = true; preventDefault = true; } diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss index 8e57b88c3..f76abaff3 100644 --- a/src/client/views/Main.scss +++ b/src/client/views/Main.scss @@ -292,7 +292,6 @@ ul#add-options-list { width: 100%; height: 100%; position: absolute; - background: darkslategray; z-index: 999; transition: 0.5s all ease; pointer-events: none; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 4443eea6d..383efa1e3 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,7 +1,7 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; import { faArrowDown, faCloudUploadAlt, faArrowUp, faClone, faCheck, faPlay, faPause, faCaretUp, faLongArrowAltRight, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faPortrait, faMusic, faObjectGroup, faPenNib, faRedoAlt, faTable, faThumbtack, faTree, faUndoAlt, faCat, faBolt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, configure, observable, runInAction, reaction, trace } from 'mobx'; +import { action, computed, configure, observable, runInAction, reaction, trace, autorun } from 'mobx'; import { observer } from 'mobx-react'; import "normalize.css"; import * as React from 'react'; @@ -167,6 +167,8 @@ export class MainView extends React.Component { } } + autorun(() => console.log(`this.isListening = ${this.isListening}`)); + library.add(faFont); library.add(faExclamation); library.add(faPortrait); @@ -523,7 +525,7 @@ export class MainView extends React.Component { render() { let display = this.dictationOverlayVisible; let success = this.dictationSuccess; - let result = this.isListening ? "Listening..." : `"${this.dictatedPhrase}"`; + let result = this.isListening ? DictationManager.placeholder : `"${this.dictatedPhrase}"`; return (
{result}
{this.mainContent} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 1d9cb3c80..a415aefda 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -38,6 +38,7 @@ import "./DocumentView.scss"; import { FormattedTextBox } from './FormattedTextBox'; import React = require("react"); import { DictationManager } from '../../util/DictationManager'; +import { MainView } from '../MainView'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -536,9 +537,18 @@ export class DocumentView extends DocComponent(Docu } listen = async () => { - let options = { continuous: { indefinite: true }, delimiter: " " }; - let transcript = await DictationManager.Controls.listen(options); - transcript && (Doc.GetProto(this.props.Document).transcript = transcript); + let dataDoc = Doc.GetProto(this.props.Document); + let options = { + continuous: { indefinite: true }, + delimiter: " ", + interimHandler: (results: string) => { + MainView.Instance.isListening = false; + MainView.Instance.dictationSuccess = true; + MainView.Instance.dictatedPhrase = results; + } + }; + let final = await DictationManager.Controls.listen(options); + final && (dataDoc.transcript = final); } @action -- cgit v1.2.3-70-g09d2 From 95ff1649631457449bdc580a5a47937718c00e3b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 4 Aug 2019 05:19:14 -0400 Subject: factored out stop UI code --- src/client/util/DictationManager.ts | 10 ++++++++++ src/client/views/GlobalKeyHandler.ts | 7 ------- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'src/client/views/GlobalKeyHandler.ts') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index a882994c1..2af7c53cb 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -149,8 +149,18 @@ export namespace DictationManager { }; export const stop = (salvageSession = true) => { + if (!isListening) { + return; + } isManuallyStopped = true; salvageSession ? recognizer.stop() : recognizer.abort(); + if (MainView.Instance.dictationOverlayVisible) { + MainView.Instance.cancelDictationFade(); + MainView.Instance.dictationOverlayVisible = false; + MainView.Instance.isListening = true; + MainView.Instance.dictatedPhrase = ""; + MainView.Instance.dictationSuccess = undefined; + } }; const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 0989e8db1..e773014e3 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -72,13 +72,6 @@ export default class KeyManager { main.toggleColorPicker(true); SelectionManager.DeselectAll(); DictationManager.Controls.stop(); - if (main.dictationOverlayVisible) { - main.cancelDictationFade(); - main.dictationOverlayVisible = false; - main.isListening = true; - main.dictatedPhrase = ""; - main.dictationSuccess = undefined; - } break; case "delete": case "backspace": -- cgit v1.2.3-70-g09d2