aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/client/cognitive_services/CognitiveServices.ts6
-rw-r--r--src/client/util/DictationManager.ts39
-rw-r--r--src/client/views/ContextMenu.tsx8
-rw-r--r--src/client/views/GlobalKeyHandler.ts29
-rw-r--r--src/client/views/nodes/DocumentView.tsx10
5 files changed, 81 insertions, 11 deletions
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index 076907f09..c118d91d3 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -1,15 +1,11 @@
import * as request from "request-promise";
import { Doc, Field, Opt } from "../../new_fields/Doc";
import { Cast } from "../../new_fields/Types";
-import { ImageField } from "../../new_fields/URLField";
-import { List } from "../../new_fields/List";
import { Docs } from "../documents/Documents";
import { RouteStore } from "../../server/RouteStore";
import { Utils } from "../../Utils";
-import { CompileScript } from "../util/Scripting";
-import { ComputedField } from "../../new_fields/ScriptField";
import { InkData } from "../../new_fields/InkField";
-import { undoBatch, UndoManager } from "../util/UndoManager";
+import { UndoManager } from "../util/UndoManager";
type APIManager<D> = { converter: BodyConverter<D>, requester: RequestExecutor, analyzer: AnalysisApplier };
type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise<string>;
diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
new file mode 100644
index 000000000..b58bdb6c7
--- /dev/null
+++ b/src/client/util/DictationManager.ts
@@ -0,0 +1,39 @@
+namespace CORE {
+ export interface IWindow extends Window {
+ webkitSpeechRecognition: any;
+ }
+}
+
+const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+
+export default class DictationManager {
+ public static Instance = new DictationManager();
+ private isListening = false;
+ private recognizer: any;
+
+ constructor() {
+ this.recognizer = new webkitSpeechRecognition();
+ this.recognizer.interimResults = false;
+ this.recognizer.continuous = true;
+ }
+
+ finish = (handler: any, data: any) => {
+ handler(data);
+ this.isListening = false;
+ this.recognizer.stop();
+ }
+
+ listen = () => {
+ if (this.isListening) {
+ return undefined;
+ }
+ this.isListening = true;
+ this.recognizer.start();
+ return new Promise<string>((resolve, reject) => {
+ this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript);
+ this.recognizer.onerror = (e: any) => this.finish(reject, e);
+ });
+
+ }
+
+} \ No newline at end of file
diff --git a/src/client/views/ContextMenu.tsx b/src/client/views/ContextMenu.tsx
index a608e448a..98025ac31 100644
--- a/src/client/views/ContextMenu.tsx
+++ b/src/client/views/ContextMenu.tsx
@@ -38,8 +38,12 @@ export class ContextMenu extends React.Component {
this._items = [];
}
- findByDescription = (target: string) => {
- return this._items.find(menuItem => menuItem.description === target);
+ findByDescription = (target: string, toLowerCase = false) => {
+ return this._items.find(menuItem => {
+ let reference = menuItem.description;
+ toLowerCase && (reference = reference.toLowerCase());
+ reference === target;
+ });
}
@action
diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts
index e31b44514..373584b4e 100644
--- a/src/client/views/GlobalKeyHandler.ts
+++ b/src/client/views/GlobalKeyHandler.ts
@@ -5,9 +5,13 @@ import { MainView } from "./MainView";
import { DragManager } from "../util/DragManager";
import { action } from "mobx";
import { Doc } from "../../new_fields/Doc";
+import { CognitiveServices } from "../cognitive_services/CognitiveServices";
+import DictationManager from "../util/DictationManager";
+import { ContextMenu } from "./ContextMenu";
+import { ContextMenuProps } from "./ContextMenuItem";
const modifiers = ["control", "meta", "shift", "alt"];
-type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo;
+type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise<KeyControlInfo>;
type KeyControlInfo = {
preventDefault: boolean,
stopPropagation: boolean
@@ -25,9 +29,10 @@ export default class KeyManager {
this.router.set(isMac ? "0001" : "0100", this.ctrl);
this.router.set(isMac ? "0100" : "0010", this.alt);
this.router.set(isMac ? "1001" : "1100", this.ctrl_shift);
+ this.router.set("1000", this.shift);
}
- public handle = (e: KeyboardEvent) => {
+ public handle = async (e: KeyboardEvent) => {
let keyname = e.key.toLowerCase();
this.handleGreedy(keyname);
@@ -43,7 +48,7 @@ export default class KeyManager {
return;
}
- let control = handleConstrained(keyname, e);
+ let control = await handleConstrained(keyname, e);
control.stopPropagation && e.stopPropagation();
control.preventDefault && e.preventDefault();
@@ -95,6 +100,24 @@ export default class KeyManager {
};
});
+ private shift = async (keyname: string) => {
+ let stopPropagation = true;
+ let preventDefault = true;
+
+ switch (keyname) {
+ case " ":
+ let transcript = await DictationManager.Instance.listen();
+ console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`);
+ let command: ContextMenuProps | undefined;
+ transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event();
+ }
+
+ return {
+ stopPropagation: stopPropagation,
+ preventDefault: preventDefault
+ };
+ }
+
private alt = action((keyname: string) => {
let stopPropagation = true;
let preventDefault = true;
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 4b5cf3a43..dc56c1c8f 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -41,6 +41,8 @@ import { ClientUtils } from '../../util/ClientUtils';
import { EditableView } from '../EditableView';
import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons';
import { DocumentDecorations } from '../DocumentDecorations';
+import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
+import DictationManager from '../../util/DictationManager';
const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
library.add(fa.faTrash);
@@ -62,7 +64,7 @@ library.add(fa.faCrosshairs);
library.add(fa.faDesktop);
library.add(fa.faUnlock);
library.add(fa.faLock);
-library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake);
+library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake, fa.faMicrophone);
// const linkSchema = createSchema({
// title: "string",
@@ -535,6 +537,11 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
this.props.Document.lockedPosition = BoolCast(this.props.Document.lockedPosition) ? undefined : true;
}
+ listen = async () => {
+ let transcript = await DictationManager.Instance.listen();
+ transcript && (Doc.GetProto(this.props.Document).transcript = transcript);
+ }
+
@action
onContextMenu = async (e: React.MouseEvent): Promise<void> => {
e.persist();
@@ -558,6 +565,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
cm.addItem({ description: BoolCast(this.props.Document.ignoreAspect, false) || !this.props.Document.nativeWidth || !this.props.Document.nativeHeight ? "Freeze" : "Unfreeze", event: this.freezeNativeDimensions, icon: "snowflake" });
cm.addItem({ description: "Pin to Presentation", event: () => PresentationView.Instance.PinDoc(this.props.Document), icon: "map-pin" });
cm.addItem({ description: BoolCast(this.props.Document.lockedPosition) ? "Unlock Position" : "Lock Position", event: this.toggleLockPosition, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" });
+ cm.addItem({ description: "Transcribe Speech", event: this.listen, icon: "microphone" });
let makes: ContextMenuProps[] = [];
makes.push({ description: "Make Background", event: this.makeBackground, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" });
makes.push({ description: this.props.Document.isButton ? "Remove Button" : "Make Button", event: this.makeBtnClicked, icon: "concierge-bell" });
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 16:55:30 +0000