aboutsummaryrefslogtreecommitdiff
path: root/src/client/util/DictationManager.ts
diff options
context:
space:
mode:
Diffstat (limited to 'src/client/util/DictationManager.ts')
-rw-r--r--src/client/util/DictationManager.ts123
1 files changed, 90 insertions, 33 deletions
diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
index 82c63695c..bc9fe813f 100644
--- a/src/client/util/DictationManager.ts
+++ b/src/client/util/DictationManager.ts
@@ -1,18 +1,22 @@
+/* eslint-disable no-use-before-define */
import * as interpreter from 'words-to-numbers';
// @ts-ignore bcz: how are you supposed to include these definitions since dom-speech-recognition isn't a module?
import type {} from '@types/dom-speech-recognition';
+import { ClientUtils } from '../../ClientUtils';
import { Doc, Opt } from '../../fields/Doc';
+import { DocData } from '../../fields/DocSymbols';
import { List } from '../../fields/List';
import { RichTextField } from '../../fields/RichTextField';
import { listSpec } from '../../fields/Schema';
-import { Cast, CastCtor, DocCast } from '../../fields/Types';
+import { Cast, CastCtor } from '../../fields/Types';
import { AudioField, ImageField } from '../../fields/URLField';
-import { Utils } from '../../Utils';
-import { Docs } from '../documents/Documents';
+import { AudioAnnoState } from '../../server/SharedMediaTypes';
+import { Networking } from '../Network';
import { DocumentType } from '../documents/DocumentTypes';
+import { Docs } from '../documents/Documents';
import { DictationOverlay } from '../views/DictationOverlay';
-import { DocumentView, OpenWhere } from '../views/nodes/DocumentView';
-import { SelectionManager } from './SelectionManager';
+import { DocumentView } from '../views/nodes/DocumentView';
+import { OpenWhere } from '../views/nodes/OpenWhere';
import { UndoManager } from './UndoManager';
/**
@@ -61,12 +65,13 @@ export namespace DictationManager {
const intraSession = '. ';
const interSession = ' ... ';
- export let isListening = false;
+ let isListening = false;
let isManuallyStopped = false;
- let current: string | undefined = undefined;
+ let current: string | undefined;
let sessionResults: string[] = [];
+ // eslint-disable-next-line new-cap
const recognizer: Opt<SpeechRecognition> = webkitSpeechRecognition ? new webkitSpeechRecognition() : undefined;
export type InterimResultHandler = (results: string) => any;
@@ -87,7 +92,7 @@ export namespace DictationManager {
let pendingListen: Promise<string> | string | undefined;
export const listen = async (options?: Partial<ListeningOptions>) => {
- if (pendingListen instanceof Promise) return pendingListen.then(pl => innerListen(options));
+ if (pendingListen instanceof Promise) return pendingListen.then(() => innerListen(options));
return innerListen(options);
};
const innerListen = async (options?: Partial<ListeningOptions>) => {
@@ -103,7 +108,7 @@ export namespace DictationManager {
results = await (pendingListen = listenImpl(options));
pendingListen = undefined;
if (results) {
- Utils.CopyText(results);
+ ClientUtils.CopyText(results);
if (overlay) {
DictationOverlay.Instance.isListening = false;
const execute = options?.tryExecute;
@@ -150,29 +155,29 @@ export namespace DictationManager {
recognizer.start();
- return new Promise<string>((resolve, reject) => {
+ return new Promise<string>(resolve => {
recognizer.onerror = (e: any) => {
// e is SpeechRecognitionError but where is that defined?
if (!(indefinite && e.error === 'no-speech')) {
recognizer.stop();
resolve(e);
- //reject(e);
}
};
recognizer.onresult = (e: SpeechRecognitionEvent) => {
current = synthesize(e, intra);
- let matchedTerminator: string | undefined;
- if (options?.terminators && (matchedTerminator = options.terminators.find(end => (current ? current.trim().toLowerCase().endsWith(end.toLowerCase()) : false)))) {
+ const matchedTerminator = options?.terminators?.find(end => (current ? current.trim().toLowerCase().endsWith(end.toLowerCase()) : false));
+ if (options?.terminators && matchedTerminator) {
current = matchedTerminator;
recognizer.abort();
return complete();
}
!isManuallyStopped && handler?.(current);
- //isManuallyStopped && complete();
+ // isManuallyStopped && complete()
+ return undefined;
};
- recognizer.onend = (e: Event) => {
+ recognizer.onend = () => {
if (!indefinite || isManuallyStopped) {
return complete();
}
@@ -182,6 +187,7 @@ export namespace DictationManager {
current = undefined;
}
recognizer.start();
+ return undefined;
};
const complete = () => {
@@ -202,7 +208,7 @@ export namespace DictationManager {
});
};
- export const stop = (salvageSession = true) => {
+ export const stop = (/* salvageSession = true */) => {
if (!isListening || !recognizer) {
return;
}
@@ -212,7 +218,7 @@ export namespace DictationManager {
};
const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => {
- const results = e.results;
+ const { results } = e;
const transcripts: string[] = [];
for (let i = 0; i < results.length; i++) {
transcripts.push(results.item(i).item(0).transcript.trim());
@@ -231,24 +237,30 @@ export namespace DictationManager {
export type DependentEntry = { expression: RegExp; action: DependentAction; restrictTo?: DocumentType[] };
export const RegisterIndependent = (key: string, value: IndependentEntry) => Independent.set(key, value);
- export const RegisterDependent = (entry: DependentEntry) => Dependent.push(entry);
+ export const RegisterDependent = (entry: DependentEntry) => {
+ const { expression, action, restrictTo } = entry;
+ return Dependent.push({ expression, action, restrictTo: restrictTo ?? [] });
+ };
- export const execute = async (phrase: string) => {
- return UndoManager.RunInBatch(async () => {
+ export const execute = async (phrase: string) =>
+ UndoManager.RunInBatch(async () => {
console.log('PHRASE: ' + phrase);
- const targets = SelectionManager.Views;
+ const targets = DocumentView.Selected();
if (!targets || !targets.length) {
- return;
+ return undefined;
}
+ // eslint-disable-next-line no-param-reassign
phrase = phrase.toLowerCase();
const entry = Independent.get(phrase);
if (entry) {
let success = false;
- const restrictTo = entry.restrictTo;
+ const { restrictTo } = entry;
+ // eslint-disable-next-line no-restricted-syntax
for (const target of targets) {
if (!restrictTo || validate(target, restrictTo)) {
+ // eslint-disable-next-line no-await-in-loop
await entry.action(target);
success = true;
}
@@ -256,16 +268,19 @@ export namespace DictationManager {
return success;
}
- for (const entry of Dependent) {
- const regex = entry.expression;
+ // eslint-disable-next-line no-restricted-syntax
+ for (const depEntry of Dependent) {
+ const regex = depEntry.expression;
const matches = regex.exec(phrase);
regex.lastIndex = 0;
if (matches !== null) {
let success = false;
- const restrictTo = entry.restrictTo;
+ const { restrictTo } = depEntry;
+ // eslint-disable-next-line no-restricted-syntax
for (const target of targets) {
if (!restrictTo || validate(target, restrictTo)) {
- await entry.action(target, matches);
+ // eslint-disable-next-line no-await-in-loop
+ await depEntry.action(target, matches);
success = true;
}
}
@@ -275,7 +290,6 @@ export namespace DictationManager {
return false;
}, 'Execute Command');
- };
const ConstructorMap = new Map<DocumentType, CastCtor>([
[DocumentType.COL, listSpec(Doc)],
@@ -294,6 +308,7 @@ export namespace DictationManager {
};
const validate = (target: DocumentView, types: DocumentType[]) => {
+ // eslint-disable-next-line no-restricted-syntax
for (const type of types) {
if (tryCast(target, type)) {
return true;
@@ -318,7 +333,9 @@ export namespace DictationManager {
[
'clear',
{
- action: (target: DocumentView) => (Doc.GetProto(target.Document).data = new List()),
+ action: (target: DocumentView) => {
+ Doc.GetProto(target.Document).data = new List();
+ },
restrictTo: [DocumentType.COL],
},
],
@@ -328,7 +345,7 @@ export namespace DictationManager {
{
action: (target: DocumentView) => {
const newBox = Docs.Create.TextDocument('', { _width: 400, _height: 200, title: 'My Outline', _layout_autoHeight: true });
- const proto = DocCast(newBox.proto);
+ const proto = newBox[DocData];
const prompt = 'Press alt + r to start dictating here...';
const head = 3;
const anchor = head + prompt.length;
@@ -341,7 +358,7 @@ export namespace DictationManager {
],
]);
- const Dependent = new Array<DependentEntry>(
+ const Dependent = [
{
expression: /create (\w+) documents of type (image|nested collection)/g,
action: (target: DocumentView, matches: RegExpExecArray) => {
@@ -361,6 +378,7 @@ export namespace DictationManager {
case 'nested collection':
created = Docs.Create.FreeformDocument([], {});
break;
+ default:
}
created && Doc.AddDocToList(dataDoc, fieldKey, created);
}
@@ -375,7 +393,46 @@ export namespace DictationManager {
mode && (target.Document._type_collection = mode);
},
restrictTo: [DocumentType.COL],
- }
- );
+ },
+ ];
+ }
+ export function recordAudioAnnotation(dataDoc: Doc, field: string, onRecording?: (stop: () => void) => void, onEnd?: () => void) {
+ let gumStream: any;
+ let recorder: any;
+ navigator.mediaDevices.getUserMedia({ audio: true }).then(stream => {
+ let audioTextAnnos = Cast(dataDoc[field + '_audioAnnotations_text'], listSpec('string'), null);
+ if (audioTextAnnos) audioTextAnnos.push('');
+ else audioTextAnnos = dataDoc[field + '_audioAnnotations_text'] = new List<string>(['']);
+ DictationManager.Controls.listen({
+ interimHandler: value => { audioTextAnnos[audioTextAnnos.length - 1] = value; }, // prettier-ignore
+ continuous: { indefinite: false },
+ }).then(results => {
+ if (results && [DictationManager.Controls.Infringed].includes(results)) {
+ DictationManager.Controls.stop();
+ }
+ onEnd?.();
+ });
+
+ gumStream = stream;
+ recorder = new MediaRecorder(stream);
+ recorder.ondataavailable = async (e: any) => {
+ const [{ result }] = await Networking.UploadFilesToServer({ file: e.data });
+ if (!(result instanceof Error)) {
+ const audioField = new AudioField(result.accessPaths.agnostic.client);
+ const audioAnnos = Cast(dataDoc[field + '_audioAnnotations'], listSpec(AudioField), null);
+ if (audioAnnos) audioAnnos.push(audioField);
+ else dataDoc[field + '_audioAnnotations'] = new List([audioField]);
+ }
+ };
+ recorder.start();
+ const stopFunc = () => {
+ recorder.stop();
+ DictationManager.Controls.stop(/* false */);
+ dataDoc.audioAnnoState = AudioAnnoState.stopped;
+ gumStream.getAudioTracks()[0].stop();
+ };
+ if (onRecording) onRecording(stopFunc);
+ else setTimeout(stopFunc, 5000);
+ });
}
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-07 15:53:46 +0000