aboutsummaryrefslogtreecommitdiff
path: root/src/client/util
diff options
context:
space:
mode:
Diffstat (limited to 'src/client/util')
-rw-r--r--src/client/util/DictationManager.ts366
-rw-r--r--src/client/util/DocumentManager.ts22
-rw-r--r--src/client/util/DragManager.ts15
-rw-r--r--src/client/util/LinkManager.ts14
-rw-r--r--src/client/util/SelectionManager.ts1
-rw-r--r--src/client/util/SerializationHelper.ts4
-rw-r--r--src/client/util/type_decls.d1
7 files changed, 378 insertions, 45 deletions
diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
index b58bdb6c7..9c61fe125 100644
--- a/src/client/util/DictationManager.ts
+++ b/src/client/util/DictationManager.ts
@@ -1,38 +1,348 @@
-namespace CORE {
- export interface IWindow extends Window {
- webkitSpeechRecognition: any;
+import { SelectionManager } from "./SelectionManager";
+import { DocumentView } from "../views/nodes/DocumentView";
+import { UndoManager } from "./UndoManager";
+import * as interpreter from "words-to-numbers";
+import { Doc } from "../../new_fields/Doc";
+import { List } from "../../new_fields/List";
+import { Docs, DocumentType } from "../documents/Documents";
+import { CollectionViewType } from "../views/collections/CollectionBaseView";
+import { Cast, CastCtor } from "../../new_fields/Types";
+import { listSpec } from "../../new_fields/Schema";
+import { AudioField, ImageField } from "../../new_fields/URLField";
+import { HistogramField } from "../northstar/dash-fields/HistogramField";
+import { MainView } from "../views/MainView";
+import { Utils } from "../../Utils";
+
+/**
+ * This namespace provides a singleton instance of a manager that
+ * handles the listening and text-conversion of user speech.
+ *
+ * The basic manager functionality can be attained by the DictationManager.Controls namespace, which provide
+ * a simple recording operation that returns the interpreted text as a string.
+ *
+ * Additionally, however, the DictationManager also exposes the ability to execute voice commands within Dash.
+ * It stores a default library of registered commands that can be triggered by listen()'ing for a phrase and then
+ * passing the results into the execute() function.
+ *
+ * In addition to compile-time default commands, you can invoke DictationManager.Commands.Register(Independent|Dependent)
+ * to add new commands as classes or components are constructed.
+ */
+export namespace DictationManager {
+
+ /**
+ * Some type maneuvering to access Webkit's built-in
+ * speech recognizer.
+ */
+ namespace CORE {
+ export interface IWindow extends Window {
+ webkitSpeechRecognition: any;
+ }
}
-}
+ const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+ export const placeholder = "Listening...";
-const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+ export namespace Controls {
-export default class DictationManager {
- public static Instance = new DictationManager();
- private isListening = false;
- private recognizer: any;
+ const infringe = "unable to process: dictation manager still involved in previous session";
+ const intraSession = ". ";
+ const interSession = " ... ";
- constructor() {
- this.recognizer = new webkitSpeechRecognition();
- this.recognizer.interimResults = false;
- this.recognizer.continuous = true;
- }
+ let isListening = false;
+ let isManuallyStopped = false;
- finish = (handler: any, data: any) => {
- handler(data);
- this.isListening = false;
- this.recognizer.stop();
- }
+ let current: string | undefined = undefined;
+ let sessionResults: string[] = [];
+
+ const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition();
+ recognizer.onstart = () => console.log("initiating speech recognition session...");
+
+ export type InterimResultHandler = (results: string) => any;
+ export type ContinuityArgs = { indefinite: boolean } | false;
+ export type DelimiterArgs = { inter: string, intra: string };
+ export type ListeningUIStatus = { interim: boolean } | false;
- listen = () => {
- if (this.isListening) {
- return undefined;
+ export interface ListeningOptions {
+ language: string;
+ continuous: ContinuityArgs;
+ delimiters: DelimiterArgs;
+ interimHandler: InterimResultHandler;
+ tryExecute: boolean;
}
- this.isListening = true;
- this.recognizer.start();
- return new Promise<string>((resolve, reject) => {
- this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript);
- this.recognizer.onerror = (e: any) => this.finish(reject, e);
- });
+
+ export const listen = async (options?: Partial<ListeningOptions>) => {
+ let results: string | undefined;
+ let main = MainView.Instance;
+
+ main.dictationOverlayVisible = true;
+ main.isListening = { interim: false };
+
+ try {
+ results = await listenImpl(options);
+ if (results) {
+ Utils.CopyText(results);
+ main.isListening = false;
+ let execute = options && options.tryExecute;
+ main.dictatedPhrase = execute ? results.toLowerCase() : results;
+ main.dictationSuccess = execute ? await DictationManager.Commands.execute(results) : true;
+ }
+ } catch (e) {
+ main.isListening = false;
+ main.dictatedPhrase = results = `dictation error: ${"error" in e ? e.error : "unknown error"}`;
+ main.dictationSuccess = false;
+ } finally {
+ main.initiateDictationFade();
+ }
+
+ return results;
+ };
+
+ const listenImpl = (options?: Partial<ListeningOptions>) => {
+ if (isListening) {
+ return infringe;
+ }
+ isListening = true;
+
+ let handler = options ? options.interimHandler : undefined;
+ let continuous = options ? options.continuous : undefined;
+ let indefinite = continuous && continuous.indefinite;
+ let language = options ? options.language : undefined;
+ let intra = options && options.delimiters ? options.delimiters.intra : undefined;
+ let inter = options && options.delimiters ? options.delimiters.inter : undefined;
+
+ recognizer.interimResults = handler !== undefined;
+ recognizer.continuous = continuous === undefined ? false : continuous !== false;
+ recognizer.lang = language === undefined ? "en-US" : language;
+
+ recognizer.start();
+
+ return new Promise<string>((resolve, reject) => {
+
+ recognizer.onerror = (e: SpeechRecognitionError) => {
+ if (!(indefinite && e.error === "no-speech")) {
+ recognizer.stop();
+ reject(e);
+ }
+ };
+
+ recognizer.onresult = (e: SpeechRecognitionEvent) => {
+ current = synthesize(e, intra);
+ handler && handler(current);
+ isManuallyStopped && complete();
+ };
+
+ recognizer.onend = (e: Event) => {
+ if (!indefinite || isManuallyStopped) {
+ return complete();
+ }
+
+ if (current) {
+ sessionResults.push(current);
+ current = undefined;
+ }
+ recognizer.start();
+ };
+
+ let complete = () => {
+ if (indefinite) {
+ current && sessionResults.push(current);
+ sessionResults.length && resolve(sessionResults.join(inter || interSession));
+ } else {
+ resolve(current);
+ }
+ reset();
+ };
+
+ });
+ };
+
+ export const stop = (salvageSession = true) => {
+ if (!isListening) {
+ return;
+ }
+ isManuallyStopped = true;
+ salvageSession ? recognizer.stop() : recognizer.abort();
+ let main = MainView.Instance;
+ if (main.dictationOverlayVisible) {
+ main.cancelDictationFade();
+ main.dictationOverlayVisible = false;
+ main.dictationSuccess = undefined;
+ setTimeout(() => main.dictatedPhrase = placeholder, 500);
+ }
+ };
+
+ const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => {
+ let results = e.results;
+ let transcripts: string[] = [];
+ for (let i = 0; i < results.length; i++) {
+ transcripts.push(results.item(i).item(0).transcript.trim());
+ }
+ return transcripts.join(delimiter || intraSession);
+ };
+
+ const reset = () => {
+ current = undefined;
+ sessionResults = [];
+ isListening = false;
+ isManuallyStopped = false;
+ recognizer.onresult = null;
+ recognizer.onerror = null;
+ recognizer.onend = null;
+ };
+
+ }
+
+ export namespace Commands {
+
+ export const dictationFadeDuration = 2000;
+
+ export type IndependentAction = (target: DocumentView) => any | Promise<any>;
+ export type IndependentEntry = { action: IndependentAction, restrictTo?: DocumentType[] };
+
+ export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise<any>;
+ export type DependentEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] };
+
+ export const RegisterIndependent = (key: string, value: IndependentEntry) => Independent.set(key, value);
+ export const RegisterDependent = (entry: DependentEntry) => Dependent.push(entry);
+
+ export const execute = async (phrase: string) => {
+ return UndoManager.RunInBatch(async () => {
+ let targets = SelectionManager.SelectedDocuments();
+ if (!targets || !targets.length) {
+ return;
+ }
+
+ phrase = phrase.toLowerCase();
+ let entry = Independent.get(phrase);
+
+ if (entry) {
+ let success = false;
+ let restrictTo = entry.restrictTo;
+ for (let target of targets) {
+ if (!restrictTo || validate(target, restrictTo)) {
+ await entry.action(target);
+ success = true;
+ }
+ }
+ return success;
+ }
+
+ for (let entry of Dependent) {
+ let regex = entry.expression;
+ let matches = regex.exec(phrase);
+ regex.lastIndex = 0;
+ if (matches !== null) {
+ let success = false;
+ let restrictTo = entry.restrictTo;
+ for (let target of targets) {
+ if (!restrictTo || validate(target, restrictTo)) {
+ await entry.action(target, matches);
+ success = true;
+ }
+ }
+ return success;
+ }
+ }
+
+ return false;
+ }, "Execute Command");
+ };
+
+ const ConstructorMap = new Map<DocumentType, CastCtor>([
+ [DocumentType.COL, listSpec(Doc)],
+ [DocumentType.AUDIO, AudioField],
+ [DocumentType.IMG, ImageField],
+ [DocumentType.HIST, HistogramField],
+ [DocumentType.IMPORT, listSpec(Doc)],
+ [DocumentType.TEXT, "string"]
+ ]);
+
+ const tryCast = (view: DocumentView, type: DocumentType) => {
+ let ctor = ConstructorMap.get(type);
+ if (!ctor) {
+ return false;
+ }
+ return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined;
+ };
+
+ const validate = (target: DocumentView, types: DocumentType[]) => {
+ for (let type of types) {
+ if (tryCast(target, type)) {
+ return true;
+ }
+ }
+ return false;
+ };
+
+ const interpretNumber = (number: string) => {
+ let initial = parseInt(number);
+ if (!isNaN(initial)) {
+ return initial;
+ }
+ let converted = interpreter.wordsToNumbers(number, { fuzzy: true });
+ if (converted === null) {
+ return NaN;
+ }
+ return typeof converted === "string" ? parseInt(converted) : converted;
+ };
+
+ const Independent = new Map<string, IndependentEntry>([
+
+ ["clear", {
+ action: (target: DocumentView) => Doc.GetProto(target.props.Document).data = new List(),
+ restrictTo: [DocumentType.COL]
+ }],
+
+ ["open fields", {
+ action: (target: DocumentView) => {
+ let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 });
+ target.props.addDocTab(kvp, target.dataDoc, "onRight");
+ }
+ }],
+
+ ["promote", {
+ action: (target: DocumentView) => {
+ console.log(target);
+ },
+ restrictTo: [DocumentType.TEXT]
+ }]
+
+ ]);
+
+ const Dependent = new Array<DependentEntry>(
+
+ {
+ expression: /create (\w+) documents of type (image|nested collection)/g,
+ action: (target: DocumentView, matches: RegExpExecArray) => {
+ let count = interpretNumber(matches[1]);
+ let what = matches[2];
+ let dataDoc = Doc.GetProto(target.props.Document);
+ let fieldKey = "data";
+ for (let i = 0; i < count; i++) {
+ let created: Doc | undefined;
+ switch (what) {
+ case "image":
+ created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg");
+ break;
+ case "nested collection":
+ created = Docs.Create.FreeformDocument([], {});
+ break;
+ }
+ created && Doc.AddDocToList(dataDoc, fieldKey, created);
+ }
+ },
+ restrictTo: [DocumentType.COL]
+ },
+
+ {
+ expression: /view as (freeform|stacking|masonry|schema|tree)/g,
+ action: (target: DocumentView, matches: RegExpExecArray) => {
+ let mode = CollectionViewType.valueOf(matches[1]);
+ mode && (target.props.Document.viewType = mode);
+ },
+ restrictTo: [DocumentType.COL]
+ }
+
+ );
}
diff --git a/src/client/util/DocumentManager.ts b/src/client/util/DocumentManager.ts
index 32f728c71..7f526b247 100644
--- a/src/client/util/DocumentManager.ts
+++ b/src/client/util/DocumentManager.ts
@@ -1,7 +1,7 @@
import { action, computed, observable } from 'mobx';
-import { Doc } from '../../new_fields/Doc';
+import { Doc, DocListCastAsync } from '../../new_fields/Doc';
import { Id } from '../../new_fields/FieldSymbols';
-import { BoolCast, Cast, NumCast } from '../../new_fields/Types';
+import { Cast, NumCast } from '../../new_fields/Types';
import { CollectionDockingView } from '../views/collections/CollectionDockingView';
import { CollectionPDFView } from '../views/collections/CollectionPDFView';
import { CollectionVideoView } from '../views/collections/CollectionVideoView';
@@ -104,7 +104,7 @@ export class DocumentManager {
@computed
public get LinkedDocumentViews() {
- let pairs = DocumentManager.Instance.DocumentViews.filter(dv => dv.isSelected() || BoolCast(dv.props.Document.libraryBrush)).reduce((pairs, dv) => {
+ let pairs = DocumentManager.Instance.DocumentViews.filter(dv => dv.isSelected() || Doc.IsBrushed(dv.props.Document)).reduce((pairs, dv) => {
let linksList = LinkManager.Instance.getAllRelatedLinks(dv.props.Document);
pairs.push(...linksList.reduce((pairs, link) => {
if (link) {
@@ -138,15 +138,17 @@ export class DocumentManager {
let docView: DocumentView | null;
// using forceDockFunc as a flag for splitting linked to doc to the right...can change later if needed
if (!forceDockFunc && (docView = DocumentManager.Instance.getDocumentView(doc))) {
- docView.props.Document.libraryBrush = true;
+ Doc.BrushDoc(docView.props.Document);
if (linkPage !== undefined) docView.props.Document.curPage = linkPage;
- UndoManager.RunInBatch(() => {
- docView!.props.focus(docView!.props.Document, willZoom);
- }, "focus");
+ UndoManager.RunInBatch(() => docView!.props.focus(docView!.props.Document, willZoom), "focus");
} else {
if (!contextDoc) {
- if (docContext) {
+ let docs = docContext ? await DocListCastAsync(docContext.data) : undefined;
+ let found = false;
+ docs && docs.map(d => found = found || Doc.AreProtosEqual(d, docDelegate));
+ if (docContext && found) {
let targetContextView: DocumentView | null;
+
if (!forceDockFunc && docContext && (targetContextView = DocumentManager.Instance.getDocumentView(docContext))) {
docContext.panTransformType = "Ease";
targetContextView.props.focus(docDelegate, willZoom);
@@ -158,13 +160,13 @@ export class DocumentManager {
}
} else {
const actualDoc = Doc.MakeAlias(docDelegate);
- actualDoc.libraryBrush = true;
+ Doc.BrushDoc(actualDoc);
if (linkPage !== undefined) actualDoc.curPage = linkPage;
(dockFunc || CollectionDockingView.Instance.AddRightSplit)(actualDoc, undefined);
}
} else {
let contextView: DocumentView | null;
- docDelegate.libraryBrush = true;
+ Doc.BrushDoc(docDelegate);
if (!forceDockFunc && (contextView = DocumentManager.Instance.getDocumentView(contextDoc))) {
contextDoc.panTransformType = "Ease";
contextView.props.focus(docDelegate, willZoom);
diff --git a/src/client/util/DragManager.ts b/src/client/util/DragManager.ts
index abcc3a4e1..a7aaaed7c 100644
--- a/src/client/util/DragManager.ts
+++ b/src/client/util/DragManager.ts
@@ -10,6 +10,7 @@ import { LinkManager } from "./LinkManager";
import { SelectionManager } from "./SelectionManager";
import { SchemaHeaderField } from "../../new_fields/SchemaHeaderField";
import { DocumentDecorations } from "../views/DocumentDecorations";
+import { NumberLiteralType } from "typescript";
export type dropActionType = "alias" | "copy" | undefined;
export function SetupDrag(
@@ -140,6 +141,10 @@ export namespace DragManager {
dragHasStarted?: () => void;
withoutShiftDrag?: boolean;
+
+ offsetX?: number;
+
+ offsetY?: number;
}
export interface DragDropDisposer {
@@ -399,7 +404,8 @@ export namespace DragManager {
hideSource = options.hideSource();
}
}
- eles.map(ele => (ele.hidden = hideSource));
+ eles.map(ele => (ele.hidden = hideSource) &&
+ (ele.parentElement && ele.parentElement.className.indexOf("collectionFreeFormDocumentView") !== -1 && (ele.parentElement.hidden = hideSource)));
let lastX = downX;
let lastY = downY;
@@ -423,13 +429,16 @@ export namespace DragManager {
lastX = e.pageX;
lastY = e.pageY;
dragElements.map((dragElement, i) => (dragElement.style.transform =
- `translate(${(xs[i] += moveX)}px, ${(ys[i] += moveY)}px) scale(${scaleXs[i]}, ${scaleYs[i]})`)
+ `translate(${(xs[i] += moveX) + (options ? (options.offsetX || 0) : 0)}px, ${(ys[i] += moveY) + (options ? (options.offsetY || 0) : 0)}px) scale(${scaleXs[i]}, ${scaleYs[i]})`)
);
};
let hideDragElements = () => {
dragElements.map(dragElement => dragElement.parentNode === dragDiv && dragDiv.removeChild(dragElement));
- eles.map(ele => (ele.hidden = false));
+ eles.map(ele => {
+ ele.hidden = false;
+ (ele.parentElement && ele.parentElement.className.indexOf("collectionFreeFormDocumentView") !== -1 && (ele.parentElement.hidden = false));
+ });
};
let endDrag = () => {
document.removeEventListener("pointermove", moveHandler, true);
diff --git a/src/client/util/LinkManager.ts b/src/client/util/LinkManager.ts
index a647f22c1..8a668e8d8 100644
--- a/src/client/util/LinkManager.ts
+++ b/src/client/util/LinkManager.ts
@@ -6,6 +6,7 @@ import { List } from "../../new_fields/List";
import { Id } from "../../new_fields/FieldSymbols";
import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils";
import { Docs } from "../documents/Documents";
+import { Scripting } from "./Scripting";
/*
@@ -42,7 +43,12 @@ export class LinkManager {
}
public getAllLinks(): Doc[] {
- return LinkManager.Instance.LinkManagerDoc ? LinkManager.Instance.LinkManagerDoc.allLinks ? DocListCast(LinkManager.Instance.LinkManagerDoc.allLinks) : [] : [];
+ let ldoc = LinkManager.Instance.LinkManagerDoc;
+ if (ldoc) {
+ let docs = DocListCast(ldoc.allLinks);
+ return docs;
+ }
+ return [];
}
public addLink(linkDoc: Doc): boolean {
@@ -242,4 +248,8 @@ export class LinkManager {
return Cast(linkDoc.anchor1, Doc, null);
}
}
-} \ No newline at end of file
+}
+Scripting.addGlobal(function links(doc: any) {
+ return new List(LinkManager.Instance.getAllRelatedLinks(doc));
+});
+
diff --git a/src/client/util/SelectionManager.ts b/src/client/util/SelectionManager.ts
index 9efef888d..ee623d082 100644
--- a/src/client/util/SelectionManager.ts
+++ b/src/client/util/SelectionManager.ts
@@ -4,6 +4,7 @@ import { DocumentView } from "../views/nodes/DocumentView";
import { FormattedTextBox } from "../views/nodes/FormattedTextBox";
import { NumCast, StrCast } from "../../new_fields/Types";
import { InkingControl } from "../views/InkingControl";
+import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils";
export namespace SelectionManager {
diff --git a/src/client/util/SerializationHelper.ts b/src/client/util/SerializationHelper.ts
index 13302be21..ff048f647 100644
--- a/src/client/util/SerializationHelper.ts
+++ b/src/client/util/SerializationHelper.ts
@@ -34,7 +34,7 @@ export namespace SerializationHelper {
return json;
}
- export async function Deserialize(obj: any, cb: (val: any) => void = emptyFunction): Promise<any> {
+ export async function Deserialize(obj: any): Promise<any> {
if (obj === undefined || obj === null) {
return undefined;
}
@@ -57,7 +57,7 @@ export namespace SerializationHelper {
}
const type = serializationTypes[obj.__type];
- const value = await new Promise(res => cb(deserialize(type.ctor, obj, (err, result) => res(result))));
+ const value = await new Promise(res => deserialize(type.ctor, obj, (err, result) => res(result)));
if (type.afterDeserialize) {
await type.afterDeserialize(value);
}
diff --git a/src/client/util/type_decls.d b/src/client/util/type_decls.d
index 79a4e50d5..622e10960 100644
--- a/src/client/util/type_decls.d
+++ b/src/client/util/type_decls.d
@@ -74,6 +74,7 @@ interface String {
normalize(form: "NFC" | "NFD" | "NFKC" | "NFKD"): string;
normalize(form?: string): string;
repeat(count: number): string;
+ replace(a:any, b:any):string; // bcz: fix this
startsWith(searchString: string, position?: number): boolean;
anchor(name: string): string;
big(): string;
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-18 13:29:49 +0000