diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/cognitive_services/CognitiveServices.ts | 32 | ||||
| -rw-r--r-- | src/client/documents/Documents.ts | 45 | ||||
| -rw-r--r-- | src/client/util/DictationManager.ts | 324 | ||||
| -rw-r--r-- | src/client/views/ContextMenuItem.tsx | 2 | ||||
| -rw-r--r-- | src/client/views/GlobalKeyHandler.ts | 39 | ||||
| -rw-r--r-- | src/client/views/Main.scss | 30 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 71 | ||||
| -rw-r--r-- | src/client/views/collections/CollectionBaseView.tsx | 18 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 84 | ||||
| -rw-r--r-- | src/client/views/nodes/DocumentView.tsx | 14 | ||||
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 8 | ||||
| -rw-r--r-- | src/new_fields/Types.ts | 8 |
12 files changed, 550 insertions, 125 deletions
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index c118d91d3..08fcb4883 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -7,9 +7,9 @@ import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; -type APIManager<D> = { converter: BodyConverter<D>, requester: RequestExecutor, analyzer: AnalysisApplier }; +type APIManager<D> = { converter: BodyConverter<D>, requester: RequestExecutor }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise<string>; -type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => any; +type AnalysisApplier<D> = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any; type BodyConverter<D> = (data: D) => string; type Converter = (results: any) => Field; @@ -38,7 +38,7 @@ export enum Confidence { */ export namespace CognitiveServices { - const ExecuteQuery = async <D, R>(service: Service, manager: APIManager<D>, data: D): Promise<Opt<R>> => { + const ExecuteQuery = async <D>(service: Service, manager: APIManager<D>, data: D): Promise<any> => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { @@ -46,7 +46,7 @@ export namespace CognitiveServices { return undefined; } - let results: Opt<R>; + let results: any; try { results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); } catch { @@ -99,7 +99,11 @@ export namespace CognitiveServices { return request.post(options); }, - analyzer: async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { + }; + + export namespace Appliers { + + export const ProcessImage: AnalysisApplier<string> = async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { let batch = UndoManager.StartBatch("Image Analysis"); let storageKey = keys[0]; @@ -107,7 +111,7 @@ export namespace CognitiveServices { return; } let toStore: any; - let results = await ExecuteQuery<string, any>(service, Manager, url); + let results = await ExecuteQuery(service, Manager, url); if (!results) { toStore = "Cognitive Services could not process the given image URL."; } else { @@ -120,9 +124,9 @@ export namespace CognitiveServices { target[storageKey] = toStore; batch.end(); - } + }; - }; + } export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; @@ -179,10 +183,14 @@ export namespace CognitiveServices { return new Promise<any>(promisified); }, - analyzer: async (target: Doc, keys: string[], inkData: InkData) => { + }; + + export namespace Appliers { + + export const ConcatenateHandwriting: AnalysisApplier<InkData> = async (target: Doc, keys: string[], inkData: InkData) => { let batch = UndoManager.StartBatch("Ink Analysis"); - let results = await ExecuteQuery<InkData, any>(Service.Handwriting, Manager, inkData); + let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); @@ -192,9 +200,9 @@ export namespace CognitiveServices { } batch.end(); - } + }; - }; + } export interface AzureStrokeData { id: number; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 07e38a4c0..8142b2a1b 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -1,3 +1,24 @@ +export enum DocumentType { + NONE = "none", + TEXT = "text", + HIST = "histogram", + IMG = "image", + WEB = "web", + COL = "collection", + KVP = "kvp", + VID = "video", + AUDIO = "audio", + PDF = "pdf", + ICON = "icon", + IMPORT = "import", + LINK = "link", + LINKDOC = "linkdoc", + BUTTON = "button", + TEMPLATE = "template", + EXTENSION = "extension", + YOUTUBE = "youtube", +} + import { HistogramField } from "../northstar/dash-fields/HistogramField"; import { HistogramBox } from "../northstar/dash-nodes/HistogramBox"; import { HistogramOperation } from "../northstar/operations/HistogramOperation"; @@ -25,14 +46,13 @@ import { OmitKeys, JSONUtils } from "../../Utils"; import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField } from "../../new_fields/URLField"; import { HtmlField } from "../../new_fields/HtmlField"; import { List } from "../../new_fields/List"; -import { Cast, NumCast, StrCast, ToConstructor, InterfaceValue, FieldValue } from "../../new_fields/Types"; +import { Cast, NumCast } from "../../new_fields/Types"; import { IconField } from "../../new_fields/IconField"; import { listSpec } from "../../new_fields/Schema"; import { DocServer } from "../DocServer"; import { dropActionType } from "../util/DragManager"; import { DateField } from "../../new_fields/DateField"; import { UndoManager } from "../util/UndoManager"; -import { RouteStore } from "../../server/RouteStore"; import { YoutubeBox } from "../apis/youtube/YoutubeBox"; import { CollectionDockingView } from "../views/collections/CollectionDockingView"; import { LinkManager } from "../util/LinkManager"; @@ -46,27 +66,6 @@ import { ProxyField } from "../../new_fields/Proxy"; var requestImageSize = require('../util/request-image-size'); var path = require('path'); -export enum DocumentType { - NONE = "none", - TEXT = "text", - HIST = "histogram", - IMG = "image", - WEB = "web", - COL = "collection", - KVP = "kvp", - VID = "video", - AUDIO = "audio", - PDF = "pdf", - ICON = "icon", - IMPORT = "import", - LINK = "link", - LINKDOC = "linkdoc", - BUTTON = "button", - TEMPLATE = "template", - EXTENSION = "extension", - YOUTUBE = "youtube", -} - export interface DocumentOptions { x?: number; y?: number; diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index b58bdb6c7..b6f871713 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -1,38 +1,306 @@ -namespace CORE { - export interface IWindow extends Window { - webkitSpeechRecognition: any; +import { SelectionManager } from "./SelectionManager"; +import { DocumentView } from "../views/nodes/DocumentView"; +import { UndoManager } from "./UndoManager"; +import * as interpreter from "words-to-numbers"; +import { Doc } from "../../new_fields/Doc"; +import { List } from "../../new_fields/List"; +import { Docs, DocumentType } from "../documents/Documents"; +import { CollectionViewType } from "../views/collections/CollectionBaseView"; +import { Cast, CastCtor } from "../../new_fields/Types"; +import { listSpec } from "../../new_fields/Schema"; +import { AudioField, ImageField } from "../../new_fields/URLField"; +import { HistogramField } from "../northstar/dash-fields/HistogramField"; + +/** + * This namespace provides a singleton instance of a manager that + * handles the listening and text-conversion of user speech. + * + * The basic manager functionality can be attained by the DictationManager.Controls namespace, which provide + * a simple recording operation that returns the interpreted text as a string. + * + * Additionally, however, the DictationManager also exposes the ability to execute voice commands within Dash. + * It stores a default library of registered commands that can be triggered by listen()'ing for a phrase and then + * passing the results into the execute() function. + * + * In addition to compile-time default commands, you can invoke DictationManager.Commands.Register(Independent|Dependent) + * to add new commands as classes or components are constructed. + */ +export namespace DictationManager { + + /** + * Some type maneuvering to access Webkit's built-in + * speech recognizer. + */ + namespace CORE { + export interface IWindow extends Window { + webkitSpeechRecognition: any; + } } -} + const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; -const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; + export namespace Controls { -export default class DictationManager { - public static Instance = new DictationManager(); - private isListening = false; - private recognizer: any; + const intraSession = ". "; + const interSession = " ... "; - constructor() { - this.recognizer = new webkitSpeechRecognition(); - this.recognizer.interimResults = false; - this.recognizer.continuous = true; - } + let isListening = false; + let isManuallyStopped = false; - finish = (handler: any, data: any) => { - handler(data); - this.isListening = false; - this.recognizer.stop(); - } + let current: string | undefined = undefined; + let sessionResults: string[] = []; + + const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition(); + recognizer.onstart = () => console.log("initiating speech recognition session..."); - listen = () => { - if (this.isListening) { - return undefined; + export type InterimResultHandler = (results: any) => any; + export type ContinuityArgs = { indefinite: boolean } | false; + export type DelimiterArgs = { inter: string, intra: string }; + + export interface ListeningOptions { + language: string; + continuous: ContinuityArgs; + delimiters: DelimiterArgs; + interimHandler: InterimResultHandler; } - this.isListening = true; - this.recognizer.start(); - return new Promise<string>((resolve, reject) => { - this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript); - this.recognizer.onerror = (e: any) => this.finish(reject, e); - }); + + export const listen = async (options?: Partial<ListeningOptions>) => { + let results: any; + try { + results = await listenImpl(options); + } catch (e) { + results = `Dictation Error: ${"error" in e ? e.error : "unknown error"}`; + } + return results; + }; + + const listenImpl = (options?: Partial<ListeningOptions>) => { + if (isListening) { + return undefined; + } + isListening = true; + + let handler = options ? options.interimHandler : undefined; + let continuous = options ? options.continuous : undefined; + let indefinite = continuous && continuous.indefinite; + let language = options ? options.language : undefined; + let intra = options && options.delimiters ? options.delimiters.intra : undefined; + let inter = options && options.delimiters ? options.delimiters.inter : undefined; + + recognizer.interimResults = handler !== undefined; + recognizer.continuous = continuous === undefined ? false : continuous !== false; + recognizer.lang = language === undefined ? "en-US" : language; + + recognizer.start(); + + return new Promise<string>((resolve, reject) => { + + recognizer.onerror = (e: SpeechRecognitionError) => { + if (!(indefinite && e.error === "no-speech")) { + recognizer.stop(); + reject(e); + } + }; + + recognizer.onresult = (e: SpeechRecognitionEvent) => { + current = synthesize(e, intra); + handler && handler(current); + isManuallyStopped && complete(); + }; + + recognizer.onend = (e: Event) => { + if (!indefinite || isManuallyStopped) { + return complete(); + } + + if (current) { + sessionResults.push(current); + current = undefined; + } + recognizer.start(); + }; + + let complete = () => { + if (indefinite) { + current && sessionResults.push(current); + resolve(sessionResults.join(inter || interSession)); + } else { + resolve(current); + } + reset(); + }; + + }); + }; + + export const stop = (salvageSession = true) => { + isManuallyStopped = true; + salvageSession ? recognizer.stop() : recognizer.abort(); + }; + + const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => { + let results = e.results; + let transcripts: string[] = []; + for (let i = 0; i < results.length; i++) { + transcripts.push(results.item(i).item(0).transcript.trim()); + } + return transcripts.join(delimiter || intraSession); + }; + + const reset = () => { + isListening = false; + isManuallyStopped = false; + recognizer.onresult = null; + recognizer.onerror = null; + recognizer.onend = null; + sessionResults = []; + }; + + } + + export namespace Commands { + + export const dictationFadeDuration = 2000; + + export type IndependentAction = (target: DocumentView) => any | Promise<any>; + export type IndependentEntry = { action: IndependentAction, restrictTo?: DocumentType[] }; + + export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise<any>; + export type DependentEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] }; + + export const RegisterIndependent = (key: string, value: IndependentEntry) => Independent.set(key, value); + export const RegisterDependent = (entry: DependentEntry) => Dependent.push(entry); + + export const execute = async (phrase: string) => { + return UndoManager.RunInBatch(async () => { + let targets = SelectionManager.SelectedDocuments(); + if (!targets || !targets.length) { + return; + } + + phrase = phrase.toLowerCase(); + let entry = Independent.get(phrase); + + if (entry) { + let success = false; + let restrictTo = entry.restrictTo; + for (let target of targets) { + if (!restrictTo || validate(target, restrictTo)) { + await entry.action(target); + success = true; + } + } + return success; + } + + for (let entry of Dependent) { + let regex = entry.expression; + let matches = regex.exec(phrase); + regex.lastIndex = 0; + if (matches !== null) { + let success = false; + let restrictTo = entry.restrictTo; + for (let target of targets) { + if (!restrictTo || validate(target, restrictTo)) { + await entry.action(target, matches); + success = true; + } + } + return success; + } + } + + return false; + }, "Execute Command"); + }; + + const ConstructorMap = new Map<DocumentType, CastCtor>([ + [DocumentType.COL, listSpec(Doc)], + [DocumentType.AUDIO, AudioField], + [DocumentType.IMG, ImageField], + [DocumentType.HIST, HistogramField], + [DocumentType.IMPORT, listSpec(Doc)] + ]); + + const tryCast = (view: DocumentView, type: DocumentType) => { + let ctor = ConstructorMap.get(type); + if (!ctor) { + return false; + } + return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined; + }; + + const validate = (target: DocumentView, types: DocumentType[]) => { + for (let type of types) { + if (tryCast(target, type)) { + return true; + } + } + return false; + }; + + const interpretNumber = (number: string) => { + let initial = parseInt(number); + if (!isNaN(initial)) { + return initial; + } + let converted = interpreter.wordsToNumbers(number, { fuzzy: true }); + if (converted === null) { + return NaN; + } + return typeof converted === "string" ? parseInt(converted) : converted; + }; + + const Independent = new Map<string, IndependentEntry>([ + + ["clear", { + action: (target: DocumentView) => Doc.GetProto(target.props.Document).data = new List(), + restrictTo: [DocumentType.COL] + }], + + ["open fields", { + action: (target: DocumentView) => { + let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 }); + target.props.addDocTab(kvp, target.dataDoc, "onRight"); + } + }] + + ]); + + const Dependent = new Array<DependentEntry>( + + { + expression: /create (\w+) documents of type (image|nested collection)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { + let count = interpretNumber(matches[1]); + let what = matches[2]; + let dataDoc = Doc.GetProto(target.props.Document); + let fieldKey = "data"; + for (let i = 0; i < count; i++) { + let created: Doc | undefined; + switch (what) { + case "image": + created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"); + break; + case "nested collection": + created = Docs.Create.FreeformDocument([], {}); + break; + } + created && Doc.AddDocToList(dataDoc, fieldKey, created); + } + }, + restrictTo: [DocumentType.COL] + }, + + { + expression: /view as (freeform|stacking|masonry|schema|tree)/g, + action: (target: DocumentView, matches: RegExpExecArray) => { + let mode = CollectionViewType.valueOf(matches[1]); + mode && (target.props.Document.viewType = mode); + }, + restrictTo: [DocumentType.COL] + } + + ); } diff --git a/src/client/views/ContextMenuItem.tsx b/src/client/views/ContextMenuItem.tsx index a1787e78f..90f7be33f 100644 --- a/src/client/views/ContextMenuItem.tsx +++ b/src/client/views/ContextMenuItem.tsx @@ -39,13 +39,13 @@ export class ContextMenuItem extends React.Component<ContextMenuProps & { select handleEvent = async (e: React.MouseEvent<HTMLDivElement>) => { if ("event" in this.props) { + this.props.closeMenu && this.props.closeMenu(); let batch: UndoManager.Batch | undefined; if (this.props.undoable !== false) { batch = UndoManager.StartBatch(`Context menu event: ${this.props.description}`); } await this.props.event(); batch && batch.end(); - this.props.closeMenu && this.props.closeMenu(); } } diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index ea2e3e196..c3e6ae6c8 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -3,12 +3,9 @@ import { SelectionManager } from "../util/SelectionManager"; import { CollectionDockingView } from "./collections/CollectionDockingView"; import { MainView } from "./MainView"; import { DragManager } from "../util/DragManager"; -import { action } from "mobx"; +import { action, runInAction } from "mobx"; import { Doc } from "../../new_fields/Doc"; -import { CognitiveServices } from "../cognitive_services/CognitiveServices"; -import DictationManager from "../util/DictationManager"; -import { ContextMenu } from "./ContextMenu"; -import { ContextMenuProps } from "./ContextMenuItem"; +import { DictationManager } from "../util/DictationManager"; const modifiers = ["control", "meta", "shift", "alt"]; type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise<KeyControlInfo>; @@ -62,7 +59,8 @@ export default class KeyManager { private unmodified = action((keyname: string, e: KeyboardEvent) => { switch (keyname) { case "escape": - if (MainView.Instance.isPointerDown) { + let main = MainView.Instance; + if (main.isPointerDown) { DragManager.AbortDrag(); } else { if (CollectionDockingView.Instance.HasFullScreen()) { @@ -71,8 +69,14 @@ export default class KeyManager { SelectionManager.DeselectAll(); } } - MainView.Instance.toggleColorPicker(true); + main.toggleColorPicker(true); SelectionManager.DeselectAll(); + DictationManager.Controls.stop(); + if (main.dictationOverlayVisible) { + main.dictationOverlayVisible = false; + main.dictationSuccess = undefined; + main.cancelDictationFade(); + } break; case "delete": case "backspace": @@ -106,13 +110,24 @@ export default class KeyManager { switch (keyname) { case " ": - let transcript = await DictationManager.Instance.listen(); - console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`); - let command: ContextMenuProps | undefined; - transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event(); + let main = MainView.Instance; + main.dictationOverlayVisible = true; + + main.isListening = true; + // let printer = (results: any) => console.log(results); + let command = await DictationManager.Controls.listen(); + main.isListening = false; + + if (!command) { + break; + } + + main.dictatedPhrase = command = command.toLowerCase(); + main.dictationSuccess = await DictationManager.Commands.execute(command); + main.initiateDictationFade(); + stopPropagation = true; preventDefault = true; - break; } return { diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss index eed2ae4fa..8e57b88c3 100644 --- a/src/client/views/Main.scss +++ b/src/client/views/Main.scss @@ -266,4 +266,34 @@ ul#add-options-list { height: 25%; position: relative; display: flex; +} + +.dictation-prompt { + position: absolute; + z-index: 1000; + text-align: center; + justify-content: center; + align-self: center; + align-content: center; + padding: 20px; + background: gainsboro; + border-radius: 10px; + border: 3px solid black; + box-shadow: #00000044 5px 5px 10px; + transform: translate(-50%, -50%); + top: 50%; + font-style: italic; + left: 50%; + transition: 0.5s all ease; + pointer-events: none; +} + +.dictation-prompt-overlay { + width: 100%; + height: 100%; + position: absolute; + background: darkslategray; + z-index: 999; + transition: 0.5s all ease; + pointer-events: none; }
\ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 53f589684..4443eea6d 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,6 +39,7 @@ import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; +import { DictationManager } from '../util/DictationManager'; @observer export class MainView extends React.Component { @@ -47,6 +48,29 @@ export class MainView extends React.Component { @observable private _workspacesShown: boolean = false; @observable public pwidth: number = 0; @observable public pheight: number = 0; + + @observable private dictationState = ""; + @observable private dictationSuccessState: boolean | undefined = undefined; + @observable private dictationDisplayState = false; + @observable private dictationListeningState = false; + + public overlayTimeout: NodeJS.Timeout | undefined; + + public initiateDictationFade = () => { + let duration = DictationManager.Commands.dictationFadeDuration; + this.overlayTimeout = setTimeout(() => { + this.dictationOverlayVisible = false; + this.dictationSuccess = undefined; + }, duration); + } + + public cancelDictationFade = () => { + if (this.overlayTimeout) { + clearTimeout(this.overlayTimeout); + this.overlayTimeout = undefined; + } + } + @computed private get mainContainer(): Opt<Doc> { return FieldValue(Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc)); } @@ -64,6 +88,38 @@ export class MainView extends React.Component { } } + public get dictatedPhrase() { + return this.dictationState; + } + + public set dictatedPhrase(value: string) { + runInAction(() => this.dictationState = value); + } + + public get dictationSuccess() { + return this.dictationSuccessState; + } + + public set dictationSuccess(value: boolean | undefined) { + runInAction(() => this.dictationSuccessState = value); + } + + public get dictationOverlayVisible() { + return this.dictationDisplayState; + } + + public set dictationOverlayVisible(value: boolean) { + runInAction(() => this.dictationDisplayState = value); + } + + public get isListening() { + return this.dictationListeningState; + } + + public set isListening(value: boolean) { + runInAction(() => this.dictationListeningState = value); + } + componentWillMount() { var tag = document.createElement('script'); @@ -465,8 +521,23 @@ export class MainView extends React.Component { } render() { + let display = this.dictationOverlayVisible; + let success = this.dictationSuccess; + let result = this.isListening ? "Listening..." : `"${this.dictatedPhrase}"`; return ( <div id="main-div"> + <div + className={"dictation-prompt"} + style={{ + opacity: display ? 1 : 0, + background: success === undefined ? "gainsboro" : success ? "lawngreen" : "red", + borderColor: this.isListening ? "red" : "black", + }} + >{result}</div> + <div + className={"dictation-prompt-overlay"} + style={{ opacity: display ? 0.4 : 0 }} + /> <DocumentDecorations /> {this.mainContent} <PreviewCursor /> diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx index 6801b94fd..b53e83eb1 100644 --- a/src/client/views/collections/CollectionBaseView.tsx +++ b/src/client/views/collections/CollectionBaseView.tsx @@ -22,6 +22,24 @@ export enum CollectionViewType { Masonry } +export namespace CollectionViewType { + + const stringMapping = new Map<string, CollectionViewType>([ + ["invalid", CollectionViewType.Invalid], + ["freeform", CollectionViewType.Freeform], + ["schema", CollectionViewType.Schema], + ["docking", CollectionViewType.Docking], + ["tree", CollectionViewType.Tree], + ["stacking", CollectionViewType.Stacking], + ["masonry", CollectionViewType.Masonry] + ]); + + export const valueOf = (value: string) => { + return stringMapping.get(value.toLowerCase()); + }; + +} + export interface CollectionRenderProps { addDocument: (document: Doc, allowDuplicates?: boolean) => boolean; removeDocument: (document: Doc) => boolean; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index c943fac74..86d019e29 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -29,15 +29,13 @@ import { DocumentViewProps, positionSchema } from "../../nodes/DocumentView"; import { pageSchema } from "../../nodes/ImageBox"; import { OverlayElementOptions, OverlayView } from "../../OverlayView"; import PDFMenu from "../../pdf/PDFMenu"; -import { ScriptBox } from "../../ScriptBox"; import { CollectionSubView } from "../CollectionSubView"; +import { ScriptBox } from "../../ScriptBox"; import { CollectionFreeFormLinksView } from "./CollectionFreeFormLinksView"; import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCursors"; import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); -import v5 = require("uuid/v5"); -import { setScheduler } from "bluebird"; import { DocumentType, Docs } from "../../../documents/Documents"; library.add(faEye as any, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload); @@ -644,11 +642,46 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } + fitToContainer = async () => this.props.Document.fitToBox = !this.fitToBox; + + arrangeContents = async () => { + const docs = await DocListCastAsync(this.Document[this.props.fieldKey]); + UndoManager.RunInBatch(() => { + if (docs) { + let startX = this.Document.panX || 0; + let x = startX; + let y = this.Document.panY || 0; + let i = 0; + const width = Math.max(...docs.map(doc => NumCast(doc.width))); + const height = Math.max(...docs.map(doc => NumCast(doc.height))); + for (const doc of docs) { + doc.x = x; + doc.y = y; + x += width + 20; + if (++i === 6) { + i = 0; + x = startX; + y += height + 20; + } + } + } + }, "arrange contents"); + } + + analyzeStrokes = async () => { + let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); + if (!data) { + return; + } + let relevantKeys = ["inkAnalysis", "handwriting"]; + CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.fieldExtensionDoc, relevantKeys, data.inkData); + } + onContextMenu = (e: React.MouseEvent) => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, - event: async () => this.props.Document.fitToBox = !this.fitToBox, + event: this.fitToContainer, icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt" }); layoutItems.push({ @@ -667,41 +700,18 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }); layoutItems.push({ description: "Arrange contents in grid", - icon: "table", - event: async () => { - const docs = await DocListCastAsync(this.Document[this.props.fieldKey]); - UndoManager.RunInBatch(() => { - if (docs) { - let startX = this.Document.panX || 0; - let x = startX; - let y = this.Document.panY || 0; - let i = 0; - const width = Math.max(...docs.map(doc => NumCast(doc.width))); - const height = Math.max(...docs.map(doc => NumCast(doc.height))); - for (const doc of docs) { - doc.x = x; - doc.y = y; - x += width + 20; - if (++i === 6) { - i = 0; - x = startX; - y += height + 20; - } - } - } - }, "arrange contents"); - } + event: this.arrangeContents, + icon: "table" }); - ContextMenu.Instance.addItem({ description: "Layout...", subitems: layoutItems, icon: "compass" }); ContextMenu.Instance.addItem({ - description: "Analyze Strokes", event: async () => { - let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); - if (!data) { - return; - } - let relevantKeys = ["inkAnalysis", "handwriting"]; - CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); - }, icon: "paint-brush" + description: "Layout...", + subitems: layoutItems, + icon: "compass" + }); + ContextMenu.Instance.addItem({ + description: "Analyze Strokes", + event: this.analyzeStrokes, + icon: "paint-brush" }); ContextMenu.Instance.addItem({ description: "Import document", icon: "upload", event: () => { diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4528cf94d..1d9cb3c80 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -15,7 +15,6 @@ import { emptyFunction, returnTrue, Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; import { Docs, DocUtils } from "../../documents/Documents"; import { ClientUtils } from '../../util/ClientUtils'; -import DictationManager from '../../util/DictationManager'; import { DocumentManager } from "../../util/DocumentManager"; import { DragManager, dropActionType } from "../../util/DragManager"; import { LinkManager } from '../../util/LinkManager'; @@ -38,6 +37,7 @@ import { DocumentContentsView } from "./DocumentContentsView"; import "./DocumentView.scss"; import { FormattedTextBox } from './FormattedTextBox'; import React = require("react"); +import { DictationManager } from '../../util/DictationManager'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -149,10 +149,6 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu set templates(templates: List<string>) { this.props.Document.templates = templates; } screenRect = (): ClientRect | DOMRect => this._mainCont.current ? this._mainCont.current.getBoundingClientRect() : new DOMRect(); - constructor(props: DocumentViewProps) { - super(props); - } - _animateToIconDisposer?: IReactionDisposer; _reactionDisposer?: IReactionDisposer; @action @@ -411,7 +407,10 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu deleteClicked = (): void => { SelectionManager.DeselectAll(); this.props.removeDocument && this.props.removeDocument(this.props.Document); } @undoBatch - fieldsClicked = (): void => { let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }); this.props.addDocTab(kvp, this.dataDoc, "onRight"); } + fieldsClicked = (): void => { + let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }); + this.props.addDocTab(kvp, this.dataDoc, "onRight"); + } @undoBatch makeBtnClicked = (): void => { @@ -537,7 +536,8 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu } listen = async () => { - let transcript = await DictationManager.Instance.listen(); + let options = { continuous: { indefinite: true }, delimiter: " " }; + let transcript = await DictationManager.Controls.listen(options); transcript && (Doc.GetProto(this.props.Document).transcript = transcript); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 0d9c2bb8a..ca0f637eb 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -234,7 +234,9 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); return faceDocs; }; - CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + if (this.url) { + CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + } } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { @@ -253,7 +255,9 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD tagDoc.confidence = threshold; return tagDoc; }; - CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + if (this.url) { + CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + } } @action diff --git a/src/new_fields/Types.ts b/src/new_fields/Types.ts index 565ae2ee3..09cbff25e 100644 --- a/src/new_fields/Types.ts +++ b/src/new_fields/Types.ts @@ -48,9 +48,11 @@ export interface Interface { } export type WithoutRefField<T extends Field> = T extends RefField ? never : T; -export function Cast<T extends ToConstructor<Field> | ListSpec<Field>>(field: FieldResult, ctor: T): FieldResult<ToType<T>>; -export function Cast<T extends ToConstructor<Field> | ListSpec<Field>>(field: FieldResult, ctor: T, defaultVal: WithoutList<WithoutRefField<ToType<T>>> | null): WithoutList<ToType<T>>; -export function Cast<T extends ToConstructor<Field> | ListSpec<Field>>(field: FieldResult, ctor: T, defaultVal?: ToType<T> | null): FieldResult<ToType<T>> | undefined { +export type CastCtor = ToConstructor<Field> | ListSpec<Field>; + +export function Cast<T extends CastCtor>(field: FieldResult, ctor: T): FieldResult<ToType<T>>; +export function Cast<T extends CastCtor>(field: FieldResult, ctor: T, defaultVal: WithoutList<WithoutRefField<ToType<T>>> | null): WithoutList<ToType<T>>; +export function Cast<T extends CastCtor>(field: FieldResult, ctor: T, defaultVal?: ToType<T> | null): FieldResult<ToType<T>> | undefined { if (field instanceof Promise) { return defaultVal === undefined ? field.then(f => Cast(f, ctor) as any) as any : defaultVal === null ? undefined : defaultVal; } |