diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/cognitive_services/CognitiveServices.ts | 34 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 1 | ||||
| -rw-r--r-- | src/client/views/nodes/DocumentView.tsx | 4 | ||||
| -rw-r--r-- | src/server/index.ts | 4 |
4 files changed, 35 insertions, 8 deletions
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 076907f09..720892b61 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -1,15 +1,13 @@ import * as request from "request-promise"; import { Doc, Field, Opt } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; -import { ImageField } from "../../new_fields/URLField"; -import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; -import { CompileScript } from "../util/Scripting"; -import { ComputedField } from "../../new_fields/ScriptField"; import { InkData } from "../../new_fields/InkField"; -import { undoBatch, UndoManager } from "../util/UndoManager"; +import "microsoft-cognitiveservices-speech-sdk"; +import "fs"; +import { UndoManager } from "../util/UndoManager"; type APIManager<D> = { converter: BodyConverter<D>, requester: RequestExecutor, analyzer: AnalysisApplier }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise<string>; @@ -17,13 +15,19 @@ type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => an type BodyConverter<D> = (data: D) => string; type Converter = (results: any) => Field; +namespace CORE { + export interface IWindow extends Window { + webkitSpeechRecognition: any; + } +} + export type Tag = { name: string, confidence: number }; export type Rectangle = { top: number, left: number, width: number, height: number }; export enum Service { ComputerVision = "vision", Face = "face", - Handwriting = "handwriting" + Handwriting = "handwriting", } export enum Confidence { @@ -215,4 +219,22 @@ export namespace CognitiveServices { } + export namespace Transcription { + + export const analyzer = (doc: Doc, keys: string[]) => { + let { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; + let recognizer = new webkitSpeechRecognition(); + recognizer.interimResults = true; + recognizer.continuous = true; + + recognizer.onresult = (e: any) => { + let result = e.results[0][0]; + doc[keys[0]] = result.transcript; + }; + + recognizer.start(); + }; + + } + }
\ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 91c8fe57c..ccf8f571e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -38,6 +38,7 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; +import { CognitiveServices } from '../cognitive_services/CognitiveServices'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; @observer diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4b5cf3a43..fd53cd451 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -41,6 +41,7 @@ import { ClientUtils } from '../../util/ClientUtils'; import { EditableView } from '../EditableView'; import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons'; import { DocumentDecorations } from '../DocumentDecorations'; +import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -62,7 +63,7 @@ library.add(fa.faCrosshairs); library.add(fa.faDesktop); library.add(fa.faUnlock); library.add(fa.faLock); -library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake); +library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake, fa.faMicrophone); // const linkSchema = createSchema({ // title: "string", @@ -558,6 +559,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu cm.addItem({ description: BoolCast(this.props.Document.ignoreAspect, false) || !this.props.Document.nativeWidth || !this.props.Document.nativeHeight ? "Freeze" : "Unfreeze", event: this.freezeNativeDimensions, icon: "snowflake" }); cm.addItem({ description: "Pin to Presentation", event: () => PresentationView.Instance.PinDoc(this.props.Document), icon: "map-pin" }); cm.addItem({ description: BoolCast(this.props.Document.lockedPosition) ? "Unlock Position" : "Lock Position", event: this.toggleLockPosition, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" }); + cm.addItem({ description: "Transcribe Speech", event: () => CognitiveServices.Transcription.analyzer(Doc.GetProto(this.props.Document), ["transcript"]), icon: "microphone" }); let makes: ContextMenuProps[] = []; makes.push({ description: "Make Background", event: this.makeBackground, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" }); makes.push({ description: this.props.Document.isButton ? "Remove Button" : "Make Button", event: this.makeBtnClicked, icon: "concierge-bell" }); diff --git a/src/server/index.ts b/src/server/index.ts index adf218be6..f4bbd4423 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -40,6 +40,8 @@ import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); import { Response } from 'express-serve-static-core'; +import { AudioInputStream, AudioConfig, SpeechConfig, SpeechRecognizer, SpeechRecognitionResult } from 'microsoft-cognitiveservices-speech-sdk'; +import { Opt } from '../new_fields/Doc'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -297,7 +299,7 @@ addSecureRoute( const ServicesApiKeyMap = new Map<string, string | undefined>([ ["face", process.env.FACE], ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] + ["handwriting", process.env.HANDWRITING], ]); addSecureRoute(Method.GET, (user, res, req) => { |