aboutsummaryrefslogtreecommitdiff
path: root/src/client/views/nodes
diff options
context:
space:
mode:
Diffstat (limited to 'src/client/views/nodes')
-rw-r--r--src/client/views/nodes/DocumentContentsView.tsx3
-rw-r--r--src/client/views/nodes/DocumentView.tsx10
-rw-r--r--src/client/views/nodes/FaceRectangles.tsx2
-rw-r--r--src/client/views/nodes/ImageBox.tsx49
-rw-r--r--src/client/views/nodes/VideoBox.tsx88
5 files changed, 110 insertions, 42 deletions
diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx
index 91d4fb524..396233551 100644
--- a/src/client/views/nodes/DocumentContentsView.tsx
+++ b/src/client/views/nodes/DocumentContentsView.tsx
@@ -18,6 +18,7 @@ import { PDFBox } from "./PDFBox";
import { VideoBox } from "./VideoBox";
import { FieldView } from "./FieldView";
import { WebBox } from "./WebBox";
+import { YoutubeBox } from "./../../apis/youtube/YoutubeBox";
import { HistogramBox } from "../../northstar/dash-nodes/HistogramBox";
import React = require("react");
import { FieldViewProps } from "./FieldView";
@@ -98,7 +99,7 @@ export class DocumentContentsView extends React.Component<DocumentViewProps & {
if (this.props.renderDepth > 7) return (null);
if (!this.layout && (this.props.layoutKey !== "overlayLayout" || !this.templates.length)) return (null);
return <ObserverJsxParser
- components={{ FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, CollectionPDFView, CollectionVideoView, WebBox, KeyValueBox, PDFBox, VideoBox, AudioBox, HistogramBox }}
+ components={{ FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, CollectionPDFView, CollectionVideoView, WebBox, KeyValueBox, PDFBox, VideoBox, AudioBox, HistogramBox, YoutubeBox }}
bindings={this.CreateBindings()}
jsx={this.finalLayout}
showWarnings={true}
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 58e2443c2..51662274d 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -41,6 +41,8 @@ import { ClientUtils } from '../../util/ClientUtils';
import { EditableView } from '../EditableView';
import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons';
import { DocumentDecorations } from '../DocumentDecorations';
+import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
+import DictationManager from '../../util/DictationManager';
const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
library.add(fa.faTrash);
@@ -63,7 +65,7 @@ library.add(fa.faCrosshairs);
library.add(fa.faDesktop);
library.add(fa.faUnlock);
library.add(fa.faLock);
-library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake);
+library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake, fa.faMicrophone);
// const linkSchema = createSchema({
// title: "string",
@@ -536,6 +538,11 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
this.props.Document.lockedPosition = BoolCast(this.props.Document.lockedPosition) ? undefined : true;
}
+ listen = async () => {
+ let transcript = await DictationManager.Instance.listen();
+ transcript && (Doc.GetProto(this.props.Document).transcript = transcript);
+ }
+
@action
onContextMenu = async (e: React.MouseEvent): Promise<void> => {
e.persist();
@@ -559,6 +566,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
cm.addItem({ description: BoolCast(this.props.Document.ignoreAspect, false) || !this.props.Document.nativeWidth || !this.props.Document.nativeHeight ? "Freeze" : "Unfreeze", event: this.freezeNativeDimensions, icon: "snowflake" });
cm.addItem({ description: "Pin to Presentation", event: () => PresentationView.Instance.PinDoc(this.props.Document), icon: "map-pin" });
cm.addItem({ description: BoolCast(this.props.Document.lockedPosition) ? "Unlock Position" : "Lock Position", event: this.toggleLockPosition, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" });
+ cm.addItem({ description: "Transcribe Speech", event: this.listen, icon: "microphone" });
let makes: ContextMenuProps[] = [];
makes.push({ description: "Make Background", event: this.makeBackground, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" });
makes.push({ description: this.props.Document.isButton ? "Remove Button" : "Make Button", event: this.makeBtnClicked, icon: "concierge-bell" });
diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx
index 3570531b2..acf1aced3 100644
--- a/src/client/views/nodes/FaceRectangles.tsx
+++ b/src/client/views/nodes/FaceRectangles.tsx
@@ -20,7 +20,7 @@ export interface RectangleTemplate {
export default class FaceRectangles extends React.Component<FaceRectanglesProps> {
render() {
- let faces = DocListCast(Doc.GetProto(this.props.document).faces);
+ let faces = DocListCast(this.props.document.faces);
let templates: RectangleTemplate[] = faces.map(faceDoc => {
let rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc;
let style = {
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index d4e0eaf98..dbe545048 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -25,9 +25,12 @@ import { Docs, DocumentType } from '../../documents/Documents';
import { DocServer } from '../../DocServer';
import { Font } from '@react-pdf/renderer';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
-import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
+import { CognitiveServices, Service, Tag, Confidence } from '../../cognitive_services/CognitiveServices';
import FaceRectangles from './FaceRectangles';
import { faEye } from '@fortawesome/free-regular-svg-icons';
+import { ComputedField } from '../../../new_fields/ScriptField';
+import { CompileScript } from '../../util/Scripting';
+import { thisExpression } from 'babel-types';
var requestImageSize = require('../../util/request-image-size');
var path = require('path');
const { Howl } = require('howler');
@@ -230,20 +233,56 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
funcs.push({ description: "Rotate", event: this.rotate, icon: "expand-arrows-alt" });
let modes: ContextMenuProps[] = [];
- let dataDoc = Doc.GetProto(this.props.Document);
- modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(dataDoc), icon: "tag" });
- modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(dataDoc), icon: "camera" });
+ modes.push({ description: "Generate Tags", event: this.generateMetadata, icon: "tag" });
+ modes.push({ description: "Find Faces", event: this.extractFaces, icon: "camera" });
ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs, icon: "asterisk" });
ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes, icon: "eye" });
}
}
+ extractFaces = () => {
+ let converter = (results: any) => {
+ let faceDocs = new List<Doc>();
+ results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!));
+ return faceDocs;
+ };
+ CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter);
+ }
+
+ generateMetadata = (threshold: Confidence = Confidence.Excellent) => {
+ let converter = (results: any) => {
+ let tagDoc = new Doc;
+ let tagsList = new List();
+ results.tags.map((tag: Tag) => {
+ tagsList.push(tag.name);
+ let sanitized = tag.name.replace(" ", "_");
+ let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`;
+ let computed = CompileScript(script, { params: { this: "Doc" } });
+ computed.compiled && (tagDoc[sanitized] = new ComputedField(computed));
+ });
+ this.extensionDoc.generatedTags = tagsList;
+ tagDoc.title = "Generated Tags Doc";
+ tagDoc.confidence = threshold;
+ return tagDoc;
+ };
+ CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter);
+ }
+
@action
onDotDown(index: number) {
this.Document.curPage = index;
}
+ @computed get fieldExtensionDoc() {
+ return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true");
+ }
+
+ @computed private get url() {
+ let data = Cast(Doc.GetProto(this.props.Document).data, ImageField);
+ return data ? data.url.href : undefined;
+ }
+
dots(paths: string[]) {
let nativeWidth = FieldValue(this.Document.nativeWidth, 1);
let dist = Math.min(nativeWidth / paths.length, 40);
@@ -398,7 +437,7 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" />
</div>
{/* {this.lightbox(paths)} */}
- <FaceRectangles document={this.props.Document} color={"#0000FF"} backgroundColor={"#0000FF"} />
+ <FaceRectangles document={this.extensionDoc} color={"#0000FF"} backgroundColor={"#0000FF"} />
</div>);
}
} \ No newline at end of file
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index 34cb47b20..1f8636826 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -8,7 +8,6 @@ import { Cast, FieldValue, NumCast } from "../../../new_fields/Types";
import { VideoField } from "../../../new_fields/URLField";
import { RouteStore } from "../../../server/RouteStore";
import { Utils } from "../../../Utils";
-import { DocServer } from "../../DocServer";
import { Docs, DocUtils } from "../../documents/Documents";
import { ContextMenu } from "../ContextMenu";
import { ContextMenuProps } from "../ContextMenuItem";
@@ -21,6 +20,9 @@ import { pageSchema } from "./ImageBox";
import "./VideoBox.scss";
import { library } from "@fortawesome/fontawesome-svg-core";
import { faVideo } from "@fortawesome/free-solid-svg-icons";
+import { CompileScript } from "../../util/Scripting";
+import { Doc } from "../../../new_fields/Doc";
+import { ScriptField } from "../../../new_fields/ScriptField";
type VideoDocument = makeInterface<[typeof positionSchema, typeof pageSchema]>;
const VideoDocument = makeInterface(positionSchema, pageSchema);
@@ -87,6 +89,56 @@ export class VideoBox extends DocComponent<FieldViewProps, VideoDocument>(VideoD
this._youtubePlayer && this.props.addDocTab(this.props.Document, this.props.DataDoc, "inTab");
}
+ @action public Snapshot() {
+ let width = NumCast(this.props.Document.width);
+ let height = NumCast(this.props.Document.height);
+ var canvas = document.createElement('canvas');
+ canvas.width = 640;
+ canvas.height = 640 * NumCast(this.props.Document.nativeHeight) / NumCast(this.props.Document.nativeWidth);
+ var ctx = canvas.getContext('2d');//draw image to canvas. scale to target dimensions
+ if (ctx) {
+ ctx.rect(0, 0, canvas.width, canvas.height);
+ ctx.fillStyle = "blue";
+ ctx.fill();
+ this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height);
+ }
+
+ if (!this._videoRef) { // can't find a way to take snapshots of videos
+ let b = Docs.Create.ButtonDocument({
+ x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y),
+ width: 150, height: 50, title: NumCast(this.props.Document.curPage).toString()
+ });
+ const script = CompileScript(`(self as any).curPage = ${NumCast(this.props.Document.curPage)}`, {
+ params: { this: Doc.name },
+ capturedVariables: { self: this.props.Document },
+ typecheck: false,
+ editable: true,
+ });
+ if (script.compiled) {
+ b.onClick = new ScriptField(script);
+ this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(b, false);
+ } else {
+ console.log(script.errors.map(error => error.messageText).join("\n"));
+ }
+ } else {
+ //convert to desired file format
+ var dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png'
+ // if you want to preview the captured image,
+ let filename = encodeURIComponent("snapshot" + this.props.Document.title + "_" + this.props.Document.curPage).replace(/\./g, "");
+ VideoBox.convertDataUri(dataUrl, filename).then(returnedFilename => {
+ if (returnedFilename) {
+ let url = Utils.prepend(returnedFilename);
+ let imageSummary = Docs.Create.ImageDocument(url, {
+ x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y),
+ width: 150, height: height / width * 150, title: "--snapshot" + NumCast(this.props.Document.curPage) + " image-"
+ });
+ this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(imageSummary, false);
+ DocUtils.MakeLink(imageSummary, this.props.Document);
+ }
+ });
+ }
+ }
+
@action
updateTimecode = () => {
this.player && (this.props.Document.curPage = this.player.currentTime);
@@ -150,39 +202,7 @@ export class VideoBox extends DocComponent<FieldViewProps, VideoDocument>(VideoD
let subitems: ContextMenuProps[] = [];
subitems.push({ description: "Copy path", event: () => { Utils.CopyText(url); }, icon: "expand-arrows-alt" });
subitems.push({ description: "Toggle Show Controls", event: action(() => VideoBox._showControls = !VideoBox._showControls), icon: "expand-arrows-alt" });
- let width = NumCast(this.props.Document.width);
- let height = NumCast(this.props.Document.height);
- subitems.push({
- description: "Take Snapshot", event: async () => {
- var canvas = document.createElement('canvas');
- canvas.width = 640;
- canvas.height = 640 * NumCast(this.props.Document.nativeHeight) / NumCast(this.props.Document.nativeWidth);
- var ctx = canvas.getContext('2d');//draw image to canvas. scale to target dimensions
- if (ctx) {
- ctx.rect(0, 0, canvas.width, canvas.height);
- ctx.fillStyle = "blue";
- ctx.fill();
- this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height);
- }
-
- //convert to desired file format
- var dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png'
- // if you want to preview the captured image,
- let filename = encodeURIComponent("snapshot" + this.props.Document.title + "_" + this.props.Document.curPage).replace(/\./g, "");
- VideoBox.convertDataUri(dataUrl, filename).then(returnedFilename => {
- if (returnedFilename) {
- let url = Utils.prepend(returnedFilename);
- let imageSummary = Docs.Create.ImageDocument(url, {
- x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y),
- width: 150, height: height / width * 150, title: "--snapshot" + NumCast(this.props.Document.curPage) + " image-"
- });
- this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(imageSummary, false);
- DocUtils.MakeLink(imageSummary, this.props.Document);
- }
- });
- },
- icon: "expand-arrows-alt"
- });
+ subitems.push({ description: "Take Snapshot", event: () => this.Snapshot(), icon: "expand-arrows-alt" });
ContextMenu.Instance.addItem({ description: "Video Funcs...", subitems: subitems, icon: "video" });
}
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-02-26 20:33:34 +0000