aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--package-lock.json82
-rw-r--r--package.json2
-rw-r--r--src/client/DocServer.ts4
-rw-r--r--src/client/views/GestureOverlay.tsx170
-rw-r--r--src/client/views/OCRUtils.ts7
-rw-r--r--src/server/Message.ts2
-rw-r--r--src/server/Websocket/Websocket.ts22
7 files changed, 229 insertions, 60 deletions
diff --git a/package-lock.json b/package-lock.json
index 6198a568a..825890038 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -647,7 +647,7 @@
},
"@types/passport": {
"version": "1.0.2",
- "resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.0.tgz",
+ "resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.2.tgz",
"integrity": "sha512-Pf39AYKf8q+YoONym3150cEwfUD66dtwHJWvbeOzKxnA0GZZ/vAXhNWv9vMhKyRQBQZiQyWQnhYBEBlKW6G8wg==",
"requires": {
"@types/express": "*"
@@ -5391,7 +5391,8 @@
},
"ansi-regex": {
"version": "2.1.1",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"aproba": {
"version": "1.2.0",
@@ -5409,11 +5410,13 @@
},
"balanced-match": {
"version": "1.0.0",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"brace-expansion": {
"version": "1.1.11",
"bundled": true,
+ "optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -5426,15 +5429,18 @@
},
"code-point-at": {
"version": "1.1.0",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"concat-map": {
"version": "0.0.1",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"console-control-strings": {
"version": "1.1.0",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"core-util-is": {
"version": "1.0.2",
@@ -5537,7 +5543,8 @@
},
"inherits": {
"version": "2.0.4",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"ini": {
"version": "1.3.5",
@@ -5547,6 +5554,7 @@
"is-fullwidth-code-point": {
"version": "1.0.0",
"bundled": true,
+ "optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@@ -5559,17 +5567,20 @@
"minimatch": {
"version": "3.0.4",
"bundled": true,
+ "optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
},
"minimist": {
"version": "0.0.8",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"minipass": {
"version": "2.9.0",
"bundled": true,
+ "optional": true,
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
@@ -5586,6 +5597,7 @@
"mkdirp": {
"version": "0.5.1",
"bundled": true,
+ "optional": true,
"requires": {
"minimist": "0.0.8"
}
@@ -5666,7 +5678,8 @@
},
"number-is-nan": {
"version": "1.0.1",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"object-assign": {
"version": "4.1.1",
@@ -5676,6 +5689,7 @@
"once": {
"version": "1.4.0",
"bundled": true,
+ "optional": true,
"requires": {
"wrappy": "1"
}
@@ -5751,7 +5765,8 @@
},
"safe-buffer": {
"version": "5.1.2",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"safer-buffer": {
"version": "2.1.2",
@@ -5781,6 +5796,7 @@
"string-width": {
"version": "1.0.2",
"bundled": true,
+ "optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@@ -5798,6 +5814,7 @@
"strip-ansi": {
"version": "3.0.1",
"bundled": true,
+ "optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@@ -5836,11 +5853,13 @@
},
"wrappy": {
"version": "1.0.2",
- "bundled": true
+ "bundled": true,
+ "optional": true
},
"yallist": {
"version": "3.1.1",
- "bundled": true
+ "bundled": true,
+ "optional": true
}
}
},
@@ -8739,6 +8758,39 @@
"resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.9.1.tgz",
"integrity": "sha512-7/Xs9gkuYF0WBimz5OrSc6UVKLDTxvBG2yLGtEK8PSx94d86o/6iQLvIe/140ATz35JDqHKWIxh3GcA3u5hB0w=="
},
+ "node-tesseract": {
+ "version": "0.2.7",
+ "resolved": "https://registry.npmjs.org/node-tesseract/-/node-tesseract-0.2.7.tgz",
+ "integrity": "sha1-yPAvuDUaQnByc1d4wFGYI/JgG4Q=",
+ "requires": {
+ "glob": "^5.0.10",
+ "node-uuid": "^1.4.1"
+ },
+ "dependencies": {
+ "glob": {
+ "version": "5.0.15",
+ "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz",
+ "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=",
+ "requires": {
+ "inflight": "^1.0.4",
+ "inherits": "2",
+ "minimatch": "2 || 3",
+ "once": "^1.3.0",
+ "path-is-absolute": "^1.0.0"
+ }
+ },
+ "node-uuid": {
+ "version": "1.4.8",
+ "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz",
+ "integrity": "sha1-sEDrCSOWivq/jTL7HxfxFn/auQc="
+ }
+ }
+ },
+ "node-tesseract-ocr": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-1.0.0.tgz",
+ "integrity": "sha512-1u6KNqrt0jGK8Fdrm8JfQD4bctEScpnDtsEV42dmZ54zQRov+OUZZb4a7uy4V+OKENn7fKpgB5WIza5ernGHzA=="
+ },
"nodemailer": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-5.1.1.tgz",
@@ -13785,7 +13837,7 @@
},
"readable-stream": {
"version": "2.3.6",
- "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+ "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"requires": {
"core-util-is": "~1.0.0",
@@ -15701,7 +15753,7 @@
},
"strip-ansi": {
"version": "3.0.1",
- "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+ "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"requires": {
"ansi-regex": "^2.0.0"
@@ -17928,7 +17980,7 @@
},
"wrap-ansi": {
"version": "2.1.0",
- "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
+ "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
"integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=",
"requires": {
"string-width": "^1.0.1",
diff --git a/package.json b/package.json
index dba0d84aa..ed3458378 100644
--- a/package.json
+++ b/package.json
@@ -173,6 +173,8 @@
"mongoose": "^5.8.9",
"node-sass": "^4.13.1",
"node-stream-zip": "^1.9.1",
+ "node-tesseract": "^0.2.7",
+ "node-tesseract-ocr": "^1.0.0",
"nodemailer": "^5.1.1",
"nodemon": "^1.19.4",
"normalize.css": "^8.0.1",
diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts
index d793b56af..5fcd2547e 100644
--- a/src/client/DocServer.ts
+++ b/src/client/DocServer.ts
@@ -218,6 +218,10 @@ export namespace DocServer {
return apiKey;
}
+ export async function analyzeImage(image: string, callback: (result: any) => void) {
+ Utils.EmitCallback(_socket, MessageStore.AnalyzeInk, image, callback);
+ }
+
export function getYoutubeVideos(videoTitle: string, callBack: (videos: any[]) => void) {
Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.SearchVideo, userInput: videoTitle }, callBack);
}
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx
index e25647e69..c0e45d36f 100644
--- a/src/client/views/GestureOverlay.tsx
+++ b/src/client/views/GestureOverlay.tsx
@@ -20,6 +20,8 @@ import { DocumentView } from "./nodes/DocumentView";
import { Transform } from "../util/Transform";
import { DocumentContentsView } from "./nodes/DocumentContentsView";
import { CognitiveServices } from "../cognitive_services/CognitiveServices";
+import { DocServer } from "../DocServer";
+import htmlToImage from "html-to-image";
@observer
export default class GestureOverlay extends Touchable {
@@ -218,13 +220,16 @@ export default class GestureOverlay extends Touchable {
console.log("not hand");
}
this.pointerIdentifier = pointer?.identifier;
- runInAction(() => this._pointerY = pointer?.clientY);
- if (thumb.identifier === this.thumbIdentifier) {
- this._thumbX = thumb.clientX;
- this._thumbY = thumb.clientY;
- this._hands.set(thumb.identifier, fingers);
- return;
- }
+ runInAction(() => {
+ this._pointerY = pointer?.clientY;
+ if (thumb.identifier === this.thumbIdentifier) {
+ this._thumbX = thumb.clientX;
+ this._thumbY = thumb.clientY;
+ this._hands.set(thumb.identifier, fingers);
+ return;
+ }
+ });
+
this.thumbIdentifier = thumb?.identifier;
this._hands.set(thumb.identifier, fingers);
const others = fingers.filter(f => f !== thumb);
@@ -295,6 +300,10 @@ export default class GestureOverlay extends Touchable {
this._palette = undefined;
this.thumbIdentifier = undefined;
this._thumbDoc = undefined;
+ this._strokes.forEach(s => {
+ this.dispatchGesture(GestureUtils.Gestures.Stroke, s);
+ });
+ this._strokes = [];
document.removeEventListener("touchend", this.handleHandUp);
}
}
@@ -303,6 +312,11 @@ export default class GestureOverlay extends Touchable {
onPointerDown = (e: React.PointerEvent) => {
if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || (InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) {
this._points.push({ X: e.clientX, Y: e.clientY });
+ const canvas = this._canvas.current;
+ const ctx = canvas?.getContext("2d");
+ if (canvas && ctx) {
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ }
e.stopPropagation();
e.preventDefault();
@@ -354,22 +368,79 @@ export default class GestureOverlay extends Touchable {
return actionPerformed;
}
+ draw = (points: InkData) => {
+ let ctx;
+ if (this._canvas.current && this._canvas.current.getContext("2d") && (ctx = this._canvas.current.getContext("2d"))) {
+ ctx.strokeStyle = this.Color;
+ ctx.lineWidth = this.Width;
+ ctx.moveTo(points[0].X, points[0].Y);
+ for (let i = 1; i < points.length; i++) {
+ ctx.lineTo(points[i].X, points[i].Y);
+ }
+ ctx.stroke();
+ }
+ }
+
@action
- onPointerUp = (e: PointerEvent) => {
+ onPointerUp = async (e: PointerEvent) => {
if (this._points.length > 1) {
const B = this.svgBounds;
const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top }));
- const xInGlass = points[0].X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && points[0].X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + this.height;
- const yInGlass = points[0].Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - this.height && points[0].Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER);
+ const initialPoint = this._points[0];
+ const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + this.height;
+ const yInGlass = initialPoint.Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - this.height && initialPoint.Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER);
if (this.Tool !== ToolglassTools.None && xInGlass && yInGlass) {
switch (this.Tool) {
case ToolglassTools.InkToText:
- this._strokes.push(this._points);
+ // this.draw(points);
+ // const dataUrl = this._canvas.current?.toDataURL("image/jpeg");
+ // if (dataUrl) {
+ // DocServer.analyzeImage(dataUrl, (result: any) => {
+ // console.log("something");
+ // console.log(result);
+ // });
+ // }
+
+ // htmlToImage.toCanvas(this._svg.current!).then((blob) => {
+ // if (blob) {
+ // console.log("blobbed");
+ // blob.toDataURL("image/jpeg")
+ // const t = blob.getContext("2d")?.getImageData(0, 0, blob.width, blob.height);
+ // if (t) {
+ // DocServer.analyzeImage(t.data, (text: any) => {
+ // console.log(text);
+ // });
+ // }
+ // }
+ // else {
+ // console.log("no blob")
+ // }
+ // runInAction(() => {
+ // this._strokes.push(this._points);
+ // this._points = [];
+ // });
+ // }).catch(e => console.log(e));
+ runInAction(() => {
+ this._strokes.push(this._points);
+ this._points = [];
+ });
+ const results = await CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes);
+ // console.log(results);
+ const wordResults = results.filter(r => r.category === "inkWord");
+ console.log(wordResults);
+ const possibilities = [wordResults[0]?.recognizedText];
+ possibilities.push(...wordResults[0].alternates?.map(a => a.recognizedString));
+ console.log(possibilities);
+
+ // return;
+ break;
+ case ToolglassTools.IgnoreGesture:
+ this.dispatchGesture(GestureUtils.Gestures.Stroke);
this._points = [];
- console.log(CognitiveServices.Inking.Appliers.InterpretStrokes([this._points]));
- return;
+ this._canvas.current?.getContext("2d")?.restore();
+ break;
}
}
else {
@@ -378,16 +449,7 @@ export default class GestureOverlay extends Touchable {
if (result && result.Score > 0.7) {
switch (result.Name) {
case GestureUtils.Gestures.Box:
- const target = document.elementFromPoint(this._points[0].X, this._points[0].Y);
- target?.dispatchEvent(new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
- {
- bubbles: true,
- detail: {
- points: this._points,
- gesture: GestureUtils.Gestures.Box,
- bounds: B
- }
- }));
+ this.dispatchGesture(GestureUtils.Gestures.Box);
actionPerformed = true;
break;
case GestureUtils.Gestures.Line:
@@ -399,24 +461,14 @@ export default class GestureOverlay extends Touchable {
}
if (actionPerformed) {
this._points = [];
+ this._canvas.current?.getContext("2d")?.restore();
}
}
if (!actionPerformed) {
- const target = document.elementFromPoint(this._points[0].X, this._points[0].Y);
- target?.dispatchEvent(
- new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
- {
- bubbles: true,
- detail: {
- points: this._points,
- gesture: GestureUtils.Gestures.Stroke,
- bounds: B
- }
- }
- )
- );
+ this.dispatchGesture(GestureUtils.Gestures.Stroke);
this._points = [];
+ this._canvas.current?.getContext("2d")?.restore();
}
}
}
@@ -424,11 +476,25 @@ export default class GestureOverlay extends Touchable {
document.removeEventListener("pointerup", this.onPointerUp);
}
- @computed get svgBounds() {
- const sxs = this._strokes.reduce((acc, curr) => acc.concat(...curr.map(p => p.X)), new Array<number>());
- const xs = this._points.map(p => p.X).concat(sxs);
- const sys = this._strokes.reduce((acc, curr) => acc.concat(...curr.map(p => p.Y)), new Array<number>());
- const ys = this._points.map(p => p.Y).concat(sys);
+ dispatchGesture = (gesture: GestureUtils.Gestures, stroke?: InkData) => {
+ const target = document.elementFromPoint((stroke ?? this._points)[0].X, (stroke ?? this._points)[0].Y);
+ target?.dispatchEvent(
+ new CustomEvent<GestureUtils.GestureEvent>("dashOnGesture",
+ {
+ bubbles: true,
+ detail: {
+ points: stroke ?? this._points,
+ gesture: gesture,
+ bounds: this.svgBounds
+ }
+ }
+ )
+ );
+ }
+
+ getBounds = (stroke: InkData) => {
+ const xs = stroke.map(p => p.X);
+ const ys = stroke.map(p => p.Y);
const right = Math.max(...xs);
const left = Math.min(...xs);
const bottom = Math.max(...ys);
@@ -436,6 +502,13 @@ export default class GestureOverlay extends Touchable {
return { right: right, left: left, bottom: bottom, top: top, width: right - left, height: bottom - top };
}
+ @computed get svgBounds() {
+ return this.getBounds(this._points);
+ }
+
+ private _canvas = React.createRef<HTMLCanvasElement>();
+ private _svg = React.createRef<HTMLDivElement>();
+
@computed get currentStrokes() {
if (this._points.length <= 1 && this._strokes.length <= 1) {
return (null);
@@ -444,10 +517,15 @@ export default class GestureOverlay extends Touchable {
const B = this.svgBounds;
return (
- <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, pointerEvents: "none", position: "absolute", zIndex: 30000 }}>
- {this._strokes.map(l => InteractionUtils.CreatePolyline(l, B.left, B.top, this.Color, this.Width))}
- {InteractionUtils.CreatePolyline(this._points, B.left, B.top, this.Color, this.Width)}
- </svg>
+ <div ref={this._svg}>
+ <svg width={B.width} height={B.height} style={{ transform: `translate(${B.left}px, ${B.top}px)`, pointerEvents: "none", position: "absolute", zIndex: 30000 }}>
+ {this._strokes.map(l => {
+ const b = this.getBounds(l);
+ InteractionUtils.CreatePolyline(l, b.left, b.top, this.Color, this.Width);
+ })}
+ {InteractionUtils.CreatePolyline(this._points, B.left, B.top, this.Color, this.Width)}
+ </svg>
+ </div>
);
}
@@ -514,6 +592,7 @@ export default class GestureOverlay extends Touchable {
touchAction: "none",
display: this.showBounds ? "unset" : "none",
}}>
+ <canvas ref={this._canvas} width={this.height} height={this.height} style={{ pointerEvents: "none", position: "absolute" }}></canvas>
</div>
</div >);
}
@@ -523,6 +602,7 @@ export default class GestureOverlay extends Touchable {
export enum ToolglassTools {
InkToText = "inktotext",
+ IgnoreGesture = "ignoregesture",
None = "none",
}
diff --git a/src/client/views/OCRUtils.ts b/src/client/views/OCRUtils.ts
new file mode 100644
index 000000000..282ec770e
--- /dev/null
+++ b/src/client/views/OCRUtils.ts
@@ -0,0 +1,7 @@
+// import tesseract from "node-tesseract-ocr";
+// const tesseract = require("node-tesseract");
+
+
+export namespace OCRUtils {
+
+}
diff --git a/src/server/Message.ts b/src/server/Message.ts
index 621abfd1e..22d2fa8a8 100644
--- a/src/server/Message.ts
+++ b/src/server/Message.ts
@@ -1,4 +1,5 @@
import { Utils } from "../Utils";
+import { Image } from "canvas";
export class Message<T> {
private _name: string;
@@ -59,4 +60,5 @@ export namespace MessageStore {
export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query");
export const DeleteField = new Message<string>("Delete field");
export const DeleteFields = new Message<string[]>("Delete fields");
+ export const AnalyzeInk = new Message<string>("Analyze Ink");
}
diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts
index a2fdc7c89..f485e1dcd 100644
--- a/src/server/Websocket/Websocket.ts
+++ b/src/server/Websocket/Websocket.ts
@@ -10,6 +10,16 @@ import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader";
import { logPort } from "../ActionUtilities";
import { timeMap } from "../ApiManagers/UserManager";
import { green } from "colors";
+import { Image } from "canvas";
+import { write, createWriteStream } from "fs";
+import { serverPathToFile, Directory } from "../ApiManagers/UploadManager";
+const tesseract = require("node-tesseract-ocr");
+const config = {
+ lang: "eng",
+ oem: 1,
+ psm: 8
+};
+const imageDataUri = require('image-data-uri');
export namespace WebSocket {
@@ -51,6 +61,7 @@ export namespace WebSocket {
Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField);
Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery);
+ Utils.AddServerHandlerCallback(socket, MessageStore.AnalyzeInk, RecognizeImage);
Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff));
Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id));
Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids));
@@ -68,6 +79,17 @@ export namespace WebSocket {
logPort("websocket", socketPort);
}
+ async function RecognizeImage([query, callback]: [string, (result: any) => any]) {
+ const path = serverPathToFile(Directory.images, "handwriting.jpg");
+ imageDataUri.outputFile(query, path).then((savedName: string) => {
+ console.log("saved " + savedName);
+ const remadePath = path.split("\\").join("\\\\");
+ tesseract.recognize(remadePath, config)
+ .then(callback)
+ .catch(console.log);
+ });
+ }
+
function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) {
const { ProjectCredentials } = GoogleCredentialsLoader;
switch (query.type) {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 19:21:39 +0000