From a26c670b49a8631779869baf493135a59b92f523 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 24 Feb 2025 16:09:09 -0500 Subject: fix & feat: changed cohere embeddings for OpenAI's new embedding model and also improved security by moving api keys to .env --- src/server/ApiManagers/AssistantManager.ts | 4 ++-- src/server/chunker/pdf_chunker.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index c41f697db..4719541b9 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -538,7 +538,7 @@ export default class AssistantManager extends ApiManager { // Spawn the Python process and track its progress/output // eslint-disable-next-line no-use-before-define - spawnPythonProcess(jobId, file_name, public_path); + spawnPythonProcess(jobId, public_path); // Send the job ID back to the client for tracking res.send({ jobId }); @@ -695,7 +695,7 @@ export default class AssistantManager extends ApiManager { * @param file_name The name of the file to process. * @param file_path The filepath of the file to process. */ -function spawnPythonProcess(jobId: string, file_name: string, file_path: string) { +function spawnPythonProcess(jobId: string, file_path: string) { const venvPath = path.join(__dirname, '../chunker/venv'); const requirementsPath = path.join(__dirname, '../chunker/requirements.txt'); const pythonScriptPath = path.join(__dirname, '../chunker/pdf_chunker.py'); diff --git a/src/server/chunker/pdf_chunker.py b/src/server/chunker/pdf_chunker.py index a9dbcbb0c..697550f2e 100644 --- a/src/server/chunker/pdf_chunker.py +++ b/src/server/chunker/pdf_chunker.py @@ -21,7 +21,7 @@ import json import os import uuid # For generating unique IDs from enum import Enum # Enums for types like document type and purpose -import cohere # Embedding client +import openai import numpy as np from PyPDF2 import PdfReader # PDF text extraction from openai import OpenAI # OpenAI client for text completion @@ -35,8 +35,8 @@ warnings.filterwarnings('ignore', message="torch.load") dotenv.load_dotenv() # Load environment variables # Fix for newer versions of PIL -if parse(PIL.__version__) >= parse('10.0.0'): - Image.LINEAR = Image.BILINEAR +# if parse(PIL.__version__) >= parse('10.0.0'): +# Image.LINEAR = Image.BILINEAR # Global dictionary to track progress of document processing jobs current_progress = {} @@ -727,19 +727,19 @@ class Document: """ Embed the text chunks using the Cohere API. """ - co = cohere.Client(os.getenv("COHERE_API_KEY")) # Initialize Cohere client with API key + openai = OpenAI() # Initialize Cohere client with API key batch_size = 90 # Batch size for embedding chunks_len = len(self.chunks) # Total number of chunks to embed for i in tqdm(range(0, chunks_len, batch_size), desc="Embedding Chunks"): batch = self.chunks[i: min(i + batch_size, chunks_len)] # Get batch of chunks texts = [chunk['metadata']['text'] for chunk in batch] # Extract text from each chunk - chunk_embs_batch = co.embed( - texts=texts, - model="embed-english-v3.0", # Use Cohere's embedding model - input_type="search_document" # Specify input type + chunk_embs_batch = openai.embeddings.create( + model="text-embedding-3-large", + input=texts, + encoding_format="float" ) - for j, emb in enumerate(chunk_embs_batch.embeddings): - self.chunks[i + j]['values'] = emb # Store the embeddings in the corresponding chunks + for j, data_val in enumerate(chunk_embs_batch.data): + self.chunks[i + j]['values'] = data_val.embedding # Store the embeddings in the corresponding chunks def _generate_summary(self) -> str: """ -- cgit v1.2.3-70-g09d2 From 91b67053456581722457bdc95da79a023f27b412 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 25 Feb 2025 22:25:39 -0500 Subject: updated packages and fixed some cimplier warnings. --- package-lock.json | 35 +++++++--------------- package.json | 4 +-- .../components/src/components/Slider/Slider.tsx | 10 +++---- src/client/util/request-image-size.ts | 9 +++--- src/server/DashUploadUtils.ts | 8 ++--- src/typings/index.d.ts | 1 + 6 files changed, 26 insertions(+), 41 deletions(-) (limited to 'src/server') diff --git a/package-lock.json b/package-lock.json index bc1baea46..ccd1ad886 100644 --- a/package-lock.json +++ b/package-lock.json @@ -146,7 +146,7 @@ "i": "^0.3.7", "iink-ts": "^2.0.1", "image-data-uri": "^2.0.1", - "image-size": "^1.0.2", + "image-size": "^2.0.0", "image-size-stream": "^1.1.0", "install": "^0.13.0", "is-plain-obj": "^4.1.0", @@ -293,7 +293,7 @@ "@types/cookie-session": "^2.0.48", "@types/d3": "^7.4.3", "@types/dom-mediacapture-record": "^1.0.19", - "@types/dompurify": "^3.2.0", + "@types/dompurify": "^3.0.5", "@types/exif": "^0.6.5", "@types/express": "^5.0.0", "@types/express-session": "^1.17.10", @@ -14551,14 +14551,13 @@ "license": "MIT" }, "node_modules/@types/dompurify": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.2.0.tgz", - "integrity": "sha512-Fgg31wv9QbLDA0SpTOXO3MaxySc4DKGLi8sna4/Utjo4r3ZRPdCt4UQee8BWr+Q5z21yifghREPJGYaEOEIACg==", - "deprecated": "This is a stub types definition. dompurify provides its own type definitions, so you do not need this installed.", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz", + "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==", "dev": true, "license": "MIT", "dependencies": { - "dompurify": "*" + "@types/trusted-types": "*" } }, "node_modules/@types/eslint": { @@ -15388,8 +15387,8 @@ "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", - "license": "MIT", - "optional": true + "devOptional": true, + "license": "MIT" }, "node_modules/@types/unist": { "version": "3.0.3", @@ -24004,13 +24003,10 @@ "license": "MIT" }, "node_modules/image-size": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.2.0.tgz", - "integrity": "sha512-4S8fwbO6w3GeCVN6OPtA9I5IGKkcDMPcKndtUlpJuCwu7JLjtj7JZpwqLuyY2nrmQT3AWsCJLSKPsc2mPBSl3w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-2.0.0.tgz", + "integrity": "sha512-HP07n1SpdIXGUL4VotUIOQz66MQOq8g7VN+Yj02YTVowqZScQ5i/JYU0+lkNr2pwt5j4hOpk94/UBV1ZCbS2fA==", "license": "MIT", - "dependencies": { - "queue": "6.0.2" - }, "bin": { "image-size": "bin/image-size.js" }, @@ -32972,15 +32968,6 @@ "node": ">=0.4.x" } }, - "node_modules/queue": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz", - "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==", - "license": "MIT", - "dependencies": { - "inherits": "~2.0.3" - } - }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", diff --git a/package.json b/package.json index 8cd6945ff..6eb2a5c98 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "@types/cookie-session": "^2.0.48", "@types/d3": "^7.4.3", "@types/dom-mediacapture-record": "^1.0.19", - "@types/dompurify": "^3.2.0", + "@types/dompurify": "^3.0.5", "@types/exif": "^0.6.5", "@types/express": "^5.0.0", "@types/express-session": "^1.17.10", @@ -230,7 +230,7 @@ "i": "^0.3.7", "iink-ts": "^2.0.1", "image-data-uri": "^2.0.1", - "image-size": "^1.0.2", + "image-size": "^2.0.0", "image-size-stream": "^1.1.0", "install": "^0.13.0", "is-plain-obj": "^4.1.0", diff --git a/packages/components/src/components/Slider/Slider.tsx b/packages/components/src/components/Slider/Slider.tsx index f6f53799c..5af945383 100644 --- a/packages/components/src/components/Slider/Slider.tsx +++ b/packages/components/src/components/Slider/Slider.tsx @@ -122,11 +122,11 @@ export const Slider = (props: ISliderProps) => { setEndNumber?.((lastEndVal = Math.max(number + (minDiff ?? 0), val))); setEndNumberLoc((lastEndVal = Math.max(number + (minDiff ?? 0), val))); }; - const Slider: (JSX.Element | null)[] = [!multithumb ? null : valSlider('end', endNumberLoc, onendchange, () => setFinalEndNumber?.(lastEndVal)), valSlider('start', valLoc, onchange, () => setFinalNumber?.(lastVal))]; + const ValSlider: (JSX.Element | null)[] = [!multithumb ? null : valSlider('end', endNumberLoc, onendchange, () => setFinalEndNumber?.(lastEndVal)), valSlider('start', valLoc, onchange, () => setFinalNumber?.(lastVal))]; - const slider: JSX.Element = ( + const slider = (
{ lastVal = valLoc; lastEndVal = endNumberLoc; @@ -143,7 +143,7 @@ export const Slider = (props: ISliderProps) => { }} style={{ height: getHeight(+(height || 0), size) }} onPointerDown={onPointerDown}> - {Slider} + {ValSlider}
{ return formLabel ? (
-
+
{formLabel}
{slider} diff --git a/src/client/util/request-image-size.ts b/src/client/util/request-image-size.ts index c619192ed..32ab23618 100644 --- a/src/client/util/request-image-size.ts +++ b/src/client/util/request-image-size.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ /** * request-image-size: Detect image dimensions via request. * Licensed under the MIT license. @@ -11,12 +10,11 @@ */ // const imageSize = require('image-size'); -const HttpError = require('standard-http-error'); +import * as HttpError from 'standard-http-error'; import * as request from 'request'; import { imageSize } from 'image-size'; import { ISizeCalculationResult } from 'image-size/dist/types/interface'; - -module.exports = function requestImageSize(url: string) { +export function requestImageSize(url: string): Promise { if (!url) { return Promise.reject(new Error('You should provide an URI string or a "request" options object.')); } @@ -60,4 +58,5 @@ module.exports = function requestImageSize(url: string) { req.on('error', reject); }); -}; +} +export default requestImageSize; diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 2177c5d97..a2747257a 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -23,6 +23,7 @@ import { AcceptableMedia, Upload } from './SharedMediaTypes'; import { Directory, clientPathToFile, filesDirectory, pathToDirectory, publicDirectory, serverPathToFile } from './SocketData'; import { resolvedServerUrl } from './server_Initialization'; import { Worker, isMainThread, parentPort } from 'worker_threads'; +import requestImageSize from '../client/util/request-image-size'; // Create an array to store worker threads enum workertasks { @@ -62,9 +63,6 @@ if (isMainThread) { } } -// eslint-disable-next-line @typescript-eslint/no-require-imports -const requestImageSize = require('../client/util/request-image-size'); - export enum SizeSuffix { Small = '_s', Medium = '_m', @@ -349,14 +347,14 @@ export namespace DashUploadUtils { imgReadStream.push(null); await Promise.all( sizes.map(({ suffix }) => - new Promise(res => + new Promise(res => imgReadStream.pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) ) )); // prettier-ignore } else { await Promise.all( sizes.map(({ suffix }) => - new Promise(res => + new Promise(res => request.get(imgSourcePath).pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) ) )); // prettier-ignore diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index bee79a38d..dbfabed51 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -13,6 +13,7 @@ declare module 'iink-js'; declare module 'pdfjs-dist/web/pdf_viewer'; declare module 'react-jsx-parser'; declare module 'type_decls.d'; +declare module 'standard-http-error'; declare module '@react-pdf/renderer' { import * as React from 'react'; -- cgit v1.2.3-70-g09d2 From 4989f9489e126b7cea4946b5519987804ba46f96 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 26 Feb 2025 11:29:47 -0500 Subject: removed query-string from project --- package-lock.json | 51 -------------- package.json | 1 - src/client/util/History.ts | 23 +++---- src/server/apis/google/GoogleApiServerUtils.ts | 93 +++++++++++++------------- 4 files changed, 56 insertions(+), 112 deletions(-) (limited to 'src/server') diff --git a/package-lock.json b/package-lock.json index 68611f50a..79e81128b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -201,7 +201,6 @@ "prosemirror-view": "^1.32.5", "pug": "^3.0.2", "puppeteer": "^24.1.1", - "query-string": "^9.1.1", "querystring-es3": "^0.2.1", "raw-loader": "^4.0.2", "rc-switch": "^4.1.0", @@ -19674,15 +19673,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/decode-uri-component": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.4.1.tgz", - "integrity": "sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==", - "license": "MIT", - "engines": { - "node": ">=14.16" - } - }, "node_modules/decompress-response": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", @@ -21777,18 +21767,6 @@ "node": ">=8" } }, - "node_modules/filter-obj": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-5.1.0.tgz", - "integrity": "sha512-qWeTREPoT7I0bifpPUXtxkZJ1XJzxWtfoWWkdVGqa+eCr3SHW/Ocp89o8vLvbUuQnadybJpjOKu4V+RwO6sGng==", - "license": "MIT", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -32945,23 +32923,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/query-string": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-9.1.1.tgz", - "integrity": "sha512-MWkCOVIcJP9QSKU52Ngow6bsAWAPlPK2MludXvcrS2bGZSl+T1qX9MZvRIkqUIkGLJquMJHWfsT6eRqUpp4aWg==", - "license": "MIT", - "dependencies": { - "decode-uri-component": "^0.4.1", - "filter-obj": "^5.1.0", - "split-on-first": "^3.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/querystring-es3": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", @@ -36166,18 +36127,6 @@ "integrity": "sha512-0kGecIZNIReCSiznK3uheYB8sbstLjCZLiwcQwbmLhgHJj2gz6OnSPkVzJQCMnmEz1BQ4gPK59ylhBoEWOhGNA==", "license": "BDS-3-Clause" }, - "node_modules/split-on-first": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-3.0.0.tgz", - "integrity": "sha512-qxQJTx2ryR0Dw0ITYyekNQWpz6f8dGd7vffGNflQQ3Iqj9NJ6qiZ7ELpZsJ/QBhIVAiDfXdag3+Gp8RvWa62AA==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/split-skip": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/split-skip/-/split-skip-0.0.2.tgz", diff --git a/package.json b/package.json index a090ab9b2..5d036156f 100644 --- a/package.json +++ b/package.json @@ -285,7 +285,6 @@ "prosemirror-view": "^1.32.5", "pug": "^3.0.2", "puppeteer": "^24.1.1", - "query-string": "^9.1.1", "querystring-es3": "^0.2.1", "raw-loader": "^4.0.2", "rc-switch": "^4.1.0", diff --git a/src/client/util/History.ts b/src/client/util/History.ts index cf156014d..9728e3177 100644 --- a/src/client/util/History.ts +++ b/src/client/util/History.ts @@ -1,7 +1,6 @@ /* eslint-disable no-use-before-define */ /* eslint-disable no-empty */ /* eslint-disable no-param-reassign */ -import queryString from 'query-string'; import { Doc } from '../../fields/Doc'; import { OmitKeys, ClientUtils } from '../../ClientUtils'; import { DocServer } from '../DocServer'; @@ -79,7 +78,7 @@ export namespace HistoryUtil { // } // } - const parsers: { [type: string]: (pathname: string[], opts: queryString.ParsedQuery) => ParsedUrl | undefined } = {}; + const parsers: { [type: string]: (pathname: string[], opts: URLSearchParams) => ParsedUrl | undefined } = {}; const stringifiers: { [type: string]: (state: ParsedUrl) => string } = {}; type ParserValue = true | 'none' | 'json' | ((value: string) => string | null | (string | null)[]); @@ -88,7 +87,7 @@ export namespace HistoryUtil { [key: string]: ParserValue; }; - function addParser(type: string, requiredFields: Parser, optionalFields: Parser, customParser?: (pathname: string[], opts: queryString.ParsedQuery, current: ParsedUrl) => ParsedUrl | null | undefined) { + function addParser(type: string, requiredFields: Parser, optionalFields: Parser, customParser?: (pathname: string[], opts: URLSearchParams, current: ParsedUrl) => ParsedUrl | null | undefined) { function parseValue(parser: ParserValue, value: string | (string | null)[] | null | undefined) { if (value === undefined || value === null) { return value; @@ -105,21 +104,21 @@ export namespace HistoryUtil { parsers[type] = (pathname, opts) => { const current: DocUrl & { [key: string]: null | (string | null)[] | string } = { type: 'doc', docId: '' }; for (const required in requiredFields) { - if (!(required in opts)) { + if (!opts.has(required)) { return undefined; } const parser = requiredFields[required]; - const value = parseValue(parser, opts[required]); + const value = parseValue(parser, opts.get(required)); if (value !== null && value !== undefined) { current[required] = value; } } for (const opt in optionalFields) { - if (!(opt in opts)) { + if (!opts.has(opt)) { continue; } const parser = optionalFields[opt]; - const value = parseValue(parser, opts[opt]); + const value = parseValue(parser, opts.get(opt)); if (value !== undefined) { current[opt] = value; } @@ -145,12 +144,12 @@ export namespace HistoryUtil { path = customStringifier(state, path); } const queryObj = OmitKeys(state, keys).extract; - const query: { [key: string]: string | null } = {}; + const query = new URLSearchParams(); Object.keys(queryObj).forEach(key => { - query[key] = queryObj[key] === null ? null : JSON.stringify(queryObj[key]); + query.set(key, queryObj[key] === null ? '' : JSON.stringify(queryObj[key])); }); - const qstr = queryString.stringify(query); - return path + (queryString ? `?${qstr}` : ''); + const qstr = query.toString(); + return path + (qstr ? `?${qstr}` : ''); }; } @@ -167,7 +166,7 @@ export namespace HistoryUtil { export function parseUrl(location: Location | URL): ParsedUrl | undefined { const pathname = location.pathname.substring(1); const { search } = location; - const opts = search.length ? queryString.parse(search, { sort: false }) : {}; + const opts = new URLSearchParams(search); const pathnameSplit = pathname.split('/'); const type = pathnameSplit[0]; diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 21c405bee..7373df473 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,7 +1,7 @@ +/* eslint-disable no-use-before-define */ import { GaxiosResponse } from 'gaxios'; import { Credentials, OAuth2Client, OAuth2ClientOptions } from 'google-auth-library'; import { google } from 'googleapis'; -import * as qs from 'query-string'; import * as request from 'request-promise'; import { Opt } from '../../../fields/Doc'; import { Database } from '../../database'; @@ -21,7 +21,6 @@ const scope = ['documents.readonly', 'documents', 'presentations', 'presentation * This namespace manages server side authentication for Google API queries, either * from the standard v1 APIs or the Google Photos REST API. */ - export namespace GoogleApiServerUtils { /** * As we expand out to more Google APIs that are accessible from @@ -71,29 +70,29 @@ export namespace GoogleApiServerUtils { /** * A briefer format for the response from a 'googleapis' API request */ - export type ApiResponse = Promise; + export type ApiResponse = Promise>; /** * A generic form for a handler that executes some request on the endpoint */ - export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; + export type ApiRouter = (endpoint: Endpoint, parameters: Record) => ApiResponse; /** * A generic form for the asynchronous function that actually submits the - * request to the API and returns the corresporing response. Helpful when + * request to the API and returns the corresponding response. Helpful when * making an extensible endpoint definition. */ - export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; + export type ApiHandler = (parameters: Record, methodOptions?: Record) => ApiResponse; /** * A literal union type indicating the valid actions for these 'googleapis' - * requestions + * requests */ export type Action = 'create' | 'retrieve' | 'update'; /** * An interface defining any entity on which one can invoke - * anuy of the following handlers. All 'googleapis' wrappers + * any of the following handlers. All 'googleapis' wrappers * such as google.docs().documents and google.slides().presentations * satisfy this interface. */ @@ -109,7 +108,7 @@ export namespace GoogleApiServerUtils { * of needless duplicate clients that would arise from * making one new client instance per request. */ - const authenticationClients = new Map(); + const authenticationClients = new Map(); /** * This function receives the target sector ("which G-Suite app's API am I interested in?") @@ -120,23 +119,21 @@ export namespace GoogleApiServerUtils { * @returns the relevant 'googleapis' wrapper, if any */ export async function GetEndpoint(sector: string, userId: string): Promise { - return new Promise(async resolve => { - const auth = await retrieveOAuthClient(userId); - if (!auth) { - return resolve(); - } - let routed: Opt; - const parameters: any = { auth, version: 'v1' }; - switch (sector) { - case Service.Documents: - routed = google.docs(parameters).documents; - break; - case Service.Slides: - routed = google.slides(parameters).presentations; - break; - } - resolve(routed); - }); + const auth = await retrieveOAuthClient(userId); + if (!auth) { + return; + } + let routed: Opt; + const parameters: { version: 'v1' } = { /* auth, */ version: 'v1' }; ///* auth: OAuth2Client;*/ + switch (sector) { + case Service.Documents: + routed = google.docs(parameters).documents; + break; + case Service.Slides: + routed = google.slides(parameters).presentations; + break; + } + return routed; } /** @@ -149,19 +146,17 @@ export namespace GoogleApiServerUtils { * security. */ export async function retrieveOAuthClient(userId: string): Promise { - return new Promise(async resolve => { - const { credentials, refreshed } = await retrieveCredentials(userId); - if (!credentials) { - return resolve(); - } - let client = authenticationClients.get(userId); - if (!client) { - authenticationClients.set(userId, (client = generateClient(credentials))); - } else if (refreshed) { - client.setCredentials(credentials); - } - resolve(client); - }); + const { credentials, refreshed } = await retrieveCredentials(userId); + if (!credentials) { + return; + } + let client = authenticationClients.get(userId); + if (!client) { + authenticationClients.set(userId, (client = generateClient(credentials))); + } else if (refreshed) { + client.setCredentials(credentials); + } + return client; } /** @@ -173,7 +168,9 @@ export namespace GoogleApiServerUtils { */ function generateClient(credentials?: Credentials): OAuth2Client { const client = new google.auth.OAuth2(oAuthOptions); - credentials && client.setCredentials(credentials); + if (credentials) { + client.setCredentials(credentials); + } return client; } @@ -206,7 +203,7 @@ export namespace GoogleApiServerUtils { */ export async function processNewUser(userId: string, authenticationCode: string): Promise { const credentials = await new Promise((resolve, reject) => { - worker.getToken(authenticationCode, async (err, credentials) => { + worker.getToken(authenticationCode, (err, credentials) => { if (err || !credentials) { reject(err); return; @@ -221,7 +218,7 @@ export namespace GoogleApiServerUtils { /** * This type represents the union of the full set of OAuth2 credentials - * and all of a Google user's publically available information. This is the strucure + * and all of a Google user's publicly available information. This is the structure * of the JSON object we ultimately store in the googleAuthentication table of the database. */ export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; @@ -297,15 +294,15 @@ export namespace GoogleApiServerUtils { async function refreshAccessToken(credentials: Credentials, userId: string): Promise { const headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; const { client_id, client_secret } = GoogleCredentialsLoader.ProjectCredentials; - const url = `https://oauth2.googleapis.com/token?${qs.stringify({ - refreshToken: credentials.refresh_token, + const params = new URLSearchParams({ + refresh_token: credentials.refresh_token!, client_id, client_secret, grant_type: 'refresh_token', - })}`; - const { access_token, expires_in } = await new Promise(async resolve => { - const response = await request.post(url, headerParameters); - resolve(JSON.parse(response)); + }); + const url = `https://oauth2.googleapis.com/token?${params.toString()}`; + const { access_token, expires_in } = await new Promise<{ access_token: string; expires_in: number }>(resolve => { + request.post(url, headerParameters).then(response => resolve(JSON.parse(response))); }); // expires_in is in seconds, but we're building the new expiry date in milliseconds const expiry_date = new Date().getTime() + expires_in * 1000; -- cgit v1.2.3-70-g09d2 From 2e03c9bf1af2796faef8b81b326b48f4cd136d95 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 26 Feb 2025 18:31:13 -0500 Subject: fixed toggling number dropdown. Added firefly to gpt popup menu. --- packages/components/src/components/Popup/Popup.tsx | 1 + src/client/Network.ts | 17 ++-- src/client/util/DocumentManager.ts | 5 +- src/client/views/nodes/ImageBox.tsx | 2 +- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 90 ++++++++++++++++------ src/client/views/smartdraw/SmartDrawHandler.tsx | 78 +++++++++++++------ src/server/ApiManagers/FireflyManager.ts | 13 ++-- 7 files changed, 142 insertions(+), 64 deletions(-) (limited to 'src/server') diff --git a/packages/components/src/components/Popup/Popup.tsx b/packages/components/src/components/Popup/Popup.tsx index aa732d568..f0920b723 100644 --- a/packages/components/src/components/Popup/Popup.tsx +++ b/packages/components/src/components/Popup/Popup.tsx @@ -71,6 +71,7 @@ export const Popup = (props: IPopupProps) => { if (rect && !(rect.left < e.clientX && rect.top < e.clientY && rect.right > e.clientX && rect.bottom > e.clientY) && !popupContainsPt?.(e.clientX, e.clientY)) { e.preventDefault(); setOpen(false); + e.stopPropagation(); } }; diff --git a/src/client/Network.ts b/src/client/Network.ts index 3b0406141..323a2648c 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,5 +1,4 @@ import formidable from 'formidable'; -import * as requestPromise from 'request-promise'; import { ClientUtils } from '../ClientUtils'; import { Utils } from '../Utils'; import { Upload } from '../server/SharedMediaTypes'; @@ -16,13 +15,17 @@ export namespace Networking { } export function PostToServer(relativeRoute: string, body?: unknown) { - const options = { - uri: ClientUtils.prepend(relativeRoute), + return fetch(ClientUtils.prepend(relativeRoute), { method: 'POST', - body, - json: true, - }; - return requestPromise.post(options); + headers: { + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }).then(async response => { + if (response.ok) return response.json(); + + return await response.text().then(text => ({ error: '' + response.status + ':' + response.statusText + '-' + text })); + }); } /** diff --git a/src/client/util/DocumentManager.ts b/src/client/util/DocumentManager.ts index acb35f7eb..e33449782 100644 --- a/src/client/util/DocumentManager.ts +++ b/src/client/util/DocumentManager.ts @@ -354,9 +354,10 @@ export class DocumentManager { // bcz: should this delay be an options parameter? setTimeout(() => { Doc.linkFollowHighlight(viewSpec ? [docView.Document, viewSpec] : docView.Document, undefined, options.effect); - if (options.zoomTextSelections && Doc.IsUnhighlightTimerSet() && contextView && targetDoc.text_html) { + const zoomableText = StrCast(targetDoc.text_html, StrCast(targetDoc.ai_firefly_prompt)); + if (options.zoomTextSelections && Doc.IsUnhighlightTimerSet() && contextView && zoomableText) { // if the docView is a text anchor, the contextView is the PDF/Web/Text doc - contextView.setTextHtmlOverlay(StrCast(targetDoc.text_html), options.effect); + contextView.setTextHtmlOverlay(zoomableText, options.effect); DocumentManager._overlayViews.add(contextView); } Doc.AddUnHighlightWatcher(() => { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 496c3a40a..b6b0e4a8a 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -627,7 +627,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { } else SmartDrawHandler.Instance.regenerate([this.Document], undefined, undefined, this._regenInput || StrCast(this.Document.title), true).then( action(newImgs => { - if (newImgs[0]) { + if (newImgs[0] && !(newImgs[0] instanceof Doc)) { const url = newImgs[0].pathname; const imgField = new ImageField(url); this._prevImgs.length === 0 && diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index 2cf39bec4..efddfb841 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -3,6 +3,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; import { CgCornerUpLeft } from 'react-icons/cg'; import ReactLoading from 'react-loading'; import { TypeAnimation } from 'react-type-animation'; @@ -16,14 +17,15 @@ import { Docs } from '../../../documents/Documents'; import { SettingsManager } from '../../../util/SettingsManager'; import { SnappingManager } from '../../../util/SnappingManager'; import { undoable } from '../../../util/UndoManager'; +import { DictationButton } from '../../DictationButton'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { TagItem } from '../../TagsView'; import { ChatSortField, docSortings } from '../../collections/CollectionSubView'; import { DocumentView } from '../../nodes/DocumentView'; +import { SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; import { AnchorMenu } from '../AnchorMenu'; import './GPTPopup.scss'; -import { DictationButton } from '../../DictationButton'; -import { AiOutlineSend } from 'react-icons/ai'; +import { FireflyImageDimensions } from '../../smartdraw/FireflyConstants'; export enum GPTPopupMode { SUMMARY, // summary of seleted document text @@ -32,6 +34,7 @@ export enum GPTPopupMode { GPT_MENU, // menu for choosing type of prompts user will provide USER_PROMPT, // user prompts for sorting,filtering and asking about docs QUIZ_RESPONSE, // user definitions or explanations to be evaluated by GPT + FIREFLY, // firefly image generation } @observer @@ -93,6 +96,7 @@ export class GPTPopup extends ObservableReactComponent { } @observable private _conversationArray: string[] = ['Hi! In this pop up, you can ask ChatGPT questions about your documents and filter / sort them. ']; + @observable private _fireflyArray: string[] = ['Hi! In this pop up, you can ask Firefly to create images. ']; @observable private _chatEnabled: boolean = false; @action private setChatEnabled = (start: boolean) => (this._chatEnabled = start); @observable private _gptProcessing: boolean = false; @@ -197,6 +201,21 @@ export class GPTPopup extends ObservableReactComponent { .catch(err => console.error('GPT call failed', err)) )) // prettier-ignore + generateFireflyImage = (imgDesc: string) => { + // if (this._fireflyRefStrength) { + // DrawingFillHandler.drawingToImage(this.props.Document, this._fireflyRefStrength, this._regenInput || StrCast(this.Document.title), this.Document)?.then( + // action(() => { + // this._regenerateLoading = false; + // }) + // ); + // } else∂ + return SmartDrawHandler.CreateWithFirefly(imgDesc, FireflyImageDimensions.Square, 0) + .then(action(() => (this._userPrompt = ''))) + .catch(e => { + alert(e); + return undefined; + }); + }; /** * Generates a response to the user's question about the docs in the collection. * The type of response depends on the chat's analysis of the type of their question @@ -338,6 +357,21 @@ export class GPTPopup extends ObservableReactComponent { gptMenu = () => (
+
); - callGpt = (isUserPrompt: boolean) => { + callGpt = action((mode: GPTPopupMode) => { this.setGptProcessing(true); - if (isUserPrompt) { - this._conversationArray.push(this._userPrompt); - return this.generateUserPromptResponse(this._userPrompt).then(action(() => (this._userPrompt = ''))); + switch (mode) { + case GPTPopupMode.FIREFLY: + this._fireflyArray.push(this._userPrompt); + return this.generateFireflyImage(this._userPrompt).then(action(() => (this._userPrompt = ''))); + case GPTPopupMode.USER_PROMPT: + this._conversationArray.push(this._userPrompt); + return this.generateUserPromptResponse(this._userPrompt).then(action(() => (this._userPrompt = ''))); + case GPTPopupMode.QUIZ_RESPONSE: + this._conversationArray.push(this._quizAnswer); + return this.generateQuizAnswerAnalysis(DocumentView.SelectedDocs().lastElement(), this._quizAnswer).then(action(() => (this._quizAnswer = ''))); } - this._conversationArray.push(this._quizAnswer); - return this.generateQuizAnswerAnalysis(DocumentView.SelectedDocs().lastElement(), this._quizAnswer).then(action(() => (this._quizAnswer = ''))); - }; + }); @action - handleKeyPress = async (e: React.KeyboardEvent, isUserPrompt: boolean) => { + handleKeyPress = async (e: React.KeyboardEvent, mode: GPTPopupMode) => { this._askDictation?.stopDictation(); if (e.key === 'Enter') { e.stopPropagation(); - this.callGpt(isUserPrompt).then(() => { + this.callGpt(mode)?.then(() => { this.setGptProcessing(false); this.scrollToBottom(); }); @@ -401,7 +440,7 @@ export class GPTPopup extends ObservableReactComponent {
- {this._conversationArray.map((message, index) => ( + {(this._mode === GPTPopupMode.FIREFLY ? this._fireflyArray : this._conversationArray).map((message, index) => (
{message}
@@ -414,21 +453,21 @@ export class GPTPopup extends ObservableReactComponent { ); - promptBox = (isUserPrompt: boolean) => ( + promptBox = (heading: string, value: string, onChange: (e: string) => string, placeholder: string) => ( <>
- {this.heading(isUserPrompt ? 'ASK' : 'QUIZ')} + {this.heading(heading)} {this.gptUserInput()}
(isUserPrompt ? this.setUserPrompt : this.setQuizAnswer)(e.target.value)} - onKeyDown={e => this.handleKeyPress(e, isUserPrompt)} + onChange={e => onChange(e.target.value)} + onKeyDown={e => this.handleKeyPress(e, this._mode)} type="text" - placeholder={`${isUserPrompt ? 'Have ChatGPT sort, tag, define, or filter your documents for you!' : 'Describe/answer the selected document!'}`} + placeholder={placeholder} />
); @@ -461,7 +500,7 @@ export class GPTPopup extends ObservableReactComponent { dalle generation -
+
@@ -599,7 +638,7 @@ export class GPTPopup extends ObservableReactComponent { color={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag) ? 'red' : 'transparent'} onClick={() => this._collectionContext && Doc.setDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag, 'remove')} /> - {(this._mode === GPTPopupMode.USER_PROMPT || this._mode === GPTPopupMode.QUIZ_RESPONSE) && ( + {[GPTPopupMode.USER_PROMPT, GPTPopupMode.QUIZ_RESPONSE, GPTPopupMode.FIREFLY].includes(this._mode) && ( } onClick={() => (this._mode = GPTPopupMode.GPT_MENU)} /> )} @@ -613,8 +652,9 @@ export class GPTPopup extends ObservableReactComponent { {(() => { //prettier-ignore switch (this._mode) { - case GPTPopupMode.USER_PROMPT: - case GPTPopupMode.QUIZ_RESPONSE: return this.promptBox(this._mode === GPTPopupMode.USER_PROMPT); + case GPTPopupMode.USER_PROMPT: return this.promptBox("ASK", this._userPrompt, this.setUserPrompt, 'Ask GPT to sort, tag, define, or filter your documents for you!'); + case GPTPopupMode.FIREFLY: return this.promptBox("CREATE", this._userPrompt, this.setUserPrompt, 'Ask Firefly to generate images'); + case GPTPopupMode.QUIZ_RESPONSE: return this.promptBox("QUIZ", this._quizAnswer, this.setQuizAnswer, 'Describe/answer the selected document!'); case GPTPopupMode.GPT_MENU: return this.menuBox(); case GPTPopupMode.SUMMARY: return this.summaryBox(); case GPTPopupMode.DATA: return this.dataAnalysisBox(); diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 7db9ef133..9d67d111b 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -1,6 +1,6 @@ +import { Button, IconButton } from '@dash/components'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Checkbox, FormControlLabel, Radio, RadioGroup, Slider, Switch } from '@mui/material'; -import { Button, IconButton } from '@dash/components'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; @@ -13,7 +13,9 @@ import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import { Networking } from '../../Network'; import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; +import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; import { undoable } from '../../util/UndoManager'; @@ -21,12 +23,10 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; -import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -import './SmartDrawHandler.scss'; -import { Networking } from '../../Network'; +import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; -import { FireflyDimensionsMap, FireflyImageDimensions, FireflyImageData } from './FireflyConstants'; -import { DocumentType } from '../../documents/DocumentTypes'; +import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants'; +import './SmartDrawHandler.scss'; export interface DrawingOptions { text: string; @@ -268,28 +268,56 @@ export class SmartDrawHandler extends ObservableReactComponent { /** * Calls Firefly API to create an image based on user input */ - createImageWithFirefly = (input: string, seed?: number, changeInPlace?: boolean): Promise => { + createImageWithFirefly = (input: string, seed?: number): Promise => { + this._lastInput.text = input; + return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed); + }; /** + * Calls Firefly API to create an image based on user input + */ + recreateImageWithFirefly = (input: string, seed?: number): Promise => { this._lastInput.text = input; - const dims = FireflyDimensionsMap[this._imgDims]; + return SmartDrawHandler.ReCreateWithFirefly(input, this._imgDims, seed); + }; + public static ReCreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { + const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) .then(img => { - const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1]; - if (!changeInPlace) { - const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { - title: input.match(/^(.*?)~~~.*$/)?.[1] || input, - nativeWidth: dims.width, - nativeHeight: dims.height, - ai: 'firefly', - ai_firefly_seed: newseed, - ai_firefly_prompt: input, - }); - DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); - this._selectedDocs.push(imgDoc); + if (img.error) { + alert('recreate image failed: ' + img.error); + return undefined; } return { prompt: input, seed, pathname: img.accessPaths.agnostic.client }; }) - .catch(e => alert('create image failed: ' + e.toString())); - }; + .catch(e => { + alert('recreate image failed: ' + e.toString()); + return undefined; + }); + } + public static CreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { + const dims = FireflyDimensionsMap[imgDims]; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) + .then(img => { + if (img.error) { + alert('create image failed: ' + img.error); + return undefined; + } + const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1]; + const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { + title: input.match(/^(.*?)~~~.*$/)?.[1] || input, + nativeWidth: dims.width, + nativeHeight: dims.height, + ai: 'firefly', + ai_firefly_seed: newseed, + ai_firefly_prompt: input, + }); + DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); + return imgDoc; + }) + .catch(e => { + alert('create image failed: ' + e.toString()); + return undefined; + }); + } /** * Regenerates drawings with the option to add a specific regenerate prompt/request. @@ -307,10 +335,12 @@ export class SmartDrawHandler extends ObservableReactComponent { if (this._regenInput) { // if (this._selectedDoc) { const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput; - return this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed), changeInPlace); + return changeInPlace ? this.recreateImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)) : this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)); // } } - return this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), undefined, changeInPlace); + return changeInPlace + ? this.recreateImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)) + : this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)); case DocumentType.COL: { try { let res; diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 160a94d40..8a310aed8 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -132,7 +132,8 @@ export default class FireflyManager extends ApiManager { ], body: body, }) - .then(response2 => response2.json().then(json => ({ seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url }))) + .then(response2 => response2.json()) + .then(json => (json.error_code ? json : { seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url })) .catch(error => { console.error('Error:', error); return undefined; @@ -332,10 +333,12 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => - DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info); - }) + img.error_code + ? _invalid(res, img.message) + : DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) ), }); -- cgit v1.2.3-70-g09d2 From a162057114fd3d961e447d6c3051c1c8c0c6b46d Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 26 Feb 2025 23:41:07 -0500 Subject: made ai. button draggable to regenerate images. --- src/ClientUtils.ts | 2 +- src/client/views/DocumentButtonBar.tsx | 18 +++++++- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 6 +-- src/client/views/smartdraw/DrawingFillHandler.tsx | 6 +-- src/client/views/smartdraw/SmartDrawHandler.tsx | 51 +++++++++++++++++++--- src/server/ApiManagers/FireflyManager.ts | 10 ++--- 7 files changed, 72 insertions(+), 23 deletions(-) (limited to 'src/server') diff --git a/src/ClientUtils.ts b/src/ClientUtils.ts index c5b61477b..e1f490c1a 100644 --- a/src/ClientUtils.ts +++ b/src/ClientUtils.ts @@ -599,7 +599,7 @@ export function StopEvent(e: React.PointerEvent | React.MouseEvent | React.Keybo export function setupMoveUpEvents( target: object, - e: React.PointerEvent, + e: React.PointerEvent | PointerEvent, moveEvent: (e: PointerEvent, down: number[], delta: number[]) => boolean, upEvent: (e: PointerEvent, movement: number[], isClick: boolean) => void, clickEvent: (e: PointerEvent, doubleTap?: boolean) => unknown, diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index a9f03a658..b17dbc93d 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -28,6 +28,7 @@ import { DocumentLinksButton } from './nodes/DocumentLinksButton'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere } from './nodes/OpenWhere'; import { DashFieldView } from './nodes/formattedText/DashFieldView'; +import { SmartDrawHandler } from './smartdraw/SmartDrawHandler'; @observer export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (DocumentView | undefined)[]; stack?: unknown }> { @@ -319,7 +320,22 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( const targetDoc = this.view0?.Document; return !targetDoc ? null : ( Edit with AI}> -
this.view0?.toggleAIEditor(), 'toggle AI editor')}> +
+ setupMoveUpEvents( + this, + e, + me => { + this.view0?.docViewPath().slice(-2)[0]?.ComponentView?.showSmartDraw?.(me.x, me.y, true); + SmartDrawHandler.Instance.startDragging(me); + return true; + }, + emptyFunction, + undoable(() => this.view0?.toggleAIEditor(), 'toggle AI editor') + ) + }>
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 752e8b217..aa9b9b0fa 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1294,7 +1294,7 @@ export class CollectionFreeFormView extends CollectionSubView { const docData = doc[DocData]; - docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; + docData.title = opts.text; docData._width = opts.size; docData.ai_drawing_input = opts.text; docData.ai_drawing_complexity = opts.complexity; diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index f3aea76a5..55474cb7e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -625,11 +625,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { onClick={action(async () => { this._regenerateLoading = true; if (this._fireflyRefStrength) { - DrawingFillHandler.drawingToImage(this.props.Document, this._fireflyRefStrength, this._regenInput || StrCast(this.Document.title), this.Document)?.then( - action(() => { - this._regenerateLoading = false; - }) - ); + DrawingFillHandler.drawingToImage(this.props.Document, this._fireflyRefStrength, this._regenInput || StrCast(this.Document.title), this.Document)?.then(action(() => (this._regenerateLoading = false))); } else { SmartDrawHandler.Instance.regenerate([this.Document], undefined, undefined, this._regenInput || StrCast(this.Document.title), true).then( action(newImgs => { diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index d0a840883..7447f8afb 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -47,11 +47,11 @@ export class DrawingFillHandler { return imageUrlToBase64(structureUrl) .then(gptDescribeImage) .then((prompt, newPrompt = user_prompt || prompt) => - Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: `${newPrompt}`, width: dims.width, height: dims.height, structure: structureUrl, strength, presets: styles, styleUrl }) - .then((infos: Upload.ImageInformation[]) => { + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: `${newPrompt}`, width: dims.width, height: dims.height, structureUrl, strength, presets: styles, styleUrl }) + .then(res => { const genratedDocs = DocCast(drawing.ai_firefly_generatedDocs) ?? Docs.Create.MasonryDocument([], { _width: 400, _height: 400 }); drawing[DocData].ai_firefly_generatedDocs = genratedDocs; - infos.map(info => + (res as Upload.ImageInformation[]).map(info => Doc.AddDocToList( genratedDocs, undefined, diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 811bc6e25..fbf471900 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { AiOutlineSend } from 'react-icons/ai'; import ReactLoading from 'react-loading'; import { INode, parse } from 'svgson'; -import { imageUrlToBase64 } from '../../../ClientUtils'; +import { imageUrlToBase64, setupMoveUpEvents } from '../../../ClientUtils'; import { unimplementedFunction } from '../../../Utils'; import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; @@ -205,11 +205,7 @@ export class SmartDrawHandler extends ObservableReactComponent { this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { - await this.regenerate(this._selectedDocs).then( - action(() => { - this._showEditBox = false; - }) - ); + await this.regenerate(this._selectedDocs, undefined, undefined, this._regenInput).then(action(() => (this._showEditBox = false))); } else { this._showOptions = false; try { @@ -295,7 +291,7 @@ export class SmartDrawHandler extends ObservableReactComponent { } const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)?.[1]; const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { - title: input.match(/^(.*?)~~~.*$/)?.[1] || input, + title: input, nativeWidth: dims.width, nativeHeight: dims.height, ai: 'firefly', @@ -523,6 +519,19 @@ export class SmartDrawHandler extends ObservableReactComponent { return (
+ setupMoveUpEvents( + this, + e, + action(me => { + this._pageX = this._pageX + me.movementX; + this._pageY = this._pageY + me.movementY; + return false; + }), + () => {}, + () => {} + ) + } style={{ display: this._display ? '' : 'none', left: this._pageX, @@ -581,6 +590,7 @@ export class SmartDrawHandler extends ObservableReactComponent { onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} onKeyDown={this.handleKeyPress} placeholder="Edit instructions" + onPointerDown={e => e.stopPropagation()} />