From fec79d2b5b8feb361e489c9ee41ee720507d0806 Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Thu, 27 Jul 2023 13:23:42 -0400 Subject: changes --- src/client/apis/gpt/GPT.ts | 28 +---- src/client/util/DragManager.ts | 4 +- src/client/views/collections/TabDocView.tsx | 6 +- .../CollectionFreeFormLinkView.tsx | 8 +- src/client/views/global/globalScripts.ts | 50 ++++---- src/client/views/nodes/WebBox.tsx | 5 + .../views/nodes/formattedText/FormattedTextBox.tsx | 89 +++++++++----- src/client/views/pdf/AnchorMenu.tsx | 21 ++-- src/client/views/pdf/GPTPopup/GPTPopup.scss | 52 +++++++++ src/client/views/pdf/GPTPopup/GPTPopup.tsx | 129 +++++++++++++++------ src/client/views/pdf/PDFViewer.tsx | 3 +- 11 files changed, 261 insertions(+), 134 deletions(-) (limited to 'src') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 18222b32a..6b4106f56 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -14,7 +14,7 @@ type GPTCallOpts = { }; const callTypeMap: { [type: string]: GPTCallOpts } = { - summary: { model: 'text-davinci-003', maxTokens: 256, temp: 0.5, prompt: 'Summarize this text briefly: ' }, + summary: { model: 'text-davinci-003', maxTokens: 256, temp: 0.5, prompt: 'Summarize this text in simpler terms: ' }, edit: { model: 'text-davinci-003', maxTokens: 256, temp: 0.5, prompt: 'Reword this: ' }, completion: { model: 'text-davinci-003', maxTokens: 256, temp: 0.5, prompt: '' }, }; @@ -47,7 +47,7 @@ const gptAPICall = async (inputText: string, callType: GPTCallType) => { } }; -const gptImageCall = async (prompt: string) => { +const gptImageCall = async (prompt: string, n?: number) => { try { const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY, @@ -55,33 +55,15 @@ const gptImageCall = async (prompt: string) => { const openai = new OpenAIApi(configuration); const response = await openai.createImage({ prompt: prompt, - n: 1, + n: n ?? 1, size: '1024x1024', }); - return response.data.data[0].url; + return response.data.data.map(data => data.url); + // return response.data.data[0].url; } catch (err) { console.error(err); return; } }; -// const gptEditCall = async (selectedText: string, fullText: string) => { -// try { -// const configuration = new Configuration({ -// apiKey: process.env.OPENAI_KEY, -// }); -// const openai = new OpenAIApi(configuration); -// const response = await openai.createCompletion({ -// model: 'text-davinci-003', -// max_tokens: 256, -// temperature: 0.1, -// prompt: `Replace the phrase ${selectedText} inside of ${fullText}.`, -// }); -// return response.data.choices[0].text.trim(); -// } catch (err) { -// console.log(err); -// return 'Error connecting with API.'; -// } -// }; - export { gptAPICall, gptImageCall, GPTCallType }; diff --git a/src/client/util/DragManager.ts b/src/client/util/DragManager.ts index 85101fcab..ac948c7c7 100644 --- a/src/client/util/DragManager.ts +++ b/src/client/util/DragManager.ts @@ -195,7 +195,7 @@ export namespace DragManager { } // drag a document and drop it (or make an embed/copy on drop) - export function StartDocumentDrag(eles: HTMLElement[], dragData: DocumentDragData, downX: number, downY: number, options?: DragOptions, dropEvent?: () => any) { + export function StartDocumentDrag(eles: HTMLElement[], dragData: DocumentDragData, downX: number, downY: number, options?: DragOptions, onDropCompleted?: (e?: DragCompleteEvent) => any) { const addAudioTag = (dropDoc: any) => { dropDoc && !dropDoc.author_date && (dropDoc.author_date = new DateField()); dropDoc instanceof Doc && DocUtils.MakeLinkToActiveAudio(() => dropDoc); @@ -203,7 +203,7 @@ export namespace DragManager { }; const finishDrag = async (e: DragCompleteEvent) => { const docDragData = e.docDragData; - dropEvent?.(); // glr: optional additional function to be called - in this case with presentation trails + onDropCompleted?.(e); // glr: optional additional function to be called - in this case with presentation trails if (docDragData && !docDragData.droppedDocuments.length) { docDragData.dropAction = dragData.userDropAction || dragData.dropAction; docDragData.droppedDocuments = ( diff --git a/src/client/views/collections/TabDocView.tsx b/src/client/views/collections/TabDocView.tsx index 67b7b39dd..1ff5688f9 100644 --- a/src/client/views/collections/TabDocView.tsx +++ b/src/client/views/collections/TabDocView.tsx @@ -137,7 +137,11 @@ export class TabDocView extends React.Component { setupMoveUpEvents( this, e, - e => !e.defaultPrevented && DragManager.StartDocumentDrag([iconWrap], new DragManager.DocumentDragData([doc], doc.dropAction as dropActionType), e.clientX, e.clientY), + e => + !e.defaultPrevented && + DragManager.StartDocumentDrag([iconWrap], new DragManager.DocumentDragData([doc], doc.dropAction as dropActionType), e.clientX, e.clientY, undefined, () => { + CollectionDockingView.CloseSplit(doc); + }), returnFalse, action(e => { if (this.view) { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx index db000d5de..747ab9249 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx @@ -238,11 +238,11 @@ export class CollectionFreeFormLinkView extends React.Component {} }; diff --git a/src/client/views/nodes/WebBox.tsx b/src/client/views/nodes/WebBox.tsx index 34a1229ba..5cdd6b5f2 100644 --- a/src/client/views/nodes/WebBox.tsx +++ b/src/client/views/nodes/WebBox.tsx @@ -354,6 +354,11 @@ export class WebBox extends ViewBoxAnnotatableComponent, dataDoc: Doc) => void; @@ -896,12 +897,13 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent { - console.log(this.dataDoc.text); - if (resIndex < this.gptRes.length) { - this.dataDoc.text = (this.dataDoc.text as RichTextField)?.Text + this.gptRes[resIndex]; + animateRes = (resIndex: number, newText: string) => { + if (resIndex < newText.length) { + const marks = this._editorView?.state.storedMarks ?? []; + // if (!marks) return; + this._editorView?.dispatch(this._editorView.state.tr.setStoredMarks(marks).insertText(newText[resIndex]).setStoredMarks(marks)); setTimeout(() => { - this.animateRes(resIndex + 1); + this.animateRes(resIndex + 1, newText); }, 20); } }; @@ -912,47 +914,68 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent { + // const state = this._editorView?.state; + // if (!state) return; + // const to = state.selection.to; + // const updated = TextSelection.create(state.doc, to, to); + // this._editorView?.dispatch(state.tr.setSelection(updated).insertText('\n', to)); + // this._editorView?.dispatch(this._editorView.state.tr.setStoredMarks(marks).insertText('\nTesting').setStoredMarks(marks)); + // console.log('After ', this._editorView?.state.storedMarks); try { let res = await gptAPICall((this.dataDoc.text as RichTextField)?.Text, GPTCallType.COMPLETION); - if (res) { - this.gptRes = res; - this.animateRes(0); + if (!res) { + console.error('GPT call failed'); + this.animateRes(0, 'Something went wrong.'); + } else { + this.animateRes(0, res); } } catch (err) { - console.log(err); - this.dataDoc.text = (this.dataDoc.text as RichTextField)?.Text + 'Something went wrong'; + console.error('GPT call failed'); + this.animateRes(0, 'Something went wrong.'); } }); generateImage = async () => { console.log('Generate image from text: ', (this.dataDoc.text as RichTextField)?.Text); + GPTPopup.Instance?.setImgTargetDoc(this.rootDoc); + GPTPopup.Instance.setImgUrls([]); + GPTPopup.Instance.setMode(GPTPopupMode.IMAGE); + GPTPopup.Instance.setVisible(true); + GPTPopup.Instance.addToCollection = this.props.addDocument; + GPTPopup.Instance.setLoading(true); + try { - let image_url = await gptImageCall((this.dataDoc.text as RichTextField)?.Text); - if (image_url) { - const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [image_url] }); + // make this support multiple images + let image_urls = await gptImageCall((this.dataDoc.text as RichTextField)?.Text); + console.log(image_urls); + if (image_urls) { + const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [image_urls[0]] }); const source = Utils.prepend(result.accessPaths.agnostic.client); - const newDoc = Docs.Create.ImageDocument(source, { - x: NumCast(this.rootDoc.x) + NumCast(this.layoutDoc._width) + 10, - y: NumCast(this.rootDoc.y), - _height: 200, - _width: 200, - data_nativeWidth: result.nativeWidth, - data_nativeHeight: result.nativeHeight, - }); - if (Doc.IsInMyOverlay(this.rootDoc)) { - newDoc.overlayX = this.rootDoc.x; - newDoc.overlayY = NumCast(this.rootDoc.y) + NumCast(this.rootDoc._height); - Doc.AddToMyOverlay(newDoc); - } else { - this.props.addDocument?.(newDoc); - } - // Create link between prompt and image - DocUtils.MakeLink(this.rootDoc, newDoc, { link_relationship: 'Image Prompt' }); + GPTPopup.Instance.setImgUrls([source]); + + // const newDoc = Docs.Create.ImageDocument(source, { + // x: NumCast(this.rootDoc.x) + NumCast(this.layoutDoc._width) + 10, + // y: NumCast(this.rootDoc.y), + // _height: 200, + // _width: 200, + // data_nativeWidth: result.nativeWidth, + // data_nativeHeight: result.nativeHeight, + // }); + // if (Doc.IsInMyOverlay(this.rootDoc)) { + // newDoc.overlayX = this.rootDoc.x; + // newDoc.overlayY = NumCast(this.rootDoc.y) + NumCast(this.rootDoc._height); + // Doc.AddToMyOverlay(newDoc); + // } else { + // this.props.addDocument?.(newDoc); + // } + // // Create link between prompt and image + // DocUtils.MakeLink(this.rootDoc, newDoc, { link_relationship: 'Image Prompt' }); } } catch (err) { console.log(err); return ''; } + GPTPopup.Instance.setLoading(false); }; breakupDictation = () => { @@ -1249,8 +1272,10 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent { gptSummarize = async (e: React.PointerEvent) => { GPTPopup.Instance.setVisible(true); this.setHighlightRange(undefined); - this.setGPTMode(GPTPopupMode.SUMMARY); + GPTPopup.Instance.setMode(GPTPopupMode.SUMMARY); GPTPopup.Instance.setLoading(true); try { @@ -183,6 +183,10 @@ export class AnchorMenu extends AntimodeMenu { GPTPopup.Instance.setLoading(false); }; + gptImage = async () => { + console.log(this.GetAnchor(undefined, false)); + }; + /** * Replaces text suggestions from GPT. */ @@ -293,21 +297,12 @@ export class AnchorMenu extends AntimodeMenu { )} - {/* */} {AnchorMenu.Instance.OnAudio === unimplementedFunction ? null : ( } color={StrCast(Doc.UserDoc().userColor)} /> )} - {this.canEdit() && } color={StrCast(Doc.UserDoc().userColor)} />} + {/* Removed text editing for now, not quite ready */} + {/* {this.canEdit() && } color={StrCast(Doc.UserDoc().userColor)} />} */} + {} color={StrCast(Doc.UserDoc().userColor)} />} } popup={} color={StrCast(Doc.UserDoc().userColor)} /> {AnchorMenu.Instance.StartCropDrag === unimplementedFunction ? null : ( } color={StrCast(Doc.UserDoc().userColor)} /> diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.scss b/src/client/views/pdf/GPTPopup/GPTPopup.scss index 2f0ff83e2..478b7d4ba 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.scss +++ b/src/client/views/pdf/GPTPopup/GPTPopup.scss @@ -24,6 +24,7 @@ $highlightedText: #82e0ff; .summary-heading { display: flex; + justify-content: space-between; align-items: center; border-bottom: 1px solid $greyborder; padding-bottom: 5px; @@ -110,6 +111,57 @@ $highlightedText: #82e0ff; } } +.image-content-wrapper { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 8px; + padding-bottom: 16px; + + .img-wrapper { + position: relative; + cursor: pointer; + + .img-container { + position: relative; + + img { + position: relative; + } + } + + .img-container::after { + content: ''; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.5); + opacity: 0; + transition: opacity 0.3s ease; + } + + .btn-container { + position: absolute; + right: 8px; + bottom: 8px; + opacity: 0; + transition: opacity 0.3s ease; + } + + &:hover { + .img-container::after { + opacity: 1; + } + + .btn-container { + opacity: 1; + } + } + } +} + // Typist CSS .Typist .Cursor { display: inline-block; diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index 9f28cb5d1..aeee90d16 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -1,4 +1,5 @@ import React = require('react'); +import './GPTPopup.scss'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, observable } from 'mobx'; import { observer } from 'mobx-react'; @@ -6,13 +7,14 @@ import ReactLoading from 'react-loading'; import Typist from 'react-typist'; import { Doc } from '../../../../fields/Doc'; import { DocUtils, Docs } from '../../../documents/Documents'; -import './GPTPopup.scss'; -import { Button, IconButton } from 'browndash-components'; -import { StrCast } from '../../../../fields/Types'; +import { Button, IconButton, Type } from 'browndash-components'; +import { NumCast, StrCast } from '../../../../fields/Types'; +import { CgClose } from 'react-icons/cg'; export enum GPTPopupMode { SUMMARY, EDIT, + IMAGE, } interface GPTPopupProps {} @@ -39,6 +41,14 @@ export class GPTPopup extends React.Component { public setText = (text: string) => { this.text = text; }; + + @observable + public imageUrls: string[] = []; + @action + public setImgUrls = (imgs: string[]) => { + this.imageUrls = imgs; + }; + @observable public mode: GPTPopupMode = GPTPopupMode.SUMMARY; @action @@ -58,6 +68,8 @@ export class GPTPopup extends React.Component { public setDone = (done: boolean) => { this.done = done; }; + + // change what can be a ref into a ref @observable private sidebarId: string = ''; @action @@ -65,19 +77,30 @@ export class GPTPopup extends React.Component { this.sidebarId = id; }; - // reference - // if (focusNode) { - // const anchor = srcWeb?.ComponentView?.getAnchor?.(true); - // anchor && DocUtils.MakeLink(htmlDoc, anchor, {}); - // } + // pdfs and webpages + @observable + private targetAnchor: Doc | undefined; + @action + public setTargetAnchor = (anchor: Doc) => { + this.targetAnchor = anchor; + }; + @observable - private pdfAnchor: Doc | undefined; + private imgTargetDoc: Doc | undefined; @action - public setPdfAnchor = (anchor: Doc) => { - this.pdfAnchor = anchor; + public setImgTargetDoc = (anchor: Doc) => { + this.imgTargetDoc = anchor; + }; + + @observable + private textAnchor: Doc | undefined; + @action + public setTextAnchor = (anchor: Doc) => { + this.textAnchor = anchor; }; public addDoc: (doc: Doc | Doc[], sidebarKey?: string | undefined) => boolean = () => false; + public addToCollection: ((doc: Doc | Doc[], annotationKey?: string | undefined) => boolean) | undefined; /** * Transfers the summarization text to a sidebar annotation text document. @@ -90,13 +113,42 @@ export class GPTPopup extends React.Component { _layout_autoHeight: true, }); this.addDoc(newDoc, this.sidebarId); - if (this.pdfAnchor) { - DocUtils.MakeLink(newDoc, this.pdfAnchor, { + if (this.targetAnchor) { + DocUtils.MakeLink(newDoc, this.targetAnchor, { link_relationship: 'GPT Summary', }); } }; + /** + * Transfers the image urls to actual image docs + */ + private transferToImage = (source: string) => { + console.log('Text Anchor', this.textAnchor); + console.log('Whole doc anchor', this.imgTargetDoc); + const textAnchor = this.textAnchor ?? this.imgTargetDoc; + if (!textAnchor) return; + const newDoc = Docs.Create.ImageDocument(source, { + x: NumCast(textAnchor.x) + NumCast(textAnchor._width) + 10, + y: NumCast(textAnchor.y), + _height: 200, + _width: 200, + data_nativeWidth: 1024, + data_nativeHeight: 1024, + }); + if (Doc.IsInMyOverlay(textAnchor)) { + newDoc.overlayX = textAnchor.x; + newDoc.overlayY = NumCast(textAnchor.y) + NumCast(textAnchor._height); + Doc.AddToMyOverlay(newDoc); + } else { + this.addToCollection?.(newDoc); + } + // Create link between prompt and image + DocUtils.MakeLink(textAnchor, newDoc, { link_relationship: 'Image Prompt' }); + }; + + private getPreviewUrl = (source: string) => source.split('.').join('_m.'); + constructor(props: GPTPopupProps) { super(props); GPTPopup.Instance = this; @@ -108,6 +160,26 @@ export class GPTPopup extends React.Component { } }; + imageBox = () => { + return ( +
+ {this.heading('GENERATED IMAGE')} +
+ {this.imageUrls.map(rawSrc => ( +
+
+ dalle generation +
+
+
+
+ ))} +
+
+ ); + }; + summaryBox = () => ( <>
@@ -135,30 +207,21 @@ export class GPTPopup extends React.Component {
{this.done ? ( <> - {/* - */} - } color={StrCast(Doc.UserDoc().userVariantColor)} /> - + }} + color={StrCast(Doc.UserDoc().userVariantColor)} + type={Type.TERT} + />
)}
@@ -216,14 +279,14 @@ export class GPTPopup extends React.Component { heading = (headingText: string) => (
- {this.loading && } + {this.loading ? : } onClick={() => this.setVisible(false)} />}
); render() { return (
- {this.mode === GPTPopupMode.SUMMARY ? this.summaryBox() : this.editBox()} + {this.mode === GPTPopupMode.SUMMARY ? this.summaryBox() : this.mode === GPTPopupMode.IMAGE ? this.imageBox() : this.editBox()}
); } diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index 4fc31ffe3..1d1c34f4f 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -425,8 +425,7 @@ export class PDFViewer extends React.Component { GPTPopup.Instance.setSidebarId('data_sidebar'); const anchor = this._getAnchor(undefined, false); if (anchor) { - console.log(anchor); - GPTPopup.Instance.setPdfAnchor(anchor); + GPTPopup.Instance.setTargetAnchor(anchor); } GPTPopup.Instance.addDoc = this.props.sidebarAddDoc; }; -- cgit v1.2.3-70-g09d2