diff options
Diffstat (limited to 'src/client/views')
28 files changed, 2684 insertions, 401 deletions
diff --git a/src/client/views/ContextMenu.scss b/src/client/views/ContextMenu.scss index 4aaf2d03b..fb29b2d29 100644 --- a/src/client/views/ContextMenu.scss +++ b/src/client/views/ContextMenu.scss @@ -116,6 +116,13 @@ cursor: pointer; } +.contextMenu-itemSelected { + background: white; + color: black; + // background: lightgoldenrodyellow; + border-style: none; +} + .contextMenu-group { // width: 11vw; //10vw height: 30px; //2vh diff --git a/src/client/views/ContextMenu.tsx b/src/client/views/ContextMenu.tsx index 5edb5fc0d..0e98d2e35 100644 --- a/src/client/views/ContextMenu.tsx +++ b/src/client/views/ContextMenu.tsx @@ -182,7 +182,7 @@ export class ContextMenu extends ObservableReactComponent<{ noexpand?: boolean } @computed get menuItems() { if (!this._searchString) { - return this._items.map((item, ind) => <ContextMenuItem key={item.description + ind} {...item} noexpand={this.itemsNeedSearch ? true : item.noexpand} closeMenu={this.closeMenu} />); + return this._items.map((item, ind) => <ContextMenuItem key={item.description + ind} {...item} selected={ind === this._selectedIndex} noexpand={this.itemsNeedSearch ? true : item.noexpand} closeMenu={this.closeMenu} />); } return this.filteredItems.map((value, index) => Array.isArray(value) ? ( @@ -241,6 +241,11 @@ export class ContextMenu extends ObservableReactComponent<{ noexpand?: boolean } } @action + setLangIndex = (ind: number) => { + this._selectedIndex = ind; + }; + + @action onKeyDown = (e: React.KeyboardEvent) => { if (e.key === 'ArrowDown') { if (this._selectedIndex < this.flatItems.length - 1) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 9f1c7da3d..64cf7fbf8 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -546,6 +546,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faRobot, fa.faSatellite, fa.faStar, + fa.faFilePen, ] ); } diff --git a/src/client/views/MarqueeAnnotator.tsx b/src/client/views/MarqueeAnnotator.tsx index 8aed34d24..024ae7ba8 100644 --- a/src/client/views/MarqueeAnnotator.tsx +++ b/src/client/views/MarqueeAnnotator.tsx @@ -34,6 +34,7 @@ export interface MarqueeAnnotatorProps { getPageFromScroll?: (top: number) => number; finishMarquee: (x?: number, y?: number) => void; anchorMenuClick?: () => undefined | ((anchor: Doc) => void); + anchorMenuFlashcard?: () => Promise<String>; anchorMenuCrop?: (anchor: Doc | undefined, addCrop: boolean) => Doc | undefined; highlightDragSrcColor?: string; } @@ -46,10 +47,10 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP makeObservable(this); } - @observable private _width: number = 0; - @observable private _height: number = 0; - @computed get top() { return Math.min(this._start.y, this._start.y + this._height); } // prettier-ignore - @computed get left() { return Math.min(this._start.x, this._start.x + this._width);} // prettier-ignore + @observable Width: number = 0; + @observable Height: number = 0; + @computed get top() { return Math.min(this._start.y, this._start.y + this.Height); } // prettier-ignore + @computed get left() { return Math.min(this._start.x, this._start.x + this.Width);} // prettier-ignore static clearAnnotations = action((savedAnnotations: ObservableMap<number, HTMLDivElement[]>) => { AnchorMenu.Instance.Status = 'marquee'; @@ -167,7 +168,7 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP @action public onInitiateSelection(down: number[]) { - this._width = this._height = 0; + this.Width = this.Height = 0; this._start = this.getTransformedScreenPt(down); document.removeEventListener('pointermove', this.onSelectMove); @@ -241,15 +242,15 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP @action onMove = (pt: number[]) => { const movLoc = this.getTransformedScreenPt(pt); - this._width = movLoc.x - this._start.x; - this._height = movLoc.y - this._start.y; + this.Width = movLoc.x - this._start.x; + this.Height = movLoc.y - this._start.y; }; @action onSelectMove = (e: PointerEvent) => { const movLoc = this.getTransformedScreenPt([e.clientX, e.clientY]); - this._width = movLoc.x - this._start.x; - this._height = movLoc.y - this._start.y; + this.Width = movLoc.x - this._start.x; + this.Height = movLoc.y - this._start.y; // e.stopPropagation(); // overlay documents are all 'active', yet they can be dragged. if we stop propagation, then they can be marqueed but not dragged. if we don't stop, then they will be marqueed and dragged, but the marquee will be zero width since the doc will move along with the cursor. }; @@ -280,11 +281,11 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP AnchorMenu.Instance.jumpTo(x, y); } this.props.finishMarquee(this.isEmpty ? x : undefined, this.isEmpty ? y : undefined); - this._width = this._height = 0; + this.Width = this.Height = 0; }; get isEmpty() { - return Math.abs(this._width) <= 10 && Math.abs(this._height) <= 10; + return Math.abs(this.Width) <= 10 && Math.abs(this.Height) <= 10; } render() { @@ -294,9 +295,9 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP style={{ left: `${this.left}px`, top: `${this.top}px`, - width: `${Math.abs(this._width)}px`, - height: `${Math.abs(this._height)}px`, - border: `${this._width === 0 ? '' : '2px dashed black'}`, + width: `${Math.abs(this.Width)}px`, + height: `${Math.abs(this.Height)}px`, + border: `${this.Width === 0 ? '' : '2px dashed black'}`, }} /> ); diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx index 28a769896..37b1adeff 100644 --- a/src/client/views/collections/CollectionCardDeckView.tsx +++ b/src/client/views/collections/CollectionCardDeckView.tsx @@ -373,7 +373,7 @@ export class CollectionCardView extends CollectionSubView() { const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; try { const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete); - const response = await gptImageLabel(hrefBase64); + const response = await gptImageLabel(hrefBase64, 'Give three to five labels to describe this image.'); image[DocData].description = response.trim(); return response; // Return the response from gptImageLabel } catch (error) { diff --git a/src/client/views/collections/CollectionCarouselView.scss b/src/client/views/collections/CollectionCarouselView.scss index f115bb40a..b402a7a32 100644 --- a/src/client/views/collections/CollectionCarouselView.scss +++ b/src/client/views/collections/CollectionCarouselView.scss @@ -1,5 +1,7 @@ .collectionCarouselView-outer { height: 100%; + position: relative; + overflow: hidden; .collectionCarouselView-caption { height: 50; display: inline-block; @@ -11,17 +13,26 @@ width: 100%; user-select: none; } + .message { + justify-content: center; + align-items: center; + display: flex; + height: 60%; + z-index: -1; + // margin: 15px; + } } + .carouselView-back, .carouselView-fwd, .carouselView-star, .carouselView-remove, -.carouselView-check { +.carouselView-check, +.carouselView-add { position: absolute; display: flex; - top: 42.5%; width: 30; - height: 15%; + height: 30; align-items: center; border-radius: 5px; justify-content: center; @@ -32,14 +43,21 @@ } } .carouselView-fwd { + top: 42.5%; right: 20; } .carouselView-back { + top: 42.5%; left: 20; } .carouselView-star { top: 0; - right: 20; + left: 0; +} +.carouselView-add { + position: absolute; + bottom: 0; + left: 0; } .carouselView-remove { top: 80%; @@ -49,6 +67,46 @@ top: 80%; right: 52%; } +.carouselView-quiz { + position: absolute; + display: flex; + top: 5px; + right: 8px; + &:hover { + color: white; + } +} + +.carouselView-practice { + position: absolute; + display: flex; + top: 22px; + right: 8px; + &:hover { + color: white; + } +} +.carouselView-starFilter { + position: absolute; + display: flex; + top: 40px; + right: 7px; + &:hover { + color: white; + } +} + +.carouselView-menu { + position: absolute; + display: flex; + top: 2px; + right: 2px; + width: 30; + height: 60; + border-radius: 5px; + color: rgba(255, 255, 255, 0.5); + background: rgba(0, 0, 0, 0.1); +} .carouselView-back:hover, .carouselView-fwd:hover { diff --git a/src/client/views/collections/CollectionCarouselView.tsx b/src/client/views/collections/CollectionCarouselView.tsx index 4bec2d963..ba7c944a0 100644 --- a/src/client/views/collections/CollectionCarouselView.tsx +++ b/src/client/views/collections/CollectionCarouselView.tsx @@ -1,6 +1,7 @@ /* eslint-disable react/jsx-props-no-spreading */ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { computed, makeObservable } from 'mobx'; +import { Tooltip } from '@mui/material'; +import { action, computed, makeObservable, observable, trace } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { StopEvent, returnFalse, returnOne, returnTrue, returnZero } from '../../../ClientUtils'; @@ -8,8 +9,8 @@ import { emptyFunction } from '../../../Utils'; import { Doc, Opt } from '../../../fields/Doc'; import { DocCast, NumCast, ScriptCast, StrCast } from '../../../fields/Types'; import { DocumentType } from '../../documents/DocumentTypes'; +import { Docs } from '../../documents/Documents'; import { DragManager } from '../../util/DragManager'; -import { ContextMenu } from '../ContextMenu'; import { StyleProp } from '../StyleProp'; import { DocumentView } from '../nodes/DocumentView'; import { FieldViewProps } from '../nodes/FieldView'; @@ -18,9 +19,15 @@ import './CollectionCarouselView.scss'; import { CollectionSubView, SubCollectionViewProps } from './CollectionSubView'; enum cardMode { - PRACTICE = 'practice', + // PRACTICE = 'practice', STAR = 'star', + // QUIZ = 'quiz', + ALL = 'all', +} +enum practiceMode { + PRACTICE = 'practice', QUIZ = 'quiz', + NORMAL = 'normal', } enum practiceVal { MISSED = 'missed', @@ -29,12 +36,22 @@ enum practiceVal { @observer export class CollectionCarouselView extends CollectionSubView() { private _dropDisposer?: DragManager.DragDropDisposer; + @observable private _practiceMessage: string | undefined; + @observable private _filterMessage: string | undefined; get practiceField() { return this.fieldKey + "_practice"; } // prettier-ignore - get starField() { return this.fieldKey + "_star"; } // prettier-ignore + get sideField() { return "_" + this.fieldKey + "_usePath"; } // prettier-ignore + get starField() { return "star"; } // prettier-ignore constructor(props: SubCollectionViewProps) { super(props); makeObservable(this); + // this.setModes(); + this.layoutDoc.filterOp = cardMode.ALL; + Doc.setDocFilter(this.Document, 'star', undefined, 'match'); + this.layoutDoc.practiceMode = practiceMode.NORMAL; + this.layoutDoc._carousel_index = 0; + this.carouselItems.forEach(item => { item.layout[this.practiceField] = undefined}); //prettier-ignore + console.log(this.carouselItems.length); } componentWillUnmount() { @@ -49,16 +66,33 @@ export class CollectionCarouselView extends CollectionSubView() { }; @computed get carouselItems() { + this.childLayoutPairs.map(pair => { + pair.layout.embedContainer = this.Document; + }); return this.childLayoutPairs.filter(pair => pair.layout.type !== DocumentType.LINK); } @computed get marginX() { return NumCast(this.layoutDoc.caption_xMargin, 50); } + @action setPracticeMessage = (mes: string | undefined) => { + this._practiceMessage = mes; + }; + @action setFilterMessage = (mes: string | undefined) => { + this._filterMessage = mes; + }; + + setModes = () => { + this.layoutDoc.filterOp = cardMode.ALL; + Doc.setDocFilter(this.Document, 'data_star', undefined, 'match'); + this.layoutDoc.practiceMode = practiceMode.NORMAL; + this.layoutDoc._carousel_index = 0; + }; + move = (dir: number) => { const moveToCardWithField = (match: (doc: Doc) => boolean): boolean => { let startInd = (NumCast(this.layoutDoc._carousel_index) + dir) % this.carouselItems.length; - while (!match(this.carouselItems?.[startInd].layout) && (startInd + dir + this.carouselItems.length) % this.carouselItems.length !== this.layoutDoc._carousel_index) { + while (!match(this.carouselItems?.[startInd].layout) && (startInd + this.carouselItems.length) % this.carouselItems.length !== this.layoutDoc._carousel_index) { startInd = (startInd + dir + this.carouselItems.length) % this.carouselItems.length; } if (match(this.carouselItems?.[startInd].layout)) { @@ -67,23 +101,28 @@ export class CollectionCarouselView extends CollectionSubView() { } return match(this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)].layout); }; - switch (StrCast(this.layoutDoc.filterOp)) { - case cardMode.STAR: // go to a flashcard that is starred, skip the ones that aren't + + switch (this.layoutDoc.practiceMode && this.layoutDoc.filterOp) { + case practiceMode.PRACTICE && cardMode.ALL: + if (!moveToCardWithField((doc: Doc) => doc[this.practiceField] !== practiceVal.CORRECT)) { + this._practiceMessage = 'Finished! Unselect practice mode to view all flashcards.'; + this.carouselItems.forEach(item => { item.layout[this.practiceField] = undefined}); //prettier-ignore + } + break; + case !practiceMode.PRACTICE && cardMode.STAR: if (!moveToCardWithField((doc: Doc) => !!doc[this.starField])) { - this.layoutDoc.filterOp = undefined; // if there aren't any starred, show all cards + this._filterMessage = 'No starred items. Unselect this view to see all flashcards and star them.'; } break; - case cardMode.PRACTICE: // go to a new index that is missed, skip the ones that are correct - if (!moveToCardWithField((doc: Doc) => doc[this.practiceField] !== practiceVal.CORRECT)) { - this.layoutDoc.filterOp = undefined; // if all of the cards are correct, show all cards and exit practice mode - - this.carouselItems.forEach(item => { // reset all the practice values - item.layout[this.practiceField] = undefined; - }); + case practiceMode.PRACTICE && cardMode.STAR: + if (!moveToCardWithField((doc: Doc) => doc[this.practiceField] !== practiceVal.CORRECT && doc[this.starField] === true)) { + this._filterMessage = 'No flashcards to show! Unselect mode to view all flashcards.'; + this._practiceMessage = undefined; } break; - default: moveToCardWithField(returnTrue); - } // prettier-ignore + default: + moveToCardWithField(returnTrue); + } }; /** @@ -109,6 +148,8 @@ export class CollectionCarouselView extends CollectionSubView() { e.stopPropagation(); const curDoc = this.carouselItems[NumCast(this.layoutDoc._carousel_index)]; curDoc.layout[this.starField] = curDoc.layout[this.starField] ? undefined : true; + // if (!curDoc.layout[this.starField]) this.move(1); + // this.layoutDoc._carousel_index = undefined; }; /* @@ -130,22 +171,43 @@ export class CollectionCarouselView extends CollectionSubView() { onContentDoubleClick = () => ScriptCast(this.layoutDoc.onChildDoubleClick); onContentClick = () => ScriptCast(this.layoutDoc.onChildClick); captionWidth = () => this._props.PanelWidth() - 2 * this.marginX; - specificMenu = (): void => { - const cm = ContextMenu.Instance; - - const revealOptions = cm.findByDescription('Filter Flashcards'); - const revealItems = revealOptions?.subitems ?? []; - revealItems.push({description: 'All', event: () => {this.layoutDoc.filterOp = undefined;}, icon: 'layer-group',}); // prettier-ignore - revealItems.push({description: 'Star', event: () => {this.layoutDoc.filterOp = cardMode.STAR;}, icon: 'star',}); // prettier-ignore - revealItems.push({description: 'Practice Mode', event: () => {this.layoutDoc.filterOp = cardMode.PRACTICE;}, icon: 'check',}); // prettier-ignore - revealItems.push({description: 'Quiz Cards', event: () => {this.layoutDoc.filterOp = cardMode.QUIZ;}, icon: 'pencil',}); // prettier-ignore - !revealOptions && cm.addItem({ description: 'Filter Flashcards', addDivider: false, noexpand: true, subitems: revealItems, icon: 'layer-group' }); + + setPracticeMode = (mode: practiceMode) => { + this.layoutDoc.practiceMode = mode; + this.carouselItems?.map(doc => (doc.layout[this.practiceField] = undefined)); + switch (mode) { + case practiceMode.QUIZ: + this.carouselItems?.map(doc => (doc.layout[this.sideField] = undefined)); + break; + case practiceMode.NORMAL: + this.setPracticeMessage(undefined); + break; + } }; + + setFilterMode = (mode: cardMode) => { + this.layoutDoc.filterOp = mode; + switch (mode) { + case cardMode.STAR: + // Doc.setDocFilter(this.Document, 'data_star', true, 'match'); + this.move(1); + break; + default: + this.setFilterMessage(undefined); // prettier-ignore + // Doc.setDocFilter(this.Document, 'data_star', true, 'remove'); + } + }; + @computed get content() { + trace(); + if (this.layoutDoc._carousel_index === this.carouselItems.length && this.layoutDoc._carousel_index !== 0) { + this.move(1); + } const index = NumCast(this.layoutDoc._carousel_index); const curDoc = this.carouselItems?.[index]; const captionProps = { ...this._props, NativeScaling: returnOne, PanelWidth: this.captionWidth, fieldKey: 'caption', setHeight: undefined, setContentView: undefined }; const carouselShowsCaptions = StrCast(this.layoutDoc._layout_showCaption); + return !(curDoc?.layout instanceof Doc) ? null : ( <> <div className="collectionCarouselView-image" key="image"> @@ -155,10 +217,12 @@ export class CollectionCarouselView extends CollectionSubView() { NativeHeight={returnZero} fitWidth={undefined} setContentViewBox={undefined} + childFilters={this.childDocFilters} onDoubleClickScript={this.onContentDoubleClick} onClickScript={this.onContentClick} isDocumentActive={this._props.childDocumentsActive?.() ? this._props.isDocumentActive : this._props.isContentActive} isContentActive={(this._props.childContentsActive ?? this._props.isContentActive() === false) ? returnFalse : emptyFunction} + addDocument={this._props.addDocument} hideCaptions={!!carouselShowsCaptions} // hide captions if the carousel is configured to show the captions renderDepth={this._props.renderDepth + 1} LayoutTemplate={this._props.childLayoutTemplate} @@ -185,6 +249,25 @@ export class CollectionCarouselView extends CollectionSubView() { </> ); } + + containsDifTypes = (): boolean => { + return this.carouselItems.filter(doc => !doc.layout._layout_isFlashcard).length !== 0; + }; + + addFlashcard() { + const newDoc = Docs.Create.ComparisonDocument('', { _layout_isFlashcard: true, _width: 300, _height: 300 }); + this.addDocument?.(newDoc); + // DocUtils.copyDragFactory(newDoc); + // this._props.addDocument?.(); + // newDoc.layout = this.layoutDoc; + // newDoc.data = this.dataDoc; + // Doc.AddDocToList() + // this._props.parent._props.addDocument(); + // this.childLayoutPairs.push({ newDoc.layout, newDoc.data}); + // this._props.addDocument?.(newDoc); + // console.log('HERE'); + } + @computed get buttons() { if (!this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)]) return null; return ( @@ -195,54 +278,108 @@ export class CollectionCarouselView extends CollectionSubView() { <div key="fwd" className="carouselView-fwd" onClick={this.advance}> <FontAwesomeIcon icon="chevron-right" size="2x" /> </div> - <div key="star" className="carouselView-star" onClick={this.star}> - <FontAwesomeIcon icon="star" color={this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)].layout[this.starField] ? 'yellow' : 'gray'} size="1x" /> - </div> - <div key="remove" className="carouselView-remove" onClick={e => this.setPracticeVal(e, practiceVal.MISSED)} style={{ visibility: this.layoutDoc.filterOp === cardMode.PRACTICE ? 'visible' : 'hidden' }}> - <FontAwesomeIcon icon="xmark" color="red" size="1x" /> - </div> - <div key="check" className="carouselView-check" onClick={e => this.setPracticeVal(e, practiceVal.CORRECT)} style={{ visibility: this.layoutDoc.filterOp === cardMode.PRACTICE ? 'visible' : 'hidden' }}> - <FontAwesomeIcon icon="check" color="green" size="1x" /> - </div> + {!this.containsDifTypes() ? ( + <div> + <Tooltip title="star"> + <div key="star" className="carouselView-star" onClick={this.star}> + <FontAwesomeIcon icon="star" color={this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)].layout[this.starField] ? 'yellow' : 'gray'} size="1x" /> + </div> + </Tooltip> + {/* <Tooltip title="add new flashcard to pile"> + <div key="add" className="carouselView-add" onClick={this.addFlashcard}> + <FontAwesomeIcon icon="plus" color={this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)].layout[this.starField] ? 'yellow' : 'gray'} size="1x" /> + </div> + </Tooltip> */} + </div> + ) : null} + {this.layoutDoc.practiceMode === practiceMode.PRACTICE ? ( + <div> + <Tooltip title="Incorrect. View again later."> + <div key="remove" className="carouselView-remove" onClick={e => this.setPracticeVal(e, practiceVal.MISSED)}> + <FontAwesomeIcon icon="xmark" color="red" size="1x" /> + </div> + </Tooltip> + <Tooltip title="Correct"> + <div key="check" className="carouselView-check" onClick={e => this.setPracticeVal(e, practiceVal.CORRECT)}> + <FontAwesomeIcon icon="check" color="green" size="1x" /> + </div> + </Tooltip> + </div> + ) : null} </> ); } + togglePracticeMode = (mode: practiceMode) => { + if (mode === this.layoutDoc.practiceMode) { + this.setPracticeMode(practiceMode.NORMAL); + // this.setPracticeMessage("undefined"); + } else this.setPracticeMode(mode); + }; + toggleFilterMode = () => { this.layoutDoc.filterOp === cardMode.STAR ? this.setFilterMode(cardMode.ALL) : this.setFilterMode(cardMode.STAR)}; //prettier-ignore + setColor = (mode: practiceMode | cardMode, which: string) => { return which === mode ? 'white' : 'light gray'}; //prettier-ignore + + @computed get menu() { + return ( + <div className="carouselView-menu"> + <Tooltip title="Practice flashcards using GPT"> + <div key="back" className="carouselView-quiz" onClick={e => this.togglePracticeMode(practiceMode.QUIZ)}> + <FontAwesomeIcon icon="file-pen" color={this.setColor(practiceMode.QUIZ, StrCast(this.layoutDoc.practiceMode))} size="1x" /> + </div> + </Tooltip> + <Tooltip title={this.layoutDoc.practiceMode === practiceMode.PRACTICE ? 'Exit practice mode' : 'Practice flashcards manually'}> + <div key="back" className="carouselView-practice" onClick={e => this.togglePracticeMode(practiceMode.PRACTICE)}> + <FontAwesomeIcon icon="check" color={this.setColor(practiceMode.PRACTICE, StrCast(this.layoutDoc.practiceMode))} size="1x" /> + </div> + </Tooltip> + <Tooltip title={this.layoutDoc.filterOp === cardMode.STAR ? 'Show all cards' : 'Show only starred cards'}> + <div key="back" className="carouselView-starFilter" onClick={e => this.toggleFilterMode()}> + <FontAwesomeIcon icon="filter" color={this.setColor(cardMode.STAR, StrCast(this.layoutDoc.filterOp))} size="1x" /> + </div> + </Tooltip> + </div> + ); + } + render() { return ( <div className="collectionCarouselView-outer" ref={this.createDashEventsTarget} - onContextMenu={this.specificMenu} style={{ background: this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.BackgroundColor) as string, color: this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.Color) as string, }}> - {this.content} - {/* Displays a message to the user to add more flashcards if they are in practice mode and no flashcards are there. */} - <p - style={{ - display: !this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)] && this.layoutDoc.filterOp === cardMode.PRACTICE ? 'flex' : 'none', - justifyContent: 'center', - alignItems: 'center', - height: '100%', - zIndex: '-1', - }}> - Add flashcards! - </p> - {/* Displays a message to the user that a flashcard was recently missed if they had previously gotten it wrong. */} + {!this._practiceMessage && !this._filterMessage ? ( + this.content + ) : ( + <p className="message"> + {this._filterMessage} + {'\n'} + {this._practiceMessage} + </p> + )} + {!this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)] && this.layoutDoc.practiceMode === practiceMode.PRACTICE ? <p className="message">Add flashcards </p> : null} <p + className="missed-message" style={{ color: 'red', + fontWeight: 'bold', zIndex: '999', position: 'relative', left: '10px', top: '10px', - display: this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)] ? (this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)].layout[this.practiceField] === practiceVal.MISSED ? 'block' : 'none') : 'none', + width: '10px', + display: this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)] + ? this.carouselItems?.[NumCast(this.layoutDoc._carousel_index)].layout[this.practiceField] === practiceVal.MISSED && this.layoutDoc.practiceMode === practiceMode.PRACTICE && !this._practiceMessage + ? 'block' + : 'none' + : 'none', }}> Recently missed! </p> - {this.Document._chromeHidden ? null : this.buttons} + {!this.containsDifTypes() && this.carouselItems.length !== 0 ? this.menu : null} + {this.Document._chromeHidden || (!this._filterMessage && !this._practiceMessage) ? this.buttons : null} </div> ); } diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index ccb6bc9be..4f8ed52f0 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -2,13 +2,13 @@ import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect } from '../../../../Utils'; +import { intersectRect, numberRange } from '../../../../Utils'; import { Doc, DocListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; -import { Cast, NumCast, StrCast } from '../../../../fields/Types'; +import { Cast, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; import { DocUtils } from '../../../documents/DocUtils'; @@ -30,6 +30,9 @@ import { SubCollectionViewProps } from '../CollectionSubView'; import { ImageLabelBoxData } from './ImageLabelBox'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; +import { ImageLabelHandler } from './ImageLabelHandler'; +import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; +import { CollectionCardView } from '../CollectionCardDeckView'; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -433,13 +436,37 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps * Classifies images and assigns the labels as document fields. */ @undoBatch - classifyImages = action(async () => { - const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); - if (groupButton) { - this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); - ImageLabelBoxData.Instance.setData(this._selectedDocs); - MainView.Instance.expandFlyout(groupButton); + classifyImages = action(async (e: React.MouseEvent | undefined) => { + this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + + const imageInfos = this._selectedDocs.map(async doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + !hrefBase64 ? undefined : + gptImageLabel(hrefBase64, 'Give three to five labels to describe this image.').then(labels => + Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => + ({ doc, embeddings, labels }))) ); // prettier-ignore + }); + + (await Promise.all(imageInfos)).forEach(imageInfo => { + if (imageInfo && Array.isArray(imageInfo.embeddings)) { + imageInfo.doc[DocData].data_labels = imageInfo.labels; + numberRange(3).forEach(n => { + imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); + }); + } + }); + + if (e) { + ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); } + + // const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); + // if (groupButton) { + // this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + // ImageLabelBoxData.Instance.setData(this._selectedDocs); + // MainView.Instance.expandFlyout(groupButton); + // } }); /** diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx index 59349da8b..63a126aec 100644 --- a/src/client/views/nodes/AudioBox.tsx +++ b/src/client/views/nodes/AudioBox.tsx @@ -8,7 +8,7 @@ import { DateField } from '../../../fields/DateField'; import { Doc, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { ComputedField } from '../../../fields/ScriptField'; -import { Cast, DateCast, NumCast } from '../../../fields/Types'; +import { Cast, DateCast, DocCast, NumCast } from '../../../fields/Types'; import { AudioField, nullAudio } from '../../../fields/URLField'; import { formatTime } from '../../../Utils'; import { Docs } from '../../documents/Documents'; @@ -27,6 +27,7 @@ import './AudioBox.scss'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { OpenWhere } from './OpenWhere'; +import axios from 'axios'; /** * AudioBox @@ -257,6 +258,8 @@ export class AudioBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const [{ result }] = await Networking.UploadFilesToServer({ file: file as Blob & { name: string; lastModified: number; webkitRelativePath: string } }); if (!(result instanceof Error)) { this.Document[this.fieldKey] = new AudioField(result.accessPaths.agnostic.client); + this.Document.url = result.accessPaths.agnostic.client; + await this.pushInfo(); } }; this._recordStart = new Date().getTime(); @@ -284,9 +287,23 @@ export class AudioBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } }; + pushInfo = async () => { + const audio = { + file: this.path, + }; + const response = await axios.post('http://localhost:105/recognize/', audio, { + headers: { + 'Content-Type': 'application/json', + }, + }); + this.Document[DocData].phoneticTranscription = response.data['transcription']; + console.log('RESPONSE: ' + response.data['transcription']); + }; + // context menu specificContextMenu = (): void => { const funcs: ContextMenuProps[] = []; + // funcs.push({ description: 'Push info', event: this.pushInfo, icon: 'redo-alt' }); funcs.push({ description: (this.layoutDoc.hideAnchors ? "Don't hide" : 'Hide') + ' anchors', event: () => { this.layoutDoc.hideAnchors = !this.layoutDoc.hideAnchors; }, // prettier-ignore diff --git a/src/client/views/nodes/ComparisonBox.scss b/src/client/views/nodes/ComparisonBox.scss index 08d9e6010..da1d352f2 100644 --- a/src/client/views/nodes/ComparisonBox.scss +++ b/src/client/views/nodes/ComparisonBox.scss @@ -5,42 +5,136 @@ width: 100%; height: 100%; position: relative; + background: gray; z-index: 0; pointer-events: none; display: flex; p { + // bcz: what is this styling for? if text in the comparison box is colored, then this causes it to render with a black outline color: rgb(0, 0, 0); -webkit-text-stroke-color: black; -webkit-text-stroke-width: 0.2px; } - .input-box { - position: relative; + position: absolute; + top: 50; padding: 10px; width: 100%; - height: 100%; + height: 70%; display: flex; } .submit-button { - position: relative; + position: absolute; padding-bottom: 10px; + padding-top: 5px; padding-left: 5px; padding-right: 5px; - width: 100%; - height: 15%; - display: flex; + // width: 80%; + border-radius: 2px; + height: 17%; + display: inline-block; + bottom: 0; + // right: 0; + + &.schema-header-button { + color: gray; + margin: 3px; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + + svg { + width: 15px; + } + } + + &.pronunciation { + width: 40%; + align-items: center; + justify-content: center; + } + &.submit { + width: 40%; + // float: right; + + // position: absolute; + // position: 10px; + // padding-left: 35%; + // padding-right: 80%; + // // width: 80px; + // // right: 0; + // right: 0; + // bottom: 0; + } + &.record { + width: 20%; + float: left; + border-radius: 2px; + // right: 0; + // height: 30%; + } button { - flex: 1; - position: relative; + // flex: 1; + // position: relative; } } + + .dropbtn { + background-color: #3498db; + color: white; + padding: 16px; + font-size: 16px; + border: none; + } + + .dropup { + position: absolute; + display: inline-block; + margin-top: 150px; + bottom: 0; + } + + .dropup-content { + display: none; + position: absolute; + background-color: #f1f1f1; + min-width: 160px; + bottom: 40px; + z-index: 1000; + } + + .dropup-content a { + color: black; + padding: 12px 16px; + text-decoration: none; + display: block; + } + + .dropup-content a:hover { + background-color: #ccc; + } + + .dropup:hover .dropup-content { + display: block; + } + + .dropup:hover .dropbtn { + background-color: #2980b9; + } + textarea { flex: 1; padding: 10px; - position: relative; + // position: relative; resize: none; + position: 'absolute'; + width: '91%'; + height: '80%'; + z-index: '-1'; + overscroll-behavior: contain; } .clip-div { @@ -117,6 +211,33 @@ opacity: 0.5; } } + + .loading-spinner { + display: flex; + justify-content: center; + align-items: center; + height: 90%; + width: 93%; + font-size: 20px; + font-weight: bold; + color: #0b0a0a; + } + + @keyframes spin { + to { + transform: rotate(360deg); + } + } +} + +.comparisonBox-explain { + position: absolute; + top: 10px; + left: 10px; + z-index: 200; + // padding: 5px; + background: #dfdfdf; + pointer-events: none; } .comparisonBox-interactive { @@ -128,8 +249,9 @@ display: flex; } } + // .input-box { - // position: relative; + // position: absolute; // padding: 10px; // } // input[type='text'] { @@ -216,24 +338,5 @@ } } } - - .loading-circle { - position: relative; - width: 50px; - height: 50px; - border-radius: 50%; - border: 3px solid #ccc; - border-top-color: #333; - animation: spin 1s infinite linear; - } - - @keyframes spin { - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(360deg); - } - } } } diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx index 1eae163df..58af3ad52 100644 --- a/src/client/views/nodes/ComparisonBox.tsx +++ b/src/client/views/nodes/ComparisonBox.tsx @@ -1,15 +1,17 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { MathJax, MathJaxContext } from 'better-react-mathjax'; import { Tooltip } from '@mui/material'; -import { action, computed, makeObservable, observable } from 'mobx'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; -import { returnFalse, returnNone, setupMoveUpEvents } from '../../../ClientUtils'; +import { returnFalse, returnNone, returnTrue, setupMoveUpEvents } from '../../../ClientUtils'; import { emptyFunction } from '../../../Utils'; import { Doc, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { RichTextField } from '../../../fields/RichTextField'; import { DocCast, NumCast, RTFCast, StrCast, toList } from '../../../fields/Types'; -import { GPTCallType, gptAPICall } from '../../apis/gpt/GPT'; +import { GPTCallType, gptAPICall, gptImageLabel } from '../../apis/gpt/GPT'; +import '../pdf/GPTPopup/GPTPopup.scss'; import { DocUtils } from '../../documents/DocUtils'; import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; @@ -23,6 +25,19 @@ import './ComparisonBox.scss'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { FormattedTextBox } from './formattedText/FormattedTextBox'; +import ReactLoading from 'react-loading'; +import { ContextMenu } from '../ContextMenu'; +import { ContextMenuProps } from '../ContextMenuItem'; +import { tickStep } from 'd3'; +import { CollectionCarouselView } from '../collections/CollectionCarouselView'; +import { FollowLinkScript } from '../../documents/DocUtils'; +import { schema } from '../nodes/formattedText/schema_rts'; +import { Id } from '../../../fields/FieldSymbols'; +import axios from 'axios'; +import ReactMarkdown from 'react-markdown'; +import { WebField, nullAudio } from '../../../fields/URLField'; + +const API_URL = 'https://api.unsplash.com/search/photos'; @observer export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @@ -30,21 +45,156 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() return FieldView.LayoutString(ComparisonBox, fieldKey); } private _disposers: (DragManager.DragDropDisposer | undefined)[] = [undefined, undefined]; - private _closeRef = React.createRef<HTMLDivElement>(); - @observable _inputValue = ''; - @observable _outputValue = ''; - @observable _loading = false; - @observable _errorMessage = ''; - @observable _outputMessage = ''; - @observable _animating = ''; - constructor(props: FieldViewProps) { super(props); makeObservable(this); + this.setListening(); + } + + @observable private _inputValue = ''; + @observable private _outputValue = ''; + @observable private _loading = false; + @observable private _isEmpty = false; + @observable private _audio: Doc = Docs.Create.TextDocument(''); + @observable childActive = false; + @observable _yRelativeToTop: boolean = true; + @observable _animating = ''; + @observable mathJaxConfig = { + loader: { load: ['input/asciimath'] }, + }; + @observable private _listening = false; + @observable transcriptElement = ''; + @observable private frontSide = false; + SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + @observable recognition = new this.SpeechRecognition(); + _closeRef = React.createRef<HTMLDivElement>(); + + get revealOp() { + return this.layoutDoc[`_${this._props.fieldKey}_revealOp`]; + } + get clipHeightKey() { + return '_' + this._props.fieldKey + '_clipHeight'; + } + + get clipWidthKey() { + return '_' + this._props.fieldKey + '_clipWidth'; + } + + @computed get clipWidth() { + return NumCast(this.layoutDoc[this.clipWidthKey], 50); + } + + @computed get clipHeight() { + return NumCast(this.layoutDoc[this.clipHeightKey], 200); } + @computed get overlayAlternateIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">flip</div>}> + <div + className="formattedTextBox-alternateButton" + onPointerDown={e => + setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => { + if (!this.revealOp || this.revealOp === 'flip') { + this.flipFlashcard(); + } + }) + } + style={{ + background: this.revealOp === 'hover' ? 'gray' : this.frontSide ? 'white' : 'black', + color: this.revealOp === 'hover' ? 'black' : this.frontSide ? 'black' : 'white', + display: 'inline-block', + }}> + <div key="alternate" className="formattedTextBox-flip"> + <FontAwesomeIcon icon="turn-up" size="1x" /> + </div> + </div> + </Tooltip> + ); + } + + @computed get flashcardMenu() { + return ( + <div> + <Tooltip + title={this.frontSide ? <div className="dash-tooltip">Flip to front side to use GPT</div> : <div className="dash-tooltip">Ask GPT to create an answer on the back side of the flashcard based on your question on the front</div>}> + <div style={{ position: 'absolute', bottom: '3px', right: '50px', cursor: 'pointer' }} onPointerDown={e => (!this.frontSide ? this.findImageTags() : null)}> + <FontAwesomeIcon icon="lightbulb" size="xl" /> + </div> + </Tooltip> + {DocCast(this.Document.embedContainer).type_collection === 'carousel' ? null : ( + <div> + <Tooltip title={<div>Create a flashcard pile</div>}> + <div style={{ position: 'absolute', bottom: '3px', right: '74px', cursor: 'pointer' }} onPointerDown={e => this.createFlashcardPile([this.Document], false)}> + <FontAwesomeIcon icon="folder-plus" size="xl" /> + </div> + </Tooltip> + <Tooltip title={<div className="dash-tooltip">Create new flashcard stack based on text</div>}> + <div style={{ position: 'absolute', bottom: '3px', right: '104px', cursor: 'pointer' }} onClick={e => this.gptFlashcardPile()}> + <FontAwesomeIcon icon="layer-group" size="xl" /> + </div> + </Tooltip> + </div> + )} + <Tooltip title={<div className="dash-tooltip">Hover to reveal</div>}> + <div style={{ position: 'absolute', bottom: '3px', right: '25px', cursor: 'pointer' }} onClick={e => this.handleHover()}> + <FontAwesomeIcon color={this.revealOp === 'hover' ? 'blue' : 'black'} icon="hand-point-up" size="xl" /> + </div> + </Tooltip> + </div> + ); + } + + @action handleHover = () => { + if (this.revealOp === 'hover') { + this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'flip'; + this.Document.forceActive = false; + } else { + this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'hover'; + this.Document.forceActive = true; + } + }; + + @action handleInputChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => { + this._inputValue = e.target.value; + console.log(this._inputValue); + }; + + @action activateContent = () => { + this.childActive = true; + }; + + @action handleRenderClick = () => { + this.frontSide = !this.frontSide; + }; + + @action handleRenderGPTClick = async () => { + console.log('Phonetic transcription: ' + DocCast(this.Document.audio).phoneticTranscription); + const phonTrans = DocCast(this.Document.audio).phoneticTranscription; + if (phonTrans) { + this._inputValue = StrCast(phonTrans); + console.log('INPUT:' + this._inputValue); + this.askGPTPhonemes(this._inputValue); + } else if (this._inputValue) this.askGPT(GPTCallType.QUIZ); + this.frontSide = false; + this._outputValue = ''; + }; + + @action + private onPointerMove = ({ movementX }: PointerEvent) => { + const width = movementX * this.ScreenToLocalBoxXf().Scale + (this.clipWidth / 100) * this._props.PanelWidth(); + if (width && width > 5 && width < this._props.PanelWidth()) { + this.layoutDoc[this.clipWidthKey] = (width * 100) / this._props.PanelWidth(); + } + return false; + }; + componentDidMount() { this._props.setContentViewBox?.(this); + reaction( + () => this._props.isSelected(), // when this reaction should update + selected => !selected && (this.childActive = false) // what it should update to + ); } protected createDropTarget = (ele: HTMLDivElement | null, fieldKey: string, disposerId: number) => { @@ -54,62 +204,19 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() } }; - @computed get revealOp() { return this.layoutDoc[`_${this.fieldKey}_revealOp`] as ('flip'|'hover'|undefined); } // prettier-ignore - @computed get clipWidth() { return NumCast(this.layoutDoc[`_${this.fieldKey}_clipWidth`], 50); } // prettier-ignore - set clipWidth(width: number) { this.layoutDoc[`_${this.fieldKey}_clipWidth`] = width; } // prettier-ignore - @computed get useAlternate() { return this.layoutDoc[`_${this.fieldKey}_usePath`] === 'alternate'; } // prettier-ignore - set useAlternate(alt: boolean) { this.layoutDoc[`_${this.fieldKey}_usePath`] = alt ? 'alternate' : undefined; } // prettier-ignore - - animateClipWidth = action((clipWidth: number, duration = 200 /* ms */) => { - this._animating = `all ${duration}ms`; // turn on clip animation transition, then turn it off at end of animation - setTimeout(action(() => { this._animating = ''; }), duration); // prettier-ignore - this.clipWidth = clipWidth; - }); - - internalDrop = undoable((e: Event, dropEvent: DragManager.DropEvent, fieldKey: string) => { + private internalDrop = undoable((e: Event, dropEvent: DragManager.DropEvent, fieldKey: string) => { if (dropEvent.complete.docDragData) { const { droppedDocuments } = dropEvent.complete.docDragData; const added = dropEvent.complete.docDragData.moveDocument?.(droppedDocuments, this.Document, (doc: Doc | Doc[]) => this.addDoc(toList(doc).lastElement(), fieldKey)); Doc.SetContainer(droppedDocuments.lastElement(), this.dataDoc); !added && e.preventDefault(); e.stopPropagation(); // prevent parent Doc from registering new position so that it snaps back into place + // this.childActive = false; return added; } return undefined; }, 'internal drop'); - registerSliding = (e: React.PointerEvent<HTMLDivElement>, targetWidth: number) => { - if (e.button !== 2) { - setupMoveUpEvents( - this, - e, - this.onPointerMove, - emptyFunction, - action((clickEv, doubleTap) => { - if (doubleTap) { - this._isAnyChildContentActive = true; - if (!this.dataDoc[this.fieldKey + '_1'] && !this.dataDoc[this.fieldKey]) this.dataDoc[this.fieldKey + '_1'] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); - if (!this.dataDoc[this.fieldKey + '_2'] && !this.dataDoc[this.fieldKey + '_alternate']) this.dataDoc[this.fieldKey + '_2'] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); - // DocumentView.addViewRenderedCb(DocCast(this.dataDoc[this.fieldKey + '_1']), dv => { - // dv?.select(false); - // }); - } - }), - true, - undefined, - () => !this._isAnyChildContentActive && this.animateClipWidth((targetWidth * 100) / this._props.PanelWidth()) - ); - } - }; - - onPointerMove = ({ movementX }: PointerEvent) => { - const width = movementX * this.ScreenToLocalBoxXf().Scale + (this.clipWidth / 100) * this._props.PanelWidth(); - if (width > 5 && width < this._props.PanelWidth()) { - this.clipWidth = (width * 100) / this._props.PanelWidth(); - } - return false; - }; - getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => { const anchor = Docs.Create.ConfigDocument({ title: 'CompareAnchor:' + this.Document.title, @@ -127,23 +234,25 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() }; clearDoc = undoable((fieldKey: string) => { - delete this.dataDoc[fieldKey]; - this.dataDoc[fieldKey] = 'empty'; + this.dataDoc[fieldKey] = undefined; + this._isEmpty = true; }, 'clear doc'); - // clearDoc = (fieldKey: string) => delete this.dataDoc[fieldKey]; moveDoc = (doc: Doc, addDocument: (document: Doc | Doc[]) => boolean, which: string) => this.remDoc(doc, which) && addDocument(doc); addDoc = (doc: Doc, which: string) => { - if (this.dataDoc[which] && this.dataDoc[which] !== 'empty') return false; + if (this.dataDoc[which] && !this._isEmpty) return false; this.dataDoc[which] = doc; return true; }; remDoc = (doc: Doc, which: string) => { if (this.dataDoc[which] === doc) { + this._isEmpty = true; // this.dataDoc[which] = 'empty'; + console.log('HEREEEE'); this.dataDoc[which] = undefined; return true; } + console.log('FALSE'); return false; }; @@ -173,123 +282,377 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() remDoc1 = (docs: Doc | Doc[]) => toList(docs).reduce((res, doc) => res && this.remDoc(doc, this.fieldKey + '_1'), true); remDoc2 = (docs: Doc | Doc[]) => toList(docs).reduce((res, doc) => res && this.remDoc(doc, this.fieldKey + '_2'), true); - /** - * Tests for whether a comparison box slot (ie, before or after) has renderable text content. - * If it does, render a FormattedTextBox for that slot that references the comparisonBox's slot field - * @param whichSlot field key for start or end slot - * @returns a JSX layout string if a text field is found, othwerise undefined - */ - testForTextFields = (whichSlot: string) => { - const slotData = Doc.Get(this.dataDoc, whichSlot, true); - const slotHasText = slotData instanceof RichTextField || typeof slotData === 'string'; - const subjectText = RTFCast(this.Document[this.fieldKey])?.Text.trim(); - const altText = RTFCast(this.Document[this.fieldKey + '_alternate'])?.Text.trim(); - const layoutTemplateString = - slotHasText ? FormattedTextBox.LayoutString(whichSlot): - whichSlot.endsWith('1') ? (subjectText !== undefined ? FormattedTextBox.LayoutString(this.fieldKey) : undefined) : - altText !== undefined ? FormattedTextBox.LayoutString(this.fieldKey + '_alternate'): undefined; // prettier-ignore + private registerSliding = (e: React.PointerEvent<HTMLDivElement>, targetWidth: number) => { + if (e.button !== 2) { + setupMoveUpEvents( + this, + e, + this.onPointerMove, + emptyFunction, + action((moveEv, doubleTap) => { + if (doubleTap) { + this.childActive = true; + if (!this.dataDoc[this.fieldKey + '_1'] && !this.dataDoc[this.fieldKey]) this.dataDoc[this.fieldKey + '_1'] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); + if (!this.dataDoc[this.fieldKey + '_2'] && !this.dataDoc[this.fieldKey + '_alternate']) this.dataDoc[this.fieldKey + '_2'] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); + } + }), + false, + undefined, + action(() => { + if (this.childActive) return; + this._animating = 'all 200ms'; + // on click, animate slider movement to the targetWidth + this.layoutDoc[this.clipWidthKey] = (targetWidth * 100) / this._props.PanelWidth(); + // this.layoutDoc[this.clipHeightKey] = (targetWidth * 100) / this._props.PanelHeight(); - // A bit hacky to try out the concept of using GPT to fill in flashcards - // If the second slot doesn't have anything in it, but the fieldKey slot has text (e.g., this.text is a string) - // and the fieldKey + "_alternate" has text that includes a GPT query (indicated by (( && )) ) that is parameterized (optionally) by the fieldKey text (this) or other metadata (this.<field>). - // eg., this.text_alternate is - // "((Provide a one sentence definition for (this) that doesn't use any word in (this.excludeWords) ))" - // where (this) is replaced by the text in the fieldKey slot abd this.excludeWords is repalced by the conetnts of the excludeWords field - // The GPT call will put the "answer" in the second slot of the comparison (eg., text_2) - if (whichSlot.endsWith('2') && !layoutTemplateString?.includes(whichSlot)) { - const queryText = altText?.replace('(this)', subjectText); // TODO: this should be done in Doc.setField but it doesn't know about the fieldKey ... - if (queryText?.match(/\(\(.*\)\)/)) { - Doc.SetField(this.Document, whichSlot, ':=' + queryText, false); // make the second slot be a computed field on the data doc that calls ChatGpt - } + setTimeout( + action(() => { + this._animating = ''; + }), + 200 + ); + }) + ); } - return layoutTemplateString; }; - /** - * Flips a flashcard to the alternate side for the user to view. - */ - flipFlashcard = () => { - this.useAlternate = !this.useAlternate; + setListening = () => { + const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + if (SpeechRecognition) { + this.recognition.continuous = true; + this.recognition.interimResults = true; + this.recognition.lang = 'en-US'; + this.recognition.onresult = this.handleResult.bind(this); + } + ContextMenu.Instance.setLangIndex(0); }; - /** - * Changes the view option to hover for a flashcard. - */ - hoverFlip = (alternate: boolean) => { - if (this.revealOp === 'hover') this.useAlternate = alternate; + startListening = () => { + this.recognition.start(); + this._listening = true; }; - /** - * Creates the button used to flip the flashcards. - */ - @computed get overlayAlternateIcon() { - return ( - <Tooltip title={<div className="dash-tooltip">flip</div>}> - <div - className="formattedTextBox-alternateButton" - onPointerDown={e => - setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => { - if (!this.revealOp || this.revealOp === 'flip') { - this.flipFlashcard(); - console.log('Print Front of cards: ' + (RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text ?? '')); - console.log('Print Back of cards: ' + (RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text ?? '')); - } - }) - } - style={{ - background: this.useAlternate ? 'white' : 'black', - color: this.useAlternate ? 'black' : 'white', - }}> - <FontAwesomeIcon icon="turn-up" size="sm" /> - </div> - </Tooltip> - ); - } + stopListening = () => { + this.recognition.stop(); + this._listening = false; + }; - @action handleRenderGPTClick = () => { - // Call the GPT model and get the output - this.useAlternate = true; - this._outputValue = ''; - if (this._inputValue) this.askGPT(); + setLanguage = (language: string, ind: number) => { + this.recognition.lang = language; + ContextMenu.Instance.setLangIndex(ind); }; - @action handleRenderClick = () => { - // Call the GPT model and get the output - this.useAlternate = false; + convertAbr = () => { + switch (this.recognition.lang) { + case 'en-US': return 'English'; //prettier-ignore + case 'es-ES': return 'Spanish'; //prettier-ignore + case 'fr-FR': return 'French'; //prettier-ignore + case 'it-IT': return 'Italian'; //prettier-ignore + case 'zh-CH': return 'Mandarin Chinese'; //prettier-ignore + case 'ja': return 'Japanese'; //prettier-ignore + default: return 'Korean'; //prettier-ignore + } }; - /** - * Calls the GPT model to create QuizCards. Evaluates how similar the user's response is to the alternate - * side of the flashcard. - */ - askGPT = async (): Promise<string | undefined> => { + openContextMenu = (x: number, y: number, evalu: boolean) => { + ContextMenu.Instance.clearItems(); + ContextMenu.Instance.addItem({ description: 'English', event: e => this.setLanguage('en-US', 0), icon: 'question' }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Spanish', event: e => this.setLanguage('es-ES', 1 ), icon: 'question'}); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'French', event: e => this.setLanguage('fr-FR', 2), icon: 'question' }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Italian', event: e => this.setLanguage('it-IT', 3), icon: 'question' }); //prettier-ignore + if (!evalu) ContextMenu.Instance.addItem({ description: 'Mandarin Chinese', event: e => this.setLanguage('zh-CH', 4), icon: 'question' }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Japanese', event: e => this.setLanguage('ja', 5), icon: 'question' }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Korean', event: e => this.setLanguage('ko', 6), icon: 'question' }); //prettier-ignore + ContextMenu.Instance.displayMenu(x, y); + }; + + evaluatePronunciation = () => { + const newAudio = Docs.Create.AudioDocument(nullAudio, { _width: 200, _height: 100 }); + this.Document.audio = newAudio[DocData]; + this._props.DocumentView?.()._props.addDocument?.(newAudio); + }; + + pushInfo = async () => { + const audio = { + file: this._audio.url, + }; + const response = await axios.post('http://localhost:105/recognize/', audio, { + headers: { + 'Content-Type': 'application/json', + }, + }); + this.Document.phoneticTranscription = response.data['transcription']; + console.log('RESPONSE: ' + response.data['transcription']); + }; + + createFlashcardPile(collectionArr: Doc[], gpt: boolean) { + const newCol = Docs.Create.CarouselDocument(collectionArr, { + _width: NumCast(this.layoutDoc['_' + this._props.fieldKey + '_width'], 250) + 50, + _height: NumCast(this.layoutDoc['_' + this._props.fieldKey + '_width'], 200) + 50, + _layout_fitWidth: false, + _layout_autoHeight: true, + }); + newCol['x'] = this.layoutDoc['x']; + newCol['y'] = NumCast(this.layoutDoc['y']) + 50; + newCol.type_collection = 'carousel'; + + if (gpt) { + this._props.DocumentView?.()._props.addDocument?.(newCol); + this._props.removeDocument?.(this.Document); + } else { + this._props.addDocument?.(newCol); + this._props.removeDocument?.(this.Document); + this.Document.embedContainer = newCol; + } + } + + gptFlashcardPile = async () => { + var text = await this.askGPT(GPTCallType.STACK); + var senArr = text?.split('Question: '); + var collectionArr: Doc[] = []; + for (let i = 1; i < senArr?.length!; i++) { + const newDoc = Docs.Create.ComparisonDocument(senArr![i], { _layout_isFlashcard: true, _width: 300, _height: 300 }); + + if (StrCast(senArr![i]).includes('Keyword: ')) { + const question = StrCast(senArr![i]).split('Keyword: '); + const img = await this.fetchImages(question[1]); + const textSide1 = question[0].includes('Answer: ') ? question[0].split('Answer: ')[0] : question[0]; + const textDoc1 = Docs.Create.TextDocument(question[0]); + const rtfiel = new RichTextField( + JSON.stringify({ + doc: { + type: 'doc', + content: [ + { + type: 'paragraph', + attrs: { align: null, color: null, id: null, indent: null, inset: null, lineSpacing: null, paddingBottom: null, paddingTop: null }, + content: [ + { type: 'text', text: question[0].includes('Answer: ') ? question[0].split('Answer: ')[0] : question[0] }, + { type: 'dashDoc', attrs: { width: '200px', height: '200px', title: 'dashDoc', float: 'unset', hidden: false, docId: img![Id] } }, + ], + }, + ], + }, + selection: { type: 'text', anchor: 2, head: 2 }, + }), + textSide1 + ); + + textDoc1[DocData].text = rtfiel; + DocCast(newDoc)[DocData][this.fieldKey + '_1'] = textDoc1; + DocCast(newDoc)[DocData][this.fieldKey + '_0'] = Docs.Create.TextDocument(question[0].includes('Answer: ') ? question[0].split('Answer: ')[1] : question[1]); + } + collectionArr.push(newDoc); + } + this.createFlashcardPile(collectionArr, true); + }; + + askGPT = async (callType: GPTCallType): Promise<string | undefined> => { const questionText = 'Question: ' + StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text); const rubricText = ' Rubric: ' + StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text); const queryText = questionText + ' UserAnswer: ' + this._inputValue + '. ' + rubricText; - + this._loading = true; + const doc = DocCast(this.dataDoc[this.props.fieldKey + '_0']); + if (callType == GPTCallType.CHATCARD) { + if (StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text) === '') { + this._loading = false; + return; + } + this.flipFlashcard(); + } try { - const res = await gptAPICall(queryText, GPTCallType.QUIZ); + console.log(queryText); + const res = await gptAPICall(callType == GPTCallType.QUIZ ? queryText : questionText, callType); if (!res) { console.error('GPT call failed'); return; } - this._outputValue = res; + if (callType == GPTCallType.CHATCARD) { + DocCast(this.dataDoc[this.props.fieldKey + '_0'])[DocData].text = res; + } else if (callType == GPTCallType.QUIZ) { + console.log(this._inputValue); + this._outputValue = res.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); + } else if (callType === GPTCallType.FLASHCARD) { + this._loading = false; + return res; + } else if (callType === GPTCallType.STACK) { + } + this._loading = false; + return res; } catch (err) { console.error('GPT call failed'); } + this._loading = false; }; layoutWidth = () => NumCast(this.layoutDoc.width, 200); layoutHeight = () => NumCast(this.layoutDoc.height, 200); + findImageTags = async () => { + const c = this.DocumentView?.().ContentDiv!.getElementsByTagName('img'); + if (c?.length === 0) await this.askGPT(GPTCallType.CHATCARD); + if (c) { + this._loading = true; + for (let i of c) { + console.log(i); + if (i.className !== 'ProseMirror-separator') await this.getImageDesc(i.src); + } + this._loading = false; + } + }; + + askGPTPhonemes = async (phonemes: string) => { + const sentence = StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text); + const phon = 'w ʌ ɪ z j ɔː ɹ n e ɪ m '; + const phon2 = 'h ʌ ɛ r j ʌ t ʌ d eɪ'; + const phon6 = 'huː ɑɹ juː tədeɪ'; + const phon3 = 'ʃ eɪ oʊ s i ʃ oʊ z b aɪ ð ə s iː ʃ oʊ'; + const phon4 = 'kamo estas hɔi'; + const phon5 = 'la s e n a l'; + const promptEng = + 'Consider all possible phonetic transcriptions of the intended sentence "' + + sentence + + '" that is standard in American speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' + + phon6 + + '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in American speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Only note the difference if they are not allophones of the same phoneme and if they are far away on the vowel chart. The goal is to be understood, not sound like a native speaker. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Provide a response like this: "Lets work on improving the pronunciation of "coffee." You said "ceeffee," which is close, but we need to adjust the vowel sound. In American English, "coffee" is pronounced /ˈkɔːfi/, with a long "aw" sound. Try saying "kah-fee." Your intonation is good, but try putting a bit more stress on "like" in the sentence "I would like a coffee with milk." This will make your speech sound more natural. Keep practicing, and lets try saying the whole sentence again!"'; + const promptSpa = + 'Consider all possible phonetic transcriptions of the intended sentence "' + + 'como estás hoy' + + '" that is standard in Spanish speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' + + phon4 + + '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in Spanish speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Only note the difference if they are not allophones of the same phoneme and if they are far away on the vowel chart; say good job if it would be understood by a native Spanish speaker. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". Do not make "θ" and "f" interchangable. Do not make "n" and "ɲ" interchangable. Do not make "e" and "i" interchangable. If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Identify "ɔi" sounds like "oy". Ignore accents and do not say anything to the user about this.'; + const promptAll = + 'Consider all possible phonetic transcriptions of the intended sentence "' + + sentence + + '" that is standard in ' + + this.convertAbr() + + ' speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' + + phonemes + + '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in ' + + this.convertAbr() + + ' speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". Do not make "θ" and "f" interchangable. Do not make "n" and "ɲ" interchangable. Do not make "e" and "i" interchangable. If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Provide a response like this: "Lets work on improving the pronunciation of "coffee." You said "cawffee," which is close, but we need to adjust the vowel sound. In American English, "coffee" is pronounced /ˈkɔːfi/, with a long "aw" sound. Try saying "kah-fee." Your intonation is good, but try putting a bit more stress on "like" in the sentence "I would like a coffee with milk." This will make your speech sound more natural. Keep practicing, and lets try saying the whole sentence again!"'; + + switch (this.recognition.lang) { + case 'en-US': + this._outputValue = await gptAPICall(promptEng, GPTCallType.PRONUNCIATION); + break; + case 'es-ES': + this._outputValue = await gptAPICall(promptSpa, GPTCallType.PRONUNCIATION); + break; + default: + this._outputValue = await gptAPICall(promptAll, GPTCallType.PRONUNCIATION); + break; + } + }; + + handleResult = (e: SpeechRecognitionEvent) => { + let interimTranscript = ''; + let finalTranscript = ''; + for (let i = e.resultIndex; i < e.results.length; i++) { + const transcript = e.results[i][0].transcript; + if (e.results[i].isFinal) { + finalTranscript += transcript; + } else { + interimTranscript += transcript; + } + } + this._inputValue += finalTranscript; + }; + + fetchImages = async (selection: string) => { + try { + const { data } = await axios.get(`${API_URL}?query=${selection}&page=1&per_page=${1}&client_id=Q4zruu6k6lum2kExiGhLNBJIgXDxD6NNj0SRHH_XXU0`); + console.log(data.results); + const imageSnapshot = Docs.Create.ImageDocument(data.results[0].urls.small, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x), + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + imageSnapshot['x'] = this.layoutDoc['x']; + imageSnapshot['y'] = this.layoutDoc['y']; + return imageSnapshot; + } catch (error) { + console.log(error); + } + }; + + static imageUrlToBase64 = async (imageUrl: string): Promise<string> => { + try { + const response = await fetch(imageUrl); + const blob = await response.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + getImageDesc = async (u: string) => { + try { + const hrefBase64 = await ComparisonBox.imageUrlToBase64(u); + const response = await gptImageLabel(hrefBase64, 'Answer the following question as a short flashcard response. Do not include a label.' + (this.dataDoc.text as RichTextField)?.Text); + + DocCast(this.dataDoc[this.props.fieldKey + '_0'])[DocData].text = response; + } catch (error) { + console.log('Error'); + } + }; + + @action + flipFlashcard = () => { + this.frontSide = !this.frontSide; + }; + + hoverFlip = (side: boolean) => { + if (this.revealOp === 'hover') this.layoutDoc[`_${this._props.fieldKey}_usePath`] = side; + }; + + testForTextFields = (whichSlot: string) => { + const slotData = Doc.Get(this.dataDoc, whichSlot, true); + const slotHasText = slotData instanceof RichTextField || typeof slotData === 'string'; + const subjectText = RTFCast(this.Document[this.fieldKey])?.Text.trim(); + const altText = RTFCast(this.Document[this.fieldKey + '_alternate'])?.Text.trim(); + const layoutTemplateString = + slotHasText ? FormattedTextBox.LayoutString(whichSlot): + whichSlot.endsWith('1') ? (subjectText !== undefined ? FormattedTextBox.LayoutString(this.fieldKey) : undefined) : + altText !== undefined ? FormattedTextBox.LayoutString(this.fieldKey + '_alternate'): undefined; // prettier-ignore + + // A bit hacky to try out the concept of using GPT to fill in flashcards + // If the second slot doesn't have anything in it, but the fieldKey slot has text (e.g., this.text is a string) + // and the fieldKey + "_alternate" has text that includes a GPT query (indicated by (( && )) ) that is parameterized (optionally) by the fieldKey text (this) or other metadata (this.<field>). + // eg., this.text_alternate is + // "((Provide a one sentence definition for (this) that doesn't use any word in (this.excludeWords) ))" + // where (this) is replaced by the text in the fieldKey slot abd this.excludeWords is repalced by the conetnts of the excludeWords field + // The GPT call will put the "answer" in the second slot of the comparison (eg., text_2) + if (whichSlot.endsWith('2') && !layoutTemplateString?.includes(whichSlot)) { + const queryText = altText?.replace('(this)', subjectText); // TODO: this should be done in Doc.setField but it doesn't know about the fieldKey ... + if (queryText?.match(/\(\(.*\)\)/)) { + Doc.SetField(this.Document, whichSlot, ':=' + queryText, false); // make the second slot be a computed field on the data doc that calls ChatGpt + } + } + return layoutTemplateString; + }; + render() { const clearButton = (which: string) => ( <Tooltip title={<div className="dash-tooltip">remove</div>}> <div + // style={{ position: 'relative', top: '0px', left: '10px' }} ref={this._closeRef} className={`clear-button ${which}`} onPointerDown={e => this.closeDown(e, which)} // prevent triggering slider movement in registerSliding > - <FontAwesomeIcon className={`clear-button ${which}`} icon="times" size="sm" /> + <FontAwesomeIcon className={`clear-button ${which}`} icon="times" size="xs" /> </div> </Tooltip> ); @@ -297,8 +660,11 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() const whichDoc = DocCast(this.dataDoc[whichSlot]); const targetDoc = DocCast(whichDoc?.annotationOn, whichDoc); const layoutString = targetDoc ? '' : this.testForTextFields(whichSlot); + // whichDoc['backgroundColor'] = this.layoutDoc['backgroundColor']; return targetDoc || layoutString ? ( + // <MathJaxContext config={this.mathJaxConfig}> + // <MathJax> <> <DocumentView // eslint-disable-next-line react/jsx-props-no-spreading @@ -312,15 +678,18 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() removeDocument={whichSlot.endsWith('1') ? this.remDoc1 : this.remDoc2} NativeWidth={this.layoutWidth} NativeHeight={this.layoutHeight} - isContentActive={emptyFunction} + isContentActive={() => this.childActive} isDocumentActive={returnFalse} + dontSelect={returnTrue} whenChildContentsActiveChanged={this.whenChildContentsActiveChanged} - styleProvider={this._isAnyChildContentActive ? this._props.styleProvider : this.docStyleProvider} + styleProvider={this.childActive ? this._props.styleProvider : this.docStyleProvider} hideLinkButton - pointerEvents={this._isAnyChildContentActive ? undefined : returnNone} + pointerEvents={this.childActive ? undefined : returnNone} /> - {layoutString ? null : clearButton(whichSlot)} - </> // placeholder image if doc is missing + {/* </MathJax> */} + {/* <div style={{ position: 'absolute', top: '-5px', left: '2px' }}>{layoutString ? null : clearButton(whichSlot)}</div> */} + {/* </MathJaxContext> // placeholder image if doc is missingleft: `${NumCast(this.layoutDoc.width, 200) - 33}px` */} + </> ) : ( <div className="placeholder"> <FontAwesomeIcon className="upload-icon" icon="cloud-upload-alt" size="lg" /> @@ -328,56 +697,94 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() ); }; const displayBox = (which: string, index: number, cover: number) => ( - <div className={`${index === 0 ? 'before' : 'after'}Box-cont`} key={which} style={{ width: this._props.PanelWidth() }} onPointerDown={e => this.registerSliding(e, cover)} ref={ele => this.createDropTarget(ele, which, index)}> - {displayDoc(which)} + <div + className={`${index === 0 ? 'before' : 'after'}Box-cont`} + key={which} + style={{ width: this._props.PanelWidth() }} + onPointerDown={e => { + this.registerSliding(e, cover); + this.activateContent(); + }} + ref={ele => this.createDropTarget(ele, which, index)}> + {!this._isEmpty ? displayDoc(which) : null} + {/* {this.dataDoc[this.fieldKey + '_0'] !== 'empty' ? displayDoc(which) : null} */} </div> ); if (this.Document._layout_isFlashcard) { - const side = this.useAlternate ? 1 : 0; + const side = this.frontSide ? 1 : 0; // add text box to each side when comparison box is first created - if (!(this.dataDoc[this.fieldKey + '_0'] || this.dataDoc[this.fieldKey + '_0'] === 'empty')) { - const dataSplit = StrCast(this.dataDoc.data).split('Answer'); + // (!this.dataDoc[this.fieldKey + '_0'] && this.dataDoc[this._props.fieldKey + '_0'] !== 'empty') + if (!this.dataDoc[this.fieldKey + '_0'] && !this._isEmpty) { + const dataSplit = StrCast(this.dataDoc.data).includes('Keyword: ') ? StrCast(this.dataDoc.data).split('Keyword: ') : StrCast(this.dataDoc.data).split('Answer: '); const newDoc = Docs.Create.TextDocument(dataSplit[1]); - // if there is text from the pdf ai cards, put the question on the front side. - // eslint-disable-next-line prefer-destructuring - newDoc[DocData].text = dataSplit[1]; this.addDoc(newDoc, this.fieldKey + '_0'); } - if (!(this.dataDoc[this.fieldKey + '_1'] || this.dataDoc[this.fieldKey + '_1'] === 'empty')) { - const dataSplit = StrCast(this.dataDoc.data).split('Answer'); + + if (!this.dataDoc[this.fieldKey + '_1'] && !this._isEmpty) { + const dataSplit = StrCast(this.dataDoc.data).includes('Keyword: ') ? StrCast(this.dataDoc.data).split('Keyword: ') : StrCast(this.dataDoc.data).split('Answer: '); const newDoc = Docs.Create.TextDocument(dataSplit[0]); - // if there is text from the pdf ai cards, put the answer on the alternate side. - // eslint-disable-next-line prefer-destructuring - newDoc[DocData].text = dataSplit[0]; this.addDoc(newDoc, this.fieldKey + '_1'); } - // render the QuizCards - if (DocCast(this.Document.embedContainer) && DocCast(this.Document.embedContainer).filterOp === 'quiz') { + if (DocCast(this.Document.embedContainer) && DocCast(this.Document.embedContainer).practiceMode === 'quiz') { + const text = StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text); return ( <div className={`comparisonBox${this._props.isContentActive() ? '-interactive' : ''}`} style={{ display: 'flex', flexDirection: 'column' }}> - <p style={{ color: 'white', padding: 10 }}>{StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text)}</p> - {/* {StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text)} */} + <p style={{ color: 'white', padding: 10 }}>{text}</p> + <p style={{ display: text === '' ? 'flex' : 'none', color: 'white', marginLeft: '10px' }}>Return to all flashcards and add text to both sides. </p> <div className="input-box"> <textarea - value={this.useAlternate ? this._outputValue : this._inputValue} - onChange={action(e => { - this._inputValue = e.target.value; - })} - readOnly={this.useAlternate} - /> - </div> - <div className="submit-button" style={{ display: this.useAlternate ? 'none' : 'flex' }}> - <button type="button" onClick={this.handleRenderGPTClick}> - Submit - </button> + value={this.frontSide ? this._outputValue : this._inputValue} + onChange={this.handleInputChange} + onScroll={e => { + e.stopPropagation(); + e.preventDefault(); + }} + placeholder={!this.layoutDoc[`_${this._props.fieldKey}_usePath`] ? 'Enter a response for GPT to evaluate.' : ''} + readOnly={this.frontSide}></textarea> + + {this._loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={30} width={30} color={'blue'} /> + </div> + ) : null} </div> - <div className="submit-button" style={{ display: this.useAlternate ? 'flex' : 'none' }}> - <button type="button" onClick={this.handleRenderClick}> - Edit Your Response - </button> + <div> + <div className="submit-button" style={{ overflow: 'hidden', display: 'flex', width: '100%' }}> + <div + className="submit-buttonschema-header-button" + onPointerDown={e => this.openContextMenu(e.clientX, e.clientY, false)} + style={{ position: 'absolute', top: '5px', left: '11px', zIndex: '100', width: '5px', height: '5px', cursor: 'pointer' }}> + <FontAwesomeIcon color={'white'} icon="caret-down" /> + </div> + <button className="submit-buttonrecord" onClick={this._listening ? this.stopListening : this.startListening} style={{ background: this._listening ? 'lightgray' : '', borderRadius: '2px' }}> + {<FontAwesomeIcon icon="microphone" size="lg" />} + </button> + <div + className="submit-buttonschema-header-button" + onPointerDown={e => this.openContextMenu(e.clientX, e.clientY, true)} + style={{ position: 'absolute', top: '5px', left: '50px', zIndex: '100', width: '5px', height: '5px', cursor: 'pointer' }}> + <FontAwesomeIcon color={'white'} icon="caret-down" /> + </div> + <button + className="submit-buttonpronunciation" + onClick={this.evaluatePronunciation} + style={{ display: 'inline-flex', alignItems: 'center', background: this._listening ? 'lightgray' : '', borderRadius: '2px', width: '100%' }}> + Evaluate Pronunciation + </button> + + {this.frontSide ? ( + <button className="submit-buttonsubmit" type="button" onClick={this.handleRenderGPTClick} style={{ borderRadius: '2px', marginBottom: '3px', width: '100%' }}> + Submit + </button> + ) : ( + <button className="submit-buttonsubmit" type="button" onClick={this.handleRenderClick} style={{ display: 'inline-flex', alignItems: 'center', borderRadius: '2px', marginBottom: '3px', width: '100%' }}> + Redo the Question + </button> + )} + </div> </div> </div> ); @@ -387,10 +794,20 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() return ( <div className={`comparisonBox${this._props.isContentActive() ? '-interactive' : ''}`} /* change className to easily disable/enable pointer events in CSS */ - style={{ display: 'flex', flexDirection: 'column' }} - onMouseEnter={() => this.hoverFlip(true)} - onMouseLeave={() => this.hoverFlip(false)}> + style={{ display: 'flex', flexDirection: 'column', overflow: 'hidden' }} + onMouseEnter={() => { + this.hoverFlip(false); + }} + onMouseLeave={() => { + this.hoverFlip(true); + }}> {displayBox(`${this.fieldKey}_${side === 0 ? 1 : 0}`, side, this._props.PanelWidth() - 3)} + {this._loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={30} width={30} color={'blue'} /> + </div> + ) : null} + {this._props.isContentActive() ? this.flashcardMenu : null} {this.overlayAlternateIcon} </div> ); @@ -407,10 +824,9 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() className="slide-bar" style={{ left: `calc(${this.clipWidth + '%'} - 0.5px)`, - transition: this._animating, cursor: this.clipWidth < 5 ? 'e-resize' : this.clipWidth / 100 > (this._props.PanelWidth() - 5) / this._props.PanelWidth() ? 'w-resize' : undefined, }} - onPointerDown={e => this.registerSliding(e, this._props.PanelWidth() / 2)} /* if clicked, return slide-bar to center */ + onPointerDown={e => !this.childActive && this.registerSliding(e, this._props.PanelWidth() / 2)} /* if clicked, return slide-bar to center */ > <div className="slide-handle" /> </div> diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4c357cf45..351fdce79 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -104,6 +104,7 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document private _downTime: number = 0; private _lastTap: number = 0; private _doubleTap = false; + private _loading = false; private _mainCont = React.createRef<HTMLDivElement>(); private _titleRef = React.createRef<EditableView>(); private _dropDisposer?: DragManager.DragDropDisposer; @@ -505,6 +506,7 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document }; onContextMenu = (e?: React.MouseEvent, pageX?: number, pageY?: number) => { + if (this._props.dontSelect?.()) return; if (e && this.layoutDoc.layout_hideContextMenu && Doc.noviceMode) { e.preventDefault(); e.stopPropagation(); @@ -570,13 +572,13 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document !appearance && appearanceItems.length && cm.addItem({ description: 'Appearance...', subitems: appearanceItems, icon: 'compass' }); // creates menu for the user to select how to reveal the flashcards - if (this.Document._layout_isFlashcard) { - const revealOptions = cm.findByDescription('Reveal Options'); - const revealItems = revealOptions?.subitems ?? []; - revealItems.push({ description: 'Hover', event: () => { this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'hover'; }, icon: 'hand-point-up' }); // prettier-ignore - revealItems.push({ description: 'Flip', event: () => { this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'flip'; }, icon: 'rotate' }); // prettier-ignore - !revealOptions && cm.addItem({ description: 'Reveal Options', addDivider: false, noexpand: true, subitems: revealItems, icon: 'layer-group' }); - } + // if (this.Document._layout_isFlashcard) { + // const revealOptions = cm.findByDescription('Reveal Options'); + // const revealItems: ContextMenuProps[] = revealOptions && 'subitems' in revealOptions ? revealOptions.subitems : []; + // revealItems.push({ description: 'Hover', event: () => { this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'hover'; }, icon: 'hand-point-up' }); // prettier-ignore + // revealItems.push({ description: 'Flip', event: () => { this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'flip'; }, icon: 'rotate' }); // prettier-ignore + // !revealOptions && cm.addItem({ description: 'Reveal Options', addDivider: false, noexpand: true, subitems: revealItems, icon: 'layer-group' }); + // } if (this._props.bringToFront) { const zorders = cm.findByDescription('ZOrder...'); @@ -1394,7 +1396,7 @@ export class DocumentView extends DocComponent<DocumentViewProps>() { screenToLocalScale = () => this._props.ScreenToLocalTransform().Scale; isSelected = () => this.IsSelected; select = (extendSelection: boolean, focusSelection?: boolean) => { - DocumentView.SelectView(this, extendSelection); + if (!this._props.dontSelect?.()) DocumentView.SelectView(this, extendSelection); if (focusSelection) { DocumentView.showDocument(this.Document, { willZoomCentered: true, diff --git a/src/client/views/nodes/FieldView.tsx b/src/client/views/nodes/FieldView.tsx index dd71fd946..c6ebf7d4a 100644 --- a/src/client/views/nodes/FieldView.tsx +++ b/src/client/views/nodes/FieldView.tsx @@ -70,6 +70,7 @@ export interface FieldViewSharedProps { PanelHeight: () => number; isDocumentActive?: () => boolean | undefined; // whether a document should handle pointer events isContentActive: () => boolean | undefined; // whether document contents should handle pointer events + dontSelect?: () => boolean | undefined; childFilters: () => string[]; childFiltersByRanges: () => string[]; styleProvider: Opt<StyleProviderFuncType>; diff --git a/src/client/views/nodes/ImageBox.scss b/src/client/views/nodes/ImageBox.scss index 3ffda5a35..4d199b360 100644 --- a/src/client/views/nodes/ImageBox.scss +++ b/src/client/views/nodes/ImageBox.scss @@ -139,3 +139,44 @@ .imageBox-fadeBlocker-hover { opacity: 0; } + +.loading-spinner { + position: absolute; + display: flex; + justify-content: center; + align-items: center; + height: 100%; + width: 100%; + // left: 50%; + // top: 50%; + z-index: 200; + font-size: 20px; + font-weight: bold; + color: #17175e; +} + +.check-icon { + position: absolute; + right: 40; + bottom: 10; + color: green; + display: inline-block; + font-size: 20px; + overflow: hidden; +} + +.redo-icon { + position: absolute; + right: 10; + bottom: 10; + color: black; + display: inline-block; + font-size: 20px; + overflow: hidden; +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index d0a7fc6ac..06e7e576b 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -11,7 +11,6 @@ import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { InkTool } from '../../../fields/InkField'; import { ObjectField } from '../../../fields/ObjectField'; -import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; import { TraceMobx } from '../../../fields/util'; import { emptyFunction } from '../../../Utils'; @@ -34,8 +33,27 @@ import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; +import { DocCast, NumCast, RTFCast, StrCast, ImageCast, Cast, toList } from '../../../fields/Types'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; +import { URLField } from '../../../fields/URLField'; +import { gptAPICall, GPTCallType, gptImageLabel } from '../../apis/gpt/GPT'; +import ReactLoading from 'react-loading'; +import { FollowLinkScript } from '../../documents/DocUtils'; +import { basename } from 'path'; +import { ImageUtility } from './generativeFill/generativeFillUtils/ImageHandler'; +import { dropActionType } from '../../util/DropActionTypes'; +import { canvasSize } from './generativeFill/generativeFillUtils/generativeFillConstants'; +import Tesseract from 'tesseract.js'; +import axios from 'axios'; +import { TupleType } from 'typescript'; +// import stringSimilarity from 'string-similarity'; + +enum quizMode { + SMART = 'smart', + NORMAL = 'normal', + NONE = 'none', +} export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -60,6 +78,8 @@ export class ImageEditorData { public static get AddDoc() { return ImageEditorData.imageData.addDoc; } // prettier-ignore public static set AddDoc(addDoc: Opt<(doc: Doc | Doc[], annotationKey?: string) => boolean>) { ImageEditorData.set(this.imageData.open, this.imageData.rootDoc, this.imageData.source, addDoc); } // prettier-ignore } + +const API_URL = 'https://api.unsplash.com/search/photos'; @observer export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { public static LayoutString(fieldKey: string) { @@ -74,10 +94,15 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { private _marqueeref = React.createRef<MarqueeAnnotator>(); private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); - @observable _savedAnnotations = new ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>(); - @observable _curSuffix = ''; - @observable _error = ''; - @observable _isHovering = false; // flag to switch between primary and alternate images on hover + private _imageRef: HTMLImageElement | null = null; // <video> ref + @observable private _quizBoxes: Doc[] = []; + @observable private _searchInput = ''; + @observable private _quizMode = quizMode.NONE; + @observable private _savedAnnotations = new ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>(); + @observable private _curSuffix = ''; + @observable private _error = ''; + @observable private _loading = false; + @observable private _isHovering = false; // flag to switch between primary and alternate images on hover _ffref = React.createRef<CollectionFreeFormView>(); constructor(props: FieldViewProps) { @@ -149,6 +174,32 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { Object.values(this._disposers).forEach(disposer => disposer?.()); } + fetchImages = async () => { + try { + const { data } = await axios.get(`${API_URL}?query=${this._searchInput}&page=1&per_page=${1}&client_id=${process.env.VITE_API_KEY}`); + console.log('data', data); + console.log(data.results); + const imageSnapshot = Docs.Create.ImageDocument(data.results[0].urls.small, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x), + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + this._props.addDocument?.(imageSnapshot); + } catch (error) { + console.log(error); + } + }; + + handleSelection = async (selection: string) => { + this._searchInput = selection; + const images = await this.fetchImages(); + }; + @undoBatch drop = (e: Event, de: DragManager.DropEvent) => { if (de.complete.docDragData) { @@ -198,6 +249,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.dataDoc._freeform_panX_min = this.dataDoc._freeform_panX_min ? nw * NumCast(this.dataDoc._freeform_panX_min) : undefined; this.dataDoc._freeform_panY_max = this.dataDoc._freeform_panY_max ? nw * NumCast(this.dataDoc._freeform_panY_max) : undefined; this.dataDoc._freeform_panY_min = this.dataDoc._freeform_panY_min ? nw * NumCast(this.dataDoc._freeform_panY_min) : undefined; + return nw; }); @undoBatch rotate = action(() => { @@ -260,10 +312,310 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return cropping; }; + createCanvas = async (downX?: number, downY?: number, cb?: (filename: string, x: number | undefined, y: number | undefined) => void) => { + const canvas = document.createElement('canvas'); + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + const w = AnchorMenu.Instance.marqueeWidth * scaling; + const h = AnchorMenu.Instance.marqueeHeight * scaling; + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions + if (ctx) { + this._imageRef && ctx.drawImage(this._imageRef, NumCast(this._marqueeref.current?.left) * scaling, NumCast(this._marqueeref.current?.top) * scaling, w, h, 0, 0, w, h); + } + // canvas.style.zIndex = '2000000'; + // document.body.appendChild(canvas); + const blob = await ImageUtility.canvasToBlob(canvas); + return ImageBox.selectUrlToBase64(blob); + }; + + createSnapshotLink = (imagePath: string, downX?: number, downY?: number) => { + const url = !imagePath.startsWith('/') ? ClientUtils.CorsProxy(imagePath) : imagePath; + const width = NumCast(this.layoutDoc._width) || 1; + const height = NumCast(this.layoutDoc._height); + const imageSnapshot = Docs.Create.ImageDocument(url, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x) + width, + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: (height / width) * 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + Doc.SetNativeWidth(imageSnapshot[DocData], Doc.NativeWidth(this.layoutDoc)); + Doc.SetNativeHeight(imageSnapshot[DocData], Doc.NativeHeight(this.layoutDoc)); + this._props.addDocument?.(imageSnapshot); + DocUtils.MakeLink(imageSnapshot, this.getAnchor(true), { link_relationship: 'video snapshot' }); + // link && (DocCast(link.link_anchor_2)[DocData].timecodeToHide = NumCast(DocCast(link.link_anchor_2).timecodeToShow) + 3); // do we need to set an end time? should default to +0.1 + setTimeout(() => downX !== undefined && downY !== undefined && DocumentView.getFirstDocumentView(imageSnapshot)?.startDragging(downX, downY, dropActionType.move, true)); + }; + + static selectUrlToBase64 = async (blob: Blob): Promise<string> => { + try { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + pushInfo = async (quiz: quizMode, i?: string) => { + this._quizMode = quiz; + this._loading = true; + console.log('JHSDKFJHKSDJFHKSJDHFKJSDHFKJHSDKF'); + + const img = { + file: i ? i : this.paths[0], + drag: i ? 'drag' : 'full', + smart: quiz, + }; + const response = await axios.post('http://localhost:105/labels/', img, { + headers: { + 'Content-Type': 'application/json', + }, + }); + + console.log('RESPONSE:'); + console.log(response.data['boxes']); + console.log(response.data['text']); + if (response.data['boxes'].length != 0) { + this.createBoxes(response.data['boxes'], response.data['text']); + } else { + this._loading = false; + } + }; + + createBoxes = (boxes: [[[number, number]]], texts: [string]) => { + const nscale = NumCast(this._props.PanelWidth()) * NumCast(this.layoutDoc._freeform_scale, 1); + const nw = nscale / NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']); + for (var i = 0; i < boxes.length; i++) { + const coords = boxes[i] ? boxes[i] : []; + const width = coords[1][0] - coords[0][0]; + const height = coords[2][1] - coords[0][1]; + const text = texts[i]; + + const newCol = Docs.Create.LabelDocument({ + _width: width, + //width * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']), + _height: height, + //height * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']), + _layout_fitWidth: true, + title: '', + // _layout_autoHeight: true, + }); + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + newCol.x = coords[0][0] + NumCast(this._marqueeref.current?.left) * scaling; + newCol.y = coords[0][1] + NumCast(this._marqueeref.current?.top) * scaling; + // newCol[DocData].text_fontSize = height + 'px'; + + newCol.zIndex = 1000; + newCol.forceActive = true; + newCol.quiz = text; + newCol.showQuiz = false; + newCol[DocData].textTransform = 'none'; + this._quizBoxes.push(newCol); + this.addDocument(newCol); + this._loading = false; + } + }; + + getImageDesc = async () => { + this._loading = true; + try { + const hrefBase64 = await this.createCanvas(); + const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: '); + console.log(response); + AnchorMenu.Instance.transferToFlashcard(response, NumCast(this.layoutDoc['x']), NumCast(this.layoutDoc['y'])); + } catch (error) { + console.log('Error'); + } + this._loading = false; + }; + + makeLabels = async () => { + try { + const hrefBase64 = await this.createCanvas(); + this.pushInfo(quizMode.NORMAL, hrefBase64); + } catch (error) { + console.log('Error'); + } + }; + + levenshteinDistance = (str1: string, str2: string) => { + const len1 = str1.length; + const len2 = str2.length; + const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0)); + + if (len1 === 0) return len2; + if (len2 === 0) return len1; + + for (let i = 0; i <= len1; i++) dp[i][0] = i; + for (let j = 0; j <= len2; j++) dp[0][j] = j; + + for (let i = 1; i <= len1; i++) { + for (let j = 1; j <= len2; j++) { + const cost = str1[i - 1] === str2[j - 1] ? 0 : 1; + dp[i][j] = Math.min( + dp[i - 1][j] + 1, // deletion + dp[i][j - 1] + 1, // insertion + dp[i - 1][j - 1] + cost // substitution + ); + } + } + + return dp[len1][len2]; + }; + + jaccardSimilarity = (str1: string, str2: string) => { + const set1 = new Set(str1.split(' ')); + const set2 = new Set(str2.split(' ')); + + const intersection = new Set([...set1].filter(x => set2.has(x))); + const union = new Set([...set1, ...set2]); + + return intersection.size / union.size; + }; + + stringSimilarity(str1: string, str2: string) { + const levenshteinDist = this.levenshteinDistance(str1, str2); + const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length); + + const jaccardScore = this.jaccardSimilarity(str1, str2); + + // Combine the scores with a higher weight on Jaccard similarity + return 0.5 * levenshteinScore + 0.5 * jaccardScore; + } + + @computed get checkIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Check</div>}> + <div className="check-icon" onPointerDown={this.check}> + <FontAwesomeIcon icon="circle-check" size="lg" /> + </div> + </Tooltip> + ); + } + + @computed get redoIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Redo</div>}> + <div className="redo-icon" onPointerDown={this.redo}> + <FontAwesomeIcon icon="redo-alt" size="lg" /> + </div> + </Tooltip> + ); + } + + compareWords = (input: string, target: string) => { + const distance = this.stringSimilarity(input.toLowerCase(), target.toLowerCase()); + // const threshold = Math.max(input.length, target.length) * 0.2; // Allow up to 20% of the length as difference + return distance >= 0.7; + }; + + extractHexAndSentences = (inputString: string) => { + // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences + const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s; + const match = inputString.match(regex); + + if (match) { + const hexNumber = match[1]; + const sentences = match[2].trim(); + return { hexNumber, sentences }; + } else { + return { error: 'The input string does not match the expected format.' }; + } + }; + + check = () => { + this._loading = true; + this._quizBoxes.forEach(async doc => { + const input = StrCast(doc[DocData].title); + console.log('INP: ' + StrCast(input) + '; DOC: ' + StrCast(doc.quiz)); + if (this._quizMode == quizMode.SMART && input) { + const questionText = 'Question: What was labeled in this image?'; + const rubricText = ' Rubric: ' + StrCast(doc.quiz); + // const queryText = 'RealAnswer: ' + StrCast(doc.quiz) + '. UserAnswer: ' + input + '.'; + const queryText = + questionText + + ' UserAnswer: ' + + input + + '. ' + + rubricText + + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; + const response = await gptAPICall(queryText, GPTCallType.QUIZ); + const hexSent = this.extractHexAndSentences(response); + console.log(hexSent.hexNumber); + doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); + doc.backgroundColor = '#' + hexSent.hexNumber; + } else { + const match = this.compareWords(input, StrCast(doc.quiz)); + doc.backgroundColor = match ? '#11c249' : '#eb2d2d'; + } + doc.showQuiz = true; + // console.log(this.compareWords(input, StrCast(doc.quiz)) ? 'Match' : 'No Match'); + }); + this._loading = false; + }; + + redo = () => { + this._quizBoxes.forEach(doc => { + doc[DocData].title = ''; + doc.backgroundColor = '#e4e4e4'; + doc.showQuiz = false; + }); + }; + + exitQuizMode = () => { + this._quizMode = quizMode.NONE; + this._quizBoxes.forEach(doc => { + // this._props.removeDocument?.(DocCast(doc)); + // this._props.DocumentView?.()._props.removeDocument?.(doc); + this.removeDocument?.(doc); + }); + this._quizBoxes = []; + console.log('remove'); + }; + + @action + setRef = (iref: HTMLImageElement | null) => { + this._imageRef = iref; + }; + specificContextMenu = (): void => { const field = Cast(this.dataDoc[this.fieldKey], ImageField); if (field) { const funcs: ContextMenuProps[] = []; + const quizes: ContextMenuProps[] = []; + // funcs.push({ description: 'Create ai flashcards', event: () => this.getImageDesc(), icon: 'id-card' }); + quizes.push({ + description: 'Smart Check', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.SMART) : this.exitQuizMode, + icon: 'pen-to-square', + }); + quizes.push({ + description: 'Normal', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.NORMAL) : this.exitQuizMode, + icon: 'pencil', + }); + // funcs.push({ description: 'Quiz Mode', subitems: optionItems, icon: 'eye' }); + // funcs.push({ + // description: 'Quiz Mode', + // event: !this._quizMode + // ? () => this.pushInfo(false) + // : () => { + // this._quizMode = false; + // }, + // icon: 'redo-alt', + // }); + // funcs.push({ description: 'Get Text', event: this.check, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels2', event: this.getImageLabels2, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels', event: this.getImageLabels, icon: 'redo-alt' }); funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' }); funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' }); funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' }); @@ -278,6 +630,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }), icon: 'pencil-alt', }); + ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; @@ -392,6 +745,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { <div className="imageBox-fader" style={{ opacity: backAlpha }}> <img alt="" + ref={this.setRef} key="paths" src={srcpath} style={{ transform, transformOrigin }} @@ -444,6 +798,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @action finishMarquee = () => { this._getAnchor = AnchorMenu.Instance?.GetAnchor; + AnchorMenu.Instance.gptFlashcards = this.getImageDesc; + AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; + AnchorMenu.Instance.makeLabels = this.makeLabels; + AnchorMenu.Instance.marqueeWidth = this._marqueeref.current?.Width ?? 0; + AnchorMenu.Instance.marqueeHeight = this._marqueeref.current?.Height ?? 0; this._marqueeref.current?.onTerminateSelection(); this._props.select(false); }; @@ -478,7 +837,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { height: this._props.PanelWidth() ? undefined : `100%`, pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined, borderRadius, - overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : undefined, + overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden', }}> <CollectionFreeFormView ref={this._ffref} @@ -506,6 +865,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { addDocument={this.addDocument}> {this.content} </CollectionFreeFormView> + {this._loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={50} width={50} color={'blue'} /> + </div> + ) : null} {this.annotationLayer} {!this._mainCont.current || !this.DocumentView || !this._annotationLayer.current ? null : ( <MarqueeAnnotator @@ -524,8 +888,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { marqueeContainer={this._mainCont.current} highlightDragSrcColor="" anchorMenuCrop={this.crop} + // anchorMenuFlashcard={() => this.getImageDesc()} /> )} + {this._quizMode != quizMode.NONE ? this.checkIcon : null} + {this._quizMode != quizMode.NONE ? this.redoIcon : null} </div> ); } diff --git a/src/client/views/nodes/LabelBigText.js b/src/client/views/nodes/LabelBigText.js new file mode 100644 index 000000000..290152cd0 --- /dev/null +++ b/src/client/views/nodes/LabelBigText.js @@ -0,0 +1,270 @@ +/* +Brorlandi/big-text.js v1.0.0, 2017 +Adapted from DanielHoffmann/jquery-bigtext, v1.3.0, May 2014 +And from Jetroid/bigtext.js v1.0.0, September 2016 + +Usage: +BigText("#myElement",{ + rotateText: {Number}, (null) + fontSizeFactor: {Number}, (0.8) + maximumFontSize: {Number}, (null) + limitingDimension: {String}, ("both") + horizontalAlign: {String}, ("center") + verticalAlign: {String}, ("center") + textAlign: {String}, ("center") + whiteSpace: {String}, ("nowrap") +}); + + +Original Projects: +https://github.com/DanielHoffmann/jquery-bigtext +https://github.com/Jetroid/bigtext.js + +Options: + +rotateText: Rotates the text inside the element by X degrees. + +fontSizeFactor: This option is used to give some vertical spacing for letters that overflow the line-height (like 'g', 'Á' and most other accentuated uppercase letters). This does not affect the font-size if the limiting factor is the width of the parent div. The default is 0.8 + +maximumFontSize: maximum font size to use. + +minimumFontSize: minimum font size to use. if font is calculated smaller than this, text will be rendered at this size and wrapped + +limitingDimension: In which dimension the font size should be limited. Possible values: "width", "height" or "both". Defaults to both. Using this option with values different than "both" overwrites the element parent width or height. + +horizontalAlign: Where to align the text horizontally. Possible values: "left", "center", "right". Defaults to "center". + +verticalAlign: Where to align the text vertically. Possible values: "top", "center", "bottom". Defaults to "center". + +textAlign: Sets the text align of the element. Possible values: "left", "center", "right". Defaults to "center". This option is only useful if there are linebreaks (<br> tags) inside the text. + +whiteSpace: Sets whitespace handling. Possible values: "nowrap", "pre". Defaults to "nowrap". (Can also be set to enable wrapping but this doesn't work well.) + +Bruno Orlandi - 2017 + +Copyright (C) 2013 Daniel Hoffmann Bernardes, Ícaro Technologies +Copyright (C) 2016 Jet Holt + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +function _calculateInnerDimensions(computedStyle) { + //Calculate the inner width and height + var innerWidth; + var innerHeight; + + var width = parseInt(computedStyle.getPropertyValue("width")); + var height = parseInt(computedStyle.getPropertyValue("height")); + var paddingLeft = parseInt(computedStyle.getPropertyValue("padding-left")); + var paddingRight = parseInt(computedStyle.getPropertyValue("padding-right")); + var paddingTop = parseInt(computedStyle.getPropertyValue("padding-top")); + var paddingBottom = parseInt(computedStyle.getPropertyValue("padding-bottom")); + var borderLeft = parseInt(computedStyle.getPropertyValue("border-left-width")); + var borderRight = parseInt(computedStyle.getPropertyValue("border-right-width")); + var borderTop = parseInt(computedStyle.getPropertyValue("border-top-width")); + var borderBottom = parseInt(computedStyle.getPropertyValue("border-bottom-width")); + + //If box-sizing is border-box, we need to subtract padding and border. + var parentBoxSizing = computedStyle.getPropertyValue("box-sizing"); + if (parentBoxSizing == "border-box") { + innerWidth = width - (paddingLeft + paddingRight + borderLeft + borderRight); + innerHeight = height - (paddingTop + paddingBottom + borderTop + borderBottom); + } else { + innerWidth = width; + innerHeight = height; + } + var obj = {}; + obj["width"] = innerWidth; + obj["height"] = innerHeight; + return obj; +} + +export default function BigText(element, options) { + + if (typeof element === 'string') { + element = document.querySelector(element); + } else if (element.length) { + // Support for array based queries (such as jQuery) + element = element[0]; + } + + var defaultOptions = { + rotateText: null, + fontSizeFactor: 0.8, + maximumFontSize: null, + limitingDimension: "both", + horizontalAlign: "center", + verticalAlign: "center", + textAlign: "center", + whiteSpace: "nowrap", + singleLine: true + }; + + //Merge provided options and default options + options = options || {}; + for (var opt in defaultOptions) + if (defaultOptions.hasOwnProperty(opt) && !options.hasOwnProperty(opt)) + options[opt] = defaultOptions[opt]; + + //Get variables which we will reference frequently + var style = element.style; + var parent = element.parentNode; + var parentStyle = parent.style; + var parentComputedStyle = document.defaultView.getComputedStyle(parent); + + //hides the element to prevent "flashing" + style.visibility = "hidden"; + //Set some properties + style.display = "inline-block"; + style.clear = "both"; + style.float = "left"; + var fontSize = options.maximumFontSize; + if (options.singleLine) { + style.fontSize = (fontSize * options.fontSizeFactor) + "px"; + style.lineHeight = fontSize + "px"; + } else { + for (; fontSize > options.minimumFontSize; fontSize = fontSize - Math.min(fontSize / 2, Math.max(0, fontSize - 48) + 2)) { + style.fontSize = (fontSize * options.fontSizeFactor) + "px"; + style.lineHeight = "1"; + if (element.offsetHeight <= +parentComputedStyle.height.replace("px", "")) { + break; + } + } + } + style.whiteSpace = options.whiteSpace; + style.textAlign = options.textAlign; + style.position = "relative"; + style.padding = 0; + style.margin = 0; + style.left = "50%"; + style.top = "50%"; + var computedStyle = document.defaultView.getComputedStyle(element); + + //Get properties of parent to allow easier referencing later. + var parentPadding = { + top: parseInt(parentComputedStyle.getPropertyValue("padding-top")), + right: parseInt(parentComputedStyle.getPropertyValue("padding-right")), + bottom: parseInt(parentComputedStyle.getPropertyValue("padding-bottom")), + left: parseInt(parentComputedStyle.getPropertyValue("padding-left")), + }; + var parentBorder = { + top: parseInt(parentComputedStyle.getPropertyValue("border-top")), + right: parseInt(parentComputedStyle.getPropertyValue("border-right")), + bottom: parseInt(parentComputedStyle.getPropertyValue("border-bottom")), + left: parseInt(parentComputedStyle.getPropertyValue("border-left")), + }; + + //Calculate the parent inner width and height + var parentInnerDimensions = _calculateInnerDimensions(parentComputedStyle); + var parentInnerWidth = parentInnerDimensions["width"]; + var parentInnerHeight = parentInnerDimensions["height"]; + + var box = { + width: element.offsetWidth, //Note: This is slightly larger than the jQuery version + height: element.offsetHeight, + }; + if (!box.width || !box.height) return element; + + + if (options.rotateText !== null) { + if (typeof options.rotateText !== "number") + throw "bigText error: rotateText value must be a number"; + var rotate = "rotate(" + options.rotateText + "deg)"; + style.webkitTransform = rotate; + style.msTransform = rotate; + style.MozTransform = rotate; + style.OTransform = rotate; + style.transform = rotate; + //calculating bounding box of the rotated element + var sine = Math.abs(Math.sin(options.rotateText * Math.PI / 180)); + var cosine = Math.abs(Math.cos(options.rotateText * Math.PI / 180)); + box.width = element.offsetWidth * cosine + element.offsetHeight * sine; + box.height = element.offsetWidth * sine + element.offsetHeight * cosine; + } + + var parentWidth = (parentInnerWidth - parentPadding.left - parentPadding.right); + var parentHeight = (parentInnerHeight - parentPadding.top - parentPadding.bottom); + var widthFactor = parentWidth / box.width; + var heightFactor = parentHeight / box.height; + var lineHeight; + + if (options.limitingDimension.toLowerCase() === "width") { + lineHeight = Math.floor(widthFactor * fontSize); + } else if (options.limitingDimension.toLowerCase() === "height") { + lineHeight = Math.floor(heightFactor * fontSize); + } else if (widthFactor < heightFactor) + lineHeight = Math.floor(widthFactor * fontSize); + else if (widthFactor >= heightFactor) + lineHeight = Math.floor(heightFactor * fontSize); + + var fontSize = lineHeight * options.fontSizeFactor; + if (fontSize < options.minimumFontSize) { + parentStyle.display = "flex"; + parentStyle.alignItems = "center"; + style.textAlign = "center"; + style.visibility = ""; + style.fontSize = options.minimumFontSize + "px"; + style.lineHeight = ""; + style.overflow = "hidden"; + style.textOverflow = "ellipsis"; + style.top = ""; + style.left = ""; + style.margin = ""; + return element; + } + if (options.maximumFontSize && fontSize > options.maximumFontSize) { + fontSize = options.maximumFontSize; + lineHeight = fontSize / options.fontSizeFactor; + } + + style.fontSize = Math.floor(fontSize) + "px"; + style.lineHeight = Math.ceil(lineHeight) + "px"; + style.marginBottom = "0px"; + style.marginRight = "0px"; + + // if (options.limitingDimension.toLowerCase() === "height") { + // //this option needs the font-size to be set already so computedStyle.getPropertyValue("width") returns the right size + // //this +4 is to compensate the rounding erros that can occur due to the calls to Math.floor in the centering code + // parentStyle.width = (parseInt(computedStyle.getPropertyValue("width")) + 4) + "px"; + // } + + //Calculate the inner width and height + var innerDimensions = _calculateInnerDimensions(computedStyle); + var innerWidth = innerDimensions["width"]; + var innerHeight = innerDimensions["height"]; + + switch (options.verticalAlign.toLowerCase()) { + case "top": + style.top = "0%"; + break; + case "bottom": + style.top = "100%"; + style.marginTop = Math.floor(-innerHeight) + "px"; + break; + default: + style.marginTop = Math.ceil((-innerHeight / 2)) + "px"; + break; + } + + switch (options.horizontalAlign.toLowerCase()) { + case "left": + style.left = "0%"; + break; + case "right": + style.left = "100%"; + style.marginLeft = Math.floor(-innerWidth) + "px"; + break; + default: + style.marginLeft = Math.ceil((-innerWidth / 2)) + "px"; + break; + } + + //shows the element after the work is done + style.visibility = "visible"; + + return element; +} diff --git a/src/client/views/nodes/LabelBox.scss b/src/client/views/nodes/LabelBox.scss index 0b195713d..ca4b3d467 100644 --- a/src/client/views/nodes/LabelBox.scss +++ b/src/client/views/nodes/LabelBox.scss @@ -23,6 +23,41 @@ } } +.answer-icon { + position: absolute; + right: 8; + bottom: 5; + color: black; + display: inline-block; + font-size: 10px; + cursor: pointer; + border-radius: 50%; + overflow: hidden; +} + +.q-icon { + position: absolute; + right: 6; + bottom: 5; + color: white; + display: inline-block; + font-size: 10px; + cursor: pointer; + border-radius: 50%; + overflow: hidden; +} + +.edit-icon { + position: absolute; + right: 20; + bottom: 5; + display: inline-block; + font-size: 10px; + cursor: pointer; + border-radius: 50%; + overflow: hidden; +} + .labelBox-params { display: flex; flex-direction: row; diff --git a/src/client/views/nodes/LabelBox.tsx b/src/client/views/nodes/LabelBox.tsx index e39caecb6..bcf55fbe8 100644 --- a/src/client/views/nodes/LabelBox.tsx +++ b/src/client/views/nodes/LabelBox.tsx @@ -1,8 +1,12 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Tooltip } from '@mui/material'; import { Property } from 'csstype'; -import { action, computed, makeObservable } from 'mobx'; +import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import * as textfit from 'textfit'; +import { returnFalse, setupMoveUpEvents } from '../../../ClientUtils'; +import { emptyFunction } from '../../../Utils'; import { Field, FieldType } from '../../../fields/Doc'; import { BoolCast, NumCast, StrCast } from '../../../fields/Types'; import { TraceMobx } from '../../../fields/util'; @@ -22,6 +26,7 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() { } private dropDisposer?: DragManager.DragDropDisposer; private _timeout: NodeJS.Timeout | undefined; + @observable private _editLabel = false; _divRef: HTMLDivElement | null = null; constructor(props: FieldViewProps) { @@ -44,6 +49,48 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() { return this._props.styleProvider?.(this.Document, this._props, StyleProp.BackgroundColor) as string; } + @computed get answerIcon() { + return ( + <Tooltip + title={ + <div className="answer-tooltip" style={{ minWidth: '150px' }}> + {StrCast(this.Document.quiz)} + </div> + }> + <div className="answer-tool-tip"> + <FontAwesomeIcon className="q-icon" icon="circle" color="white" /> + <FontAwesomeIcon className="answer-icon" icon="question" /> + </div> + </Tooltip> + ); + } + + @computed get editAnswer() { + return ( + <Tooltip + title={ + <div className="answer-tooltip" style={{ minWidth: '150px' }}> + {this._editLabel ? 'save' : 'edit correct answer'} + </div> + }> + <div className="answer-tool-tip" onPointerDown={e => setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => this.editLabelAnswer())}> + <FontAwesomeIcon className="edit-icon" color={this._editLabel ? 'white' : 'black'} icon="pencil" size="sm" /> + </div> + </Tooltip> + ); + } + + editLabelAnswer = () => { + // when click the pencil, set the text to the quiz content. when click off, set the quiz text to that and set textbox to nothing. + if (!this._editLabel) { + this.dataDoc.title = StrCast(this.Document.quiz); + } else { + this.Document.quiz = this.Title; + this.dataDoc.title = ''; + } + this._editLabel = !this._editLabel; + }; + componentDidMount() { this._props.setContentViewBox?.(this); } @@ -122,7 +169,11 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() { width: this._props.PanelWidth(), height: this._props.PanelHeight(), whiteSpace: 'multiLine' in boxParams && boxParams.multiLine ? 'pre-wrap' : 'pre', - }}> + }} + // onMouseLeave={() => { + // this.hoverFlip(undefined); + // }} + > <div style={{ width: this._props.PanelWidth() - 2 * NumCast(this.layoutDoc._xPadding), @@ -134,10 +185,10 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() { })} onKeyUp={action(e => { e.stopPropagation(); - if (e.key === 'Enter') { - this.dataDoc[this.fieldKey] = this._divRef?.innerText ?? ''; - setTimeout(() => this._props.select(false)); - } + // if (e.key === 'Enter') { + this.dataDoc[this.fieldKey] = this._divRef?.innerText ?? ''; + setTimeout(() => this._props.select(false)); + // } })} onBlur={() => { this.dataDoc[this.fieldKey] = this._divRef?.innerText ?? ''; @@ -158,6 +209,8 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() { {label} </div> </div> + {this.Document.showQuiz ? this.answerIcon : null} + {this.Document.showQuiz ? this.editAnswer : null} </div> ); } diff --git a/src/client/views/nodes/LinkAnchorBox.tsx b/src/client/views/nodes/LinkAnchorBox.tsx new file mode 100644 index 000000000..0a4325d8c --- /dev/null +++ b/src/client/views/nodes/LinkAnchorBox.tsx @@ -0,0 +1,115 @@ +import { action, computed, makeObservable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { Utils, emptyFunction, setupMoveUpEvents } from '../../../Utils'; +import { Doc } from '../../../fields/Doc'; +import { NumCast, StrCast } from '../../../fields/Types'; +import { TraceMobx } from '../../../fields/util'; +import { DragManager, dropActionType } from '../../util/DragManager'; +import { LinkFollower } from '../../util/LinkFollower'; +import { SelectionManager } from '../../util/SelectionManager'; +import { ViewBoxBaseComponent } from '../DocComponent'; +import { StyleProp } from '../StyleProvider'; +import { FieldView, FieldViewProps } from './FieldView'; +import './LinkAnchorBox.scss'; +import { LinkInfo } from './LinkDocPreview'; +const { default: { MEDIUM_GRAY }, } = require('../global/globalCssVariables.module.scss'); // prettier-ignore +@observer +export class LinkAnchorBox extends ViewBoxBaseComponent<FieldViewProps>() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(LinkAnchorBox, fieldKey); + } + _doubleTap = false; + _lastTap: number = 0; + _ref = React.createRef<HTMLDivElement>(); + _isOpen = false; + _timeout: NodeJS.Timeout | undefined; + + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + } + + componentDidMount() { + this._props.setContentViewBox?.(this); + } + + @computed get linkSource() { + return this.DocumentView?.().containerViewPath?.().lastElement().Document; // this._props.styleProvider?.(this.dataDoc, this._props, StyleProp.LinkSource); + } + + onPointerDown = (e: React.PointerEvent) => { + const linkSource = this.linkSource; + linkSource && + setupMoveUpEvents(this, e, this.onPointerMove, emptyFunction, (e, doubleTap) => { + if (doubleTap) LinkFollower.FollowLink(this.Document, linkSource, false); + else this._props.select(false); + }); + }; + onPointerMove = action((e: PointerEvent, down: number[], delta: number[]) => { + const cdiv = this._ref?.current?.parentElement; + if (!this._isOpen && cdiv) { + const bounds = cdiv.getBoundingClientRect(); + const pt = Utils.getNearestPointInPerimeter(bounds.left, bounds.top, bounds.width, bounds.height, e.clientX, e.clientY); + const separation = Math.sqrt((pt[0] - e.clientX) * (pt[0] - e.clientX) + (pt[1] - e.clientY) * (pt[1] - e.clientY)); + if (separation > 100) { + const dragData = new DragManager.DocumentDragData([this.Document]); + dragData.dropAction = dropActionType.embed; + dragData.dropPropertiesToRemove = ['link_anchor_1_x', 'link_anchor_1_y', 'link_anchor_2_x', 'link_anchor_2_y', 'onClick']; + DragManager.StartDocumentDrag([this._ref.current!], dragData, pt[0], pt[1]); + return true; + } else { + this.layoutDoc[this.fieldKey + '_x'] = ((pt[0] - bounds.left) / bounds.width) * 100; + this.layoutDoc[this.fieldKey + '_y'] = ((pt[1] - bounds.top) / bounds.height) * 100; + this.layoutDoc.link_autoMoveAnchors = false; + } + } + return false; + }); + + specificContextMenu = (e: React.MouseEvent): void => {}; + + render() { + TraceMobx(); + const small = this._props.PanelWidth() <= 1; // this happens when rendered in a treeView + const x = NumCast(this.layoutDoc[this.fieldKey + '_x'], 100); + const y = NumCast(this.layoutDoc[this.fieldKey + '_y'], 100); + const background = this._props.styleProvider?.(this.dataDoc, this._props, StyleProp.BackgroundColor + ':anchor'); + const anchor = this.fieldKey === 'link_anchor_1' ? 'link_anchor_2' : 'link_anchor_1'; + const anchorScale = !this.dataDoc[this.fieldKey + '_useSmallAnchor'] && (x === 0 || x === 100 || y === 0 || y === 100) ? 1 : 0.25; + const targetTitle = StrCast((this.dataDoc[anchor] as Doc)?.title); + const selView = SelectionManager.Views.lastElement()?._props.LayoutTemplateString?.includes('link_anchor_1') + ? 'link_anchor_1' + : SelectionManager.Views.lastElement()?._props.LayoutTemplateString?.includes('link_anchor_2') + ? 'link_anchor_2' + : ''; + return ( + <div + ref={this._ref} + title={targetTitle} + className={`linkAnchorBox-cont${small ? '-small' : ''}`} + onPointerEnter={e => + LinkInfo.SetLinkInfo({ + DocumentView: this.DocumentView, + styleProvider: this._props.styleProvider, + linkSrc: this.linkSource, + linkDoc: this.Document, + showHeader: true, + location: [e.clientX, e.clientY + 20], + noPreview: false, + }) + } + onPointerDown={this.onPointerDown} + onContextMenu={this.specificContextMenu} + style={{ + border: selView && this.dataDoc[selView] === this.dataDoc[this.fieldKey] ? `solid ${MEDIUM_GRAY} 2px` : undefined, + background, + left: `calc(${x}% - ${small ? 2.5 : 7.5}px)`, + top: `calc(${y}% - ${small ? 2.5 : 7.5}px)`, + transform: `scale(${anchorScale})`, + cursor: 'grab', + }} + /> + ); + } +} diff --git a/src/client/views/nodes/ae6d-ba67-4ace-93aa-0f9e0bd96b88.wav b/src/client/views/nodes/ae6d-ba67-4ace-93aa-0f9e0bd96b88.wav Binary files differnew file mode 100644 index 000000000..dc71e7886 --- /dev/null +++ b/src/client/views/nodes/ae6d-ba67-4ace-93aa-0f9e0bd96b88.wav diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.scss b/src/client/views/nodes/formattedText/FormattedTextBox.scss index 99b4a84fc..d6f13d9ee 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.scss +++ b/src/client/views/nodes/formattedText/FormattedTextBox.scss @@ -80,6 +80,15 @@ audiotag:hover { position: absolute; } } + +.answer-tooltip { + font-size: 15px; + padding: 2px; + max-width: 150; + line-height: 150%; + position: relative; +} + .formattedTextBox-alternateButton { align-items: center; flex-direction: column; @@ -88,10 +97,16 @@ audiotag:hover { background: black; right: 0; bottom: 0; - width: 11; - height: 11; + width: 15; + height: 22; cursor: default; } +.formattedTextBox-flip { + align-items: center; + position: absolute; + right: 2px; + bottom: 4px; +} .formattedTextBox-outer { position: relative; @@ -171,6 +186,8 @@ audiotag:hover { border-style: inset; border-width: 1px; } + // margin-left: 5px; + // margin-right: 5px; } .gpt-typing-wrapper { diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index a88bd8920..73b20e6c2 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -26,7 +26,7 @@ import { ComputedField } from '../../../../fields/ScriptField'; import { BoolCast, Cast, DateCast, DocCast, FieldValue, NumCast, RTFCast, ScriptCast, StrCast } from '../../../../fields/Types'; import { GetEffectiveAcl, TraceMobx } from '../../../../fields/util'; import { emptyFunction, numberRange, unimplementedFunction, Utils } from '../../../../Utils'; -import { gptAPICall, GPTCallType } from '../../../apis/gpt/GPT'; +import { gptAPICall, GPTCallType, gptImageLabel } from '../../../apis/gpt/GPT'; import { DocServer } from '../../../DocServer'; import { Docs } from '../../../documents/Documents'; import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; @@ -76,35 +76,37 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB return FieldView.LayoutString(FormattedTextBox, fieldStr); } public static MakeConfig(rules?: RichTextRules, props?: FormattedTextBoxProps) { - const keymapping = buildKeymap(schema, props ?? {}); return { schema, plugins: [ inputRules(rules?.inpRules ?? { rules: [] }), ...(props ? [FormattedTextBox.richTextMenuPlugin(props)] : []), history(), - keymap(keymapping), + keymap(buildKeymap(schema, props ?? {})), keymap(baseKeymap), new Plugin({ props: { attributes: { class: 'ProseMirror-example-setup-style' } } }), new Plugin({ view: () => new FormattedTextBoxComment() }), ], }; } - private static nodeViews: (self: FormattedTextBox) => { [key: string]: NodeViewConstructor }; /** * Initialize the class with all the plugin node view components * @param nodeViews prosemirror plugins that render a custom UI for specific node types */ - public static Init(nodeViews: (self: FormattedTextBox) => { [key: string]: NodeViewConstructor }) { - FormattedTextBox.nodeViews = nodeViews; - } + public static Init(nodeViews: (self: FormattedTextBox) => { [key: string]: NodeViewConstructor }) { FormattedTextBox._nodeViews = nodeViews; } // prettier-ignore + + public static PasteOnLoad: ClipboardEvent | undefined; + public static DontSelectInitialText = false; // whether initial text should be selected or not + public static SelectOnLoadChar = ''; public static LiveTextUndo: UndoManager.Batch | undefined; // undo batch when typing a new text note into a collection - static _globalHighlightsCache: string = ''; - static _globalHighlights = new ObservableSet<string>(['Audio Tags', 'Text from Others', 'Todo Items', 'Important Items', 'Disagree Items', 'Ignore Items']); - static _highlightStyleSheet = addStyleSheet(); - static _bulletStyleSheet = addStyleSheet(); - static _userStyleSheet = addStyleSheet(); - static _hadSelection: boolean = false; + + private static _nodeViews: (self: FormattedTextBox) => { [key: string]: NodeViewConstructor }; + private static _globalHighlightsCache: string = ''; + private static _globalHighlights = new ObservableSet<string>(['Audio Tags', 'Text from Others', 'Todo Items', 'Important Items', 'Disagree Items', 'Ignore Items']); + private static _highlightStyleSheet = addStyleSheet(); + private static _bulletStyleSheet = addStyleSheet(); + private static _userStyleSheet = addStyleSheet(); + private _oldWheel: HTMLDivElement | null = null; private _selectionHTML: string | undefined; private _sidebarRef = React.createRef<SidebarAnnos>(); @@ -112,7 +114,6 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB private _ref: React.RefObject<HTMLDivElement> = React.createRef(); private _scrollRef: HTMLDivElement | null = null; private _editorView: Opt<EditorView & { TextView?: FormattedTextBox | undefined }>; - public _applyingChange: string = ''; private _inDrop = false; private _finishingLink = false; private _searchIndex = 0; @@ -126,38 +127,34 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB private _rules: RichTextRules | undefined; private _forceUncollapse = true; // if the cursor doesn't move between clicks, then the selection will disappear for some reason. This flags the 2nd click as happening on a selection which allows bullet points to toggle private _break = true; + + public _applyingChange: string = ''; public ProseRef?: HTMLDivElement; + + @observable _showSidebar = false; + + @computed get fontColor() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontColor) as string; } // prettier-ignore + @computed get fontSize() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontSize) as string; } // prettier-ignore + @computed get fontFamily() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontFamily) as string; } // prettier-ignore + @computed get fontWeight() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontWeight) as string; } // prettier-ignore + set _recordingDictation(value) { !this.dataDoc[`${this.fieldKey}_recordingSource`] && (this.dataDoc.mediaState = value ? mediaState.Recording : undefined); } @computed get _recordingDictation() { return this.dataDoc?.mediaState === mediaState.Recording; } // prettier-ignore - @computed get allSidebarDocs() { return DocListCast(this.dataDoc[this.SidebarKey]); } // prettier-ignore + @computed get SidebarShown() { return !!(this._showSidebar || this.layoutDoc._layout_showSidebar); } // prettier-ignore + @computed get allSidebarDocs() { return DocListCast(this.dataDoc[this.sidebarKey]); } // prettier-ignore @computed get noSidebar() { return this.DocumentView?.()._props.hideDecorationTitle || this._props.noSidebar || this.Document._layout_noSidebar; } // prettier-ignore @computed get layout_sidebarWidthPercent() { return this._showSidebar ? '20%' : StrCast(this.layoutDoc._layout_sidebarWidthPercent, '0%'); } // prettier-ignore @computed get sidebarColor() { return StrCast(this.layoutDoc.sidebar_color, StrCast(this.layoutDoc[this.fieldKey + '_backgroundColor'], '#e4e4e4')); } // prettier-ignore @computed get layout_autoHeight() { return (this._props.forceAutoHeight || this.layoutDoc._layout_autoHeight) && !this._props.ignoreAutoHeight; } // prettier-ignore @computed get textHeight() { return NumCast(this.dataDoc[this.fieldKey + '_height']); } // prettier-ignore @computed get scrollHeight() { return NumCast(this.dataDoc[this.fieldKey + '_scrollHeight']); } // prettier-ignore - @computed get sidebarHeight() { return !this.sidebarWidth() ? 0 : NumCast(this.dataDoc[this.SidebarKey + '_height']); } // prettier-ignore + @computed get sidebarHeight() { return !this.sidebarWidth() ? 0 : NumCast(this.dataDoc[this.sidebarKey + '_height']); } // prettier-ignore @computed get titleHeight() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.HeaderMargin) as number || 0; } // prettier-ignore @computed get layout_autoHeightMargins() { return this.titleHeight + NumCast(this.layoutDoc._layout_autoHeightMargins); } // prettier-ignore - @computed get config() { - this._rules = new RichTextRules(this.Document, this); - return FormattedTextBox.MakeConfig(this._rules, this._props); - } - - public get EditorView() { - return this._editorView; - } - public get SidebarKey() { - return this.fieldKey + '_sidebar'; - } - public makeAIFlashcards: () => void = unimplementedFunction; - public addToCollection: ((doc: Doc | Doc[], annotationKey?: string | undefined) => boolean) | undefined; - - public static PasteOnLoad: ClipboardEvent | undefined; - public static DontSelectInitialText = false; // whether initial text should be selected or not - public static SelectOnLoadChar = ''; + @computed get config() { return FormattedTextBox.MakeConfig(this._rules = new RichTextRules(this.Document, this), this._props); } // prettier-ignore + @computed get sidebarKey() { return this.fieldKey + '_sidebar'; } // prettier-ignore constructor(props: FormattedTextBoxProps) { super(props); @@ -165,6 +162,11 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB this._recordingStart = Date.now(); } + public get EditorView() { return this._editorView; } // prettier-ignore + + // public makeAIFlashcards: () => void = unimplementedFunction; + public addToCollection: ((doc: Doc | Doc[], annotationKey?: string | undefined) => boolean) | undefined; + // removes all hyperlink anchors for the removed linkDoc // TODO: bcz: Argh... if a section of text has multiple anchors, this should just remove the intended one. // but since removing one anchor from the list of attr anchors isn't implemented, this will end up removing nothing. @@ -328,10 +330,10 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB const rtField = (layoutData !== prevData ? layoutData : undefined) ?? protoData; if (this._applyingChange !== this.fieldKey && (force || textChange || removeSelection(newJson) !== removeSelection(prevData?.Data))) { this._applyingChange = this.fieldKey; - textChange && (dataDoc[this.fieldKey + '_modificationDate'] = new DateField(new Date(Date.now()))); if ((!prevData && !protoData && !layoutData) || newText || (!newText && !protoData && !layoutData)) { // if no template, or there's text that didn't come from the layout template, write it to the document. (if this is driven by a template, then this overwrites the template text which is intended) if (force || ((this._finishingLink || this._props.isContentActive() || this._inDrop) && (textChange || removeSelection(newJson) !== removeSelection(prevData?.Data)))) { + textChange && (dataDoc[this.fieldKey + '_modificationDate'] = new DateField(new Date(Date.now()))); const numstring = NumCast(dataDoc[this.fieldKey], null); dataDoc[this.fieldKey] = numstring !== undefined ? Number(newText) : newText || (DocCast(dataDoc.proto)?.[this.fieldKey] === undefined && this.layoutDoc[this.fieldKey] === undefined) ? new RichTextField(newJson, newText) : undefined; @@ -340,6 +342,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB unchanged = false; } } else if (rtField) { + textChange && (dataDoc[this.fieldKey + '_modificationDate'] = new DateField(new Date(Date.now()))); // if we've deleted all the text in a note driven by a template, then restore the template data dataDoc[this.fieldKey] = undefined; this._editorView.updateState(EditorState.fromJSON(this.config, JSON.parse(rtField.Data))); @@ -709,18 +712,13 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB this.layoutDoc[DocCss] = this.layoutDoc[DocCss] + 1; // css changes happen outside of react/mobx. so we need to set a flag that will notify anyone interested in layout changes triggered by css changes (eg., CollectionLinkView) }; - @observable _showSidebar = false; - @computed get SidebarShown() { - return !!(this._showSidebar || this.layoutDoc._layout_showSidebar); - } - @action toggleSidebar = (preview: boolean = false) => { const defaultSidebar = 250; const prevWidth = 1 - this.sidebarWidth() / DivWidth(this._ref.current!); if (preview) this._showSidebar = true; else { - this.layoutDoc[this.SidebarKey + '_freeform_scale_max'] = 1; + this.layoutDoc[this.sidebarKey + '_freeform_scale_max'] = 1; this.layoutDoc._layout_showSidebar = (this.layoutDoc._layout_sidebarWidthPercent = StrCast(this.layoutDoc._layout_sidebarWidthPercent, '0%') === '0%' ? `${(defaultSidebar / (NumCast(this.layoutDoc._width) + defaultSidebar)) * 100}%` : '0%') !== '0%'; } @@ -785,6 +783,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB isTargetToggler = (anchor: Doc) => BoolCast(anchor.followLinkToggle); specificContextMenu = (e: React.MouseEvent): void => { + if (this._props.dontSelect?.()) return; const cm = ContextMenu.Instance; let target: Element | HTMLElement | null = e.target as HTMLElement; // hrefs are stored on the database of the <a> node that wraps the hyerlink <span> @@ -878,6 +877,11 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB ); const appearance = cm.findByDescription('Appearance...'); const appearanceItems = appearance?.subitems ?? []; + // appearanceItems.push({ + // description: 'Find image tags', + // event: this.findImageTags, + // icon: !this.Document._layout_noSidebar ? 'eye-slash' : 'eye', + // }); appearanceItems.push({ description: !this.Document._layout_noSidebar ? 'Hide Sidebar Handle' : 'Show Sidebar Handle', @@ -941,7 +945,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB icon: 'star', }); optionItems.push({ description: `Generate Dall-E Image`, event: () => this.generateImage(), icon: 'star' }); - optionItems.push({ description: `Make AI Flashcards`, event: () => this.makeAIFlashcards(), icon: 'lightbulb' }); + // optionItems.push({ description: `Make AI Flashcards`, event: () => this.makeAIFlashcards(), icon: 'lightbulb' }); optionItems.push({ description: `Ask GPT-3`, event: this.askGPT, icon: 'lightbulb' }); this._props.renderDepth && optionItems.push({ @@ -966,6 +970,61 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB !help && cm.addItem({ description: 'Help...', subitems: helpItems, icon: 'eye' }); }; + findImageTags = async () => { + const c = this.ProseRef?.getElementsByTagName('img'); + if (c) { + for (let i of c) { + console.log(i); + + // console.log(canvas.toDataURL()); + // canvas.style.zIndex = '2000000'; + // document.body.appendChild(canvas); + if (i.className !== 'ProseMirror-separator') this.getImageDesc(i.src); + } + } + // console.log('HI' + this.ProseRef?.getElementsByTagName('img')); + }; + + static imageUrlToBase64 = async (imageUrl: string): Promise<string> => { + try { + const response = await fetch(imageUrl); + const blob = await response.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + getImageDesc = async (u: string) => { + // if (StrCast(this.dataDoc.description)) return StrCast(this.dataDoc.description); // Return existing description + // const { href } = (u as URLField).url; + const hrefParts = u.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + try { + const hrefBase64 = await FormattedTextBox.imageUrlToBase64(u); + const response = await gptImageLabel( + hrefBase64, + 'Make flashcards out of this text and image with each question and answer labeled as question and answer. Do not label each flashcard and do not include asterisks: ' + (this.dataDoc.text as RichTextField)?.Text + ); + //const response = await gptImageLabel(u, 'Make flashcards out of this text with each question and answer labeled as question and answer. Do not label each flashcard and do not include asterisks: '); + // console.log(response); + AnchorMenu.Instance.transferToFlashcard(response || 'Something went wrong', NumCast(this.dataDoc['x']), NumCast(this.dataDoc['y'])); + // this._props.addto_; + // this.Document[DocData].description = response.trim(); + // return response; // Return the response + } catch (error) { + console.log('Error'); + } + // return ''; + }; + animateRes = (resIndex: number, newText: string) => { if (resIndex < newText.length) { const marks = this._editorView?.state.storedMarks ?? []; @@ -976,7 +1035,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB askGPT = action(async () => { try { - GPTPopup.Instance.setSidebarId(this.SidebarKey); + GPTPopup.Instance.setSidebarId(this.sidebarKey); GPTPopup.Instance.addDoc = this.sidebarAddDocument; const res = await gptAPICall((this.dataDoc.text as RichTextField)?.Text, GPTCallType.COMPLETION); if (!res) { @@ -1103,7 +1162,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB } getView = async (doc: Doc, options: FocusViewOptions) => { - if (DocListCast(this.dataDoc[this.SidebarKey]).find(anno => Doc.AreProtosEqual(doc.layout_unrendered ? DocCast(doc.annotationOn) : doc, anno))) { + if (DocListCast(this.dataDoc[this.sidebarKey]).find(anno => Doc.AreProtosEqual(doc.layout_unrendered ? DocCast(doc.annotationOn) : doc, anno))) { if (!this.SidebarShown) { this.toggleSidebar(false); options.didMove = true; @@ -1313,7 +1372,33 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB { fireImmediately: true } ); this.tryUpdateScrollHeight(); + + if (this.Document.image) { + // const node = schema.nodes.dashDoc.create({ + // width: 200, + // height: 200, + // title: 'dashDoc', + // docId: DocCast(this.Document.image)[Id], + // float: 'unset', + // }); + + // DocCast(this.Document.image)._freeform_fitContentsToBox = true; + // Doc.SetContainer(DocCast(this.Document.image), this.Document); + // const view = this._editorView!; + // try { + // this._inDrop = true; + // const pos = view.posAtCoords({ left: 0, top: 0 })?.pos; + // pos && view.dispatch(view.state.tr.insert(pos, node)); + // } catch (err) { + // console.log('Drop failed', err); + // } + // console.log('LKSDFLJ'); + this.addDocument?.(DocCast(this.Document.image)); + } + + //if (this.Document.image) this.addDocument?.(DocCast(this.Document.image)); setTimeout(this.tryUpdateScrollHeight, 250); + AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; } clipboardTextSerializer = (slice: Slice): string => { @@ -1421,7 +1506,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB return true; }, dispatchTransaction: this.dispatchTransaction, - nodeViews: FormattedTextBox.nodeViews(this), + nodeViews: FormattedTextBox._nodeViews(this), clipboardTextSerializer: this.clipboardTextSerializer, handlePaste: this.handlePaste, }); @@ -1556,7 +1641,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB } }; onSelectEnd = () => { - GPTPopup.Instance.setSidebarId(this.SidebarKey); + GPTPopup.Instance.setSidebarId(this.sidebarKey); GPTPopup.Instance.addDoc = this.sidebarAddDocument; document.removeEventListener('pointerup', this.onSelectEnd); }; @@ -1696,7 +1781,6 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB if (RichTextMenu.Instance?.view === this._editorView && !this._props.rootSelected?.()) { RichTextMenu.Instance?.updateMenu(undefined, undefined, undefined, undefined); } - FormattedTextBox._hadSelection = window.getSelection()?.toString() !== ''; // this is the markdown for @<published name> document publishing to Doc.myPublishedDocs const match = RTFCast(this.Document[this.fieldKey])?.Text.match(/^(@[a-zA-Z][a-zA-Z_0-9 -]*[a-zA-Z_0-9-]+)/); @@ -1810,14 +1894,14 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB }; fitContentsToBox = () => BoolCast(this.Document._freeform_fitContentsToBox); sidebarContentScaling = () => (this._props.NativeDimScaling?.() || 1) * NumCast(this.layoutDoc._freeform_scale, 1); - sidebarAddDocument = (doc: Doc | Doc[], sidebarKey: string = this.SidebarKey) => { + sidebarAddDocument = (doc: Doc | Doc[], sidebarKey: string = this.sidebarKey) => { if (!this.layoutDoc._layout_showSidebar) this.toggleSidebar(); return this.addDocument(doc, sidebarKey); }; - sidebarMoveDocument = (doc: Doc | Doc[], targetCollection: Doc | undefined, addDocument: (doc: Doc | Doc[]) => boolean) => this.moveDocument(doc, targetCollection, addDocument, this.SidebarKey); - sidebarRemDocument = (doc: Doc | Doc[]) => this.removeDocument(doc, this.SidebarKey); + sidebarMoveDocument = (doc: Doc | Doc[], targetCollection: Doc | undefined, addDocument: (doc: Doc | Doc[]) => boolean) => this.moveDocument(doc, targetCollection, addDocument, this.sidebarKey); + sidebarRemDocument = (doc: Doc | Doc[]) => this.removeDocument(doc, this.sidebarKey); setSidebarHeight = (height: number) => { - this.dataDoc[this.SidebarKey + '_height'] = height; + this.dataDoc[this.sidebarKey + '_height'] = height; }; sidebarWidth = () => (Number(this.layout_sidebarWidthPercent.substring(0, this.layout_sidebarWidthPercent.length - 1)) / 100) * this._props.PanelWidth(); sidebarScreenToLocal = () => @@ -1847,7 +1931,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB } @computed get sidebarHandle() { TraceMobx(); - const annotated = DocListCast(this.dataDoc[this.SidebarKey]).filter(d => d?.author).length; + const annotated = DocListCast(this.dataDoc[this.sidebarKey]).filter(d => d?.author).length; const color = !annotated ? Colors.WHITE : Colors.BLACK; const backgroundColor = !annotated ? (this.sidebarWidth() ? Colors.MEDIUM_BLUE : Colors.BLACK) : (this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.WidgetColor + (annotated ? ':annotated' : '')) as string); @@ -1901,7 +1985,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB PanelWidth={this.sidebarWidth} xPadding={0} yPadding={0} - viewField={this.SidebarKey} + viewField={this.sidebarKey} isAnnotationOverlay={false} select={emptyFunction} isAnyChildContentActive={returnFalse} @@ -1916,14 +2000,14 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB fitContentsToBox={this.fitContentsToBox} noSidebar treeViewHideTitle - fieldKey={this.layoutDoc[this.SidebarKey + '_type_collection'] === 'translation' ? `${this.fieldKey}_translation` : `${this.fieldKey}_sidebar`} + fieldKey={this.layoutDoc[this.sidebarKey + '_type_collection'] === 'translation' ? `${this.fieldKey}_translation` : `${this.fieldKey}_sidebar`} /> </div> ); }; return ( <div className={'formattedTextBox-sidebar' + (Doc.ActiveTool !== InkTool.None ? '-inking' : '')} style={{ width: `${this.layout_sidebarWidthPercent}`, backgroundColor: `${this.sidebarColor}` }}> - {renderComponent(StrCast(this.layoutDoc[this.SidebarKey + '_type_collection']))} + {renderComponent(StrCast(this.layoutDoc[this.sidebarKey + '_type_collection']))} </div> ); } @@ -1965,6 +2049,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB </Tooltip> ); } + get fieldKey() { return this._fieldKey; } @@ -1993,10 +2078,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB e.stopPropagation(); } }; - @computed get fontColor() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontColor) as string; } // prettier-ignore - @computed get fontSize() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontSize) as string; } // prettier-ignore - @computed get fontFamily() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontFamily) as string; } // prettier-ignore - @computed get fontWeight() { return this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.FontWeight) as string; } // prettier-ignore + render() { TraceMobx(); const scale = this._props.NativeDimScaling?.() || 1; @@ -2071,7 +2153,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB /> </div> {this.noSidebar || !this.SidebarShown || this.layout_sidebarWidthPercent === '0%' ? null : this.sidebarCollection} - {this.noSidebar || this.Document._layout_noSidebar || this.Document._createDocOnCR || this.layoutDoc._chromeHidden ? null : this.sidebarHandle} + {this.noSidebar || this.Document._layout_noSidebar || this.Document._createDocOnCR || this.layoutDoc._chromeHidden || this.Document.quiz ? null : this.sidebarHandle} {this.audioHandle} {this.layoutDoc._layout_enableAltContentUI && !this.layoutDoc._chromeHidden ? this.overlayAlternateIcon : null} </div> diff --git a/src/client/views/nodes/formattedText/FormattedTextBoxComment.scss b/src/client/views/nodes/formattedText/FormattedTextBoxComment.scss index 55b8446e9..bc0810f22 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBoxComment.scss +++ b/src/client/views/nodes/formattedText/FormattedTextBoxComment.scss @@ -13,7 +13,7 @@ box-shadow: 3px 3px 1.5px grey; max-width: 400; max-height: 235; - height:max-content; + height: max-content; .formattedTextBox-tooltipText { height: max-content; text-overflow: ellipsis; @@ -21,7 +21,7 @@ } .formattedTextBox-tooltip:before { - content: ""; + content: ''; height: 0; width: 0; position: absolute; @@ -34,7 +34,7 @@ } .formattedTextBox-tooltip:after { - content: ""; + content: ''; height: 0; width: 0; position: absolute; @@ -44,4 +44,4 @@ border: 5px solid transparent; border-bottom-width: 0; border-top-color: white; -}
\ No newline at end of file +} diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index 03585a8b7..6dd036cf6 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -15,6 +15,7 @@ import { LinkPopup } from '../linking/LinkPopup'; import { DocumentView } from '../nodes/DocumentView'; import './AnchorMenu.scss'; import { GPTPopup, GPTPopupMode } from './GPTPopup/GPTPopup'; +import ReactLoading from 'react-loading'; @observer export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { @@ -24,6 +25,9 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { private _disposer: IReactionDisposer | undefined; private _commentRef = React.createRef<HTMLDivElement>(); private _cropRef = React.createRef<HTMLDivElement>(); + @observable private _loading = false; + // @observable protected _top: number = -300; + // @observable protected _left: number = -300; constructor(props: AntimodeMenuProps) { super(props); @@ -38,11 +42,21 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { // GPT additions @observable private _selectedText: string = ''; + @observable private _x: number = 0; + @observable private _y: number = 0; @action public setSelectedText = (txt: string) => { this._selectedText = txt.trim(); }; + @action + public setLocation = (x: number, y: number) => { + this._x = x; + this._y = y; + }; + @computed public get selectedText() { + return this._selectedText; + } public onMakeAnchor: () => Opt<Doc> = () => undefined; // Method to get anchor from text search public OnCrop: (e: PointerEvent) => void = unimplementedFunction; @@ -57,6 +71,10 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { public MakeTargetToggle: () => void = unimplementedFunction; public ShowTargetTrail: () => void = unimplementedFunction; public IsTargetToggler: () => boolean = returnFalse; + public gptFlashcards: () => void = unimplementedFunction; + public makeLabels: () => void = unimplementedFunction; + public marqueeWidth = 0; + public marqueeHeight = 0; public get Active() { return this._left > 0; } @@ -99,30 +117,33 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { * Invokes the API with the selected text and stores it in the selected text. * @param e pointer down event */ - gptFlashcards = async () => { - const queryText = this._selectedText; - try { - const res = await gptAPICall(queryText, GPTCallType.FLASHCARD); - console.log(res); - GPTPopup.Instance.setText(res || 'Something went wrong.'); - this.transferToFlashcard(res || 'Something went wrong'); - } catch (err) { - console.error(err); - } - GPTPopup.Instance.setLoading(false); - }; + // gptPDFFlashcards = async () => { + // const queryText = this._selectedText; + // this._loading = true; + // try { + // const res = await gptAPICall(queryText, GPTCallType.FLASHCARD); + // console.log(res); + // // GPTPopup.Instance.setText(res || 'Something went wrong.'); + // this.transferToFlashcard(res || 'Something went wrong'); + // } catch (err) { + // console.error(err); + // } + // // GPTPopup.Instance.setLoading(false); + // }; /* * Transfers the flashcard text generated by GPT on flashcards and creates a collection out them. */ - transferToFlashcard = (text: string) => { + + transferToFlashcard = (text: string, x: number, y: number) => { // put each question generated by GPT on the front of the flashcard - const senArr = text.split('Question'); - const collectionArr: Doc[] = []; + var senArr = text.trim().split('Question: '); + var collectionArr: Doc[] = []; for (let i = 1; i < senArr.length; i++) { console.log('Arr ' + i + ': ' + senArr[i]); const newDoc = Docs.Create.ComparisonDocument(senArr[i], { _layout_isFlashcard: true, _width: 300, _height: 300 }); newDoc.text = senArr[i]; + collectionArr.push(newDoc); } // create a new carousel collection of these flashcards @@ -133,7 +154,14 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { _layout_autoHeight: true, }); + console.log(collectionArr); + newCol.x = x; + newCol.y = y; + console.log(this._x); + newCol.zIndex = 1000; + this.addToCollection?.(newCol); + this._loading = false; }; pointerDown = (e: React.PointerEvent) => { @@ -219,12 +247,8 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { /> )} {/* Adds a create flashcards option to the anchor menu, which calls the gptFlashcard method. */} - <IconButton - tooltip="Create flashcards" // - onPointerDown={this.gptFlashcards} - icon={<FontAwesomeIcon icon="id-card" size="lg" />} - color={SettingsManager.userColor} - /> + <IconButton tooltip="Create flashcards" onPointerDown={this.gptFlashcards} icon={<FontAwesomeIcon icon="id-card" size="lg" />} color={SettingsManager.userColor} /> + <IconButton tooltip="Create labels" onPointerDown={this.makeLabels} icon={<FontAwesomeIcon icon="tag" size="lg" />} color={SettingsManager.userColor} /> {AnchorMenu.Instance.OnAudio === unimplementedFunction ? null : ( <IconButton tooltip="Click to Record Annotation" // @@ -250,6 +274,11 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { /> </div> )} + {/* {this._loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={30} width={30} color={'white'} /> + </div> + ) : null} */} </> ) : ( <> diff --git a/src/client/views/pdf/Annotation.scss b/src/client/views/pdf/Annotation.scss index 1de60ffed..26856b74e 100644 --- a/src/client/views/pdf/Annotation.scss +++ b/src/client/views/pdf/Annotation.scss @@ -7,4 +7,21 @@ &:hover { cursor: pointer; } -}
\ No newline at end of file +} +// .loading-spinner { +// display: flex; +// justify-content: center; +// align-items: center; +// height: 90%; +// width: 93%; +// left: 10; +// font-size: 20px; +// font-weight: bold; +// color: #0b0a0a; +// } + +// @keyframes spin { +// to { +// transform: rotate(360deg); +// } +// } diff --git a/src/client/views/pdf/PDFViewer.scss b/src/client/views/pdf/PDFViewer.scss index d3dd9f727..e70102ce9 100644 --- a/src/client/views/pdf/PDFViewer.scss +++ b/src/client/views/pdf/PDFViewer.scss @@ -107,3 +107,24 @@ .pdfViewerDash-interactive { pointer-events: all; } + +.loading-spinner { + position: absolute; + display: flex; + justify-content: center; + align-items: center; + height: 100%; + width: 100%; + // left: 50%; + // top: 50%; + z-index: 200; + font-size: 20px; + font-weight: bold; + color: #17175e; +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index dee0edfae..02d310f7d 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -28,6 +28,12 @@ import { AnchorMenu } from './AnchorMenu'; import { Annotation } from './Annotation'; import { GPTPopup } from './GPTPopup/GPTPopup'; import './PDFViewer.scss'; +import { GPTCallType, gptAPICall } from '../../apis/gpt/GPT'; +import ReactLoading from 'react-loading'; +// import html2canvas from 'html2canvas'; +// import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition'; + +// pdfjsLib.GlobalWorkerOptions.workerSrc = `/assets/pdf.worker.js`; // The workerSrc property shall be specified. // Pdfjs.GlobalWorkerOptions.workerSrc = 'https://unpkg.com/pdfjs-dist@4.4.168/build/pdf.worker.mjs'; @@ -58,12 +64,50 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { super(props); makeObservable(this); } + // @observable transcriptRef = React.createRef(); + // @observable startBtnRef = React.createRef(); + // @observable stopBtnRef = React.createRef(); + // @observable transcriptElement = ''; + + // handleResult = (e: SpeechRecognitionEvent) => { + // let interimTranscript = ''; + // let finalTranscript = ''; + // console.log('H'); + // for (let i = e.resultIndex; i < e.results.length; i++) { + // const transcript = e.results[i][0].transcript; + // if (e.results[i].isFinal) { + // finalTranscript += transcript; + // } else { + // interimTranscript += transcript; + // } + // } + // console.log(interimTranscript); + // this.transcriptElement = finalTranscript || interimTranscript; + // }; + + // startListening = () => { + // const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + // if (SpeechRecognition) { + // console.log('here'); + // const recognition = new SpeechRecognition(); + // recognition.continuous = true; // Continue listening even if the user pauses + // recognition.interimResults = true; // Show interim results + // recognition.lang = 'en-US'; // Set language (optional) + // recognition.onresult = this.handleResult.bind(this); + // // recognition.onend = this.handleEnd.bind(this); + + // recognition.start(); + // // this.handleResult; + // // recognition.stop(); + // } + // }; @observable _pageSizes: { width: number; height: number }[] = []; @observable _savedAnnotations = new ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>(); @observable _textSelecting = true; @observable _showWaiting = true; @observable Index: number = -1; + @observable private _loading = false; private _pdfViewer!: PDFJSViewer.PDFViewer; private _styleRule: number | undefined; // stylesheet rule for making hyperlinks clickable @@ -394,6 +438,122 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { } }; + gptPDFFlashcards = async () => { + // const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + // if (SpeechRecognition) { + // this.recognition = new SpeechRecognition(); + // this.recognition.continuous = true; // Continue listening even if the user pauses + // this.recognition.interimResults = true; // Show interim results + // this.recognition.lang = 'en-US'; // Set language (optional) + + // this.recognition.onresult = this.handleResult; + // this.recognition.onerror = this.handleError; + // this.recognition.onend = this.handleEnd; + // } else { + // console.error("Browser doesn't support Speech Recognition API"); + // } + // const Dictaphone = () => { + // const { transcript, resetTranscript } = useSpeechRecognition(); + + // if (!SpeechRecognition.browserSupportsSpeechRecognition()) { + // return null; + // } + + // return ( + // <div> + // <button onClick={e => SpeechRecognition.startListening}>Start</button> + // <button onClick={e => SpeechRecognition.stopListening}>Stop</button> + // <button onClick={resetTranscript}>Reset</button> + // <p>{transcript}</p> + // </div> + // ); + // }; + // const grammar = + // '#JSGF V1.0; grammar colors; public <color> = aqua | azure | beige | bisque | black | blue | brown | chocolate | coral | crimson | cyan | fuchsia | ghostwhite | gold | goldenrod | gray | green | indigo | ivory | khaki | lavender | lime | linen | magenta | maroon | moccasin | navy | olive | orange | orchid | peru | pink | plum | purple | red | salmon | sienna | silver | snow | tan | teal | thistle | tomato | turquoise | violet | white | yellow ;'; + // const recognition = new SpeechRecognition(); + // const speechRecognitionList = new SpeechGrammarList(); + // speechRecognitionList.addFromString(grammar, 1); + // recognition.grammars = speechRecognitionList; + // recognition.continuous = false; + // recognition.lang = 'en-US'; + // recognition.interimResults = false; + // recognition.maxAlternatives = 1; + + // const diagnostic = document.querySelector('.output'); + // const bg = document.querySelector('html'); + + // document.body.onclick = () => { + // recognition.start(); + // console.log('Ready to receive a color command.'); + // }; + + // recognition.onresult = event => { + // const color = event.results[0][0].transcript; + // diagnostic!.textContent = `Result received: ${color}`; + // bg!.style.backgroundColor = color; + // }; + + //const SpeechRecognition = SpeechRecognition || webkitSpeechRecognition; + + // recognition.continous = true; + // recognition.interimResults = true; + // recognition.lang = 'en-US'; + + const queryText = this._selectionText; + + // const canvas = await html2canvas(); + // const image = canvas.toDataURL("image/png", 1.0); + // (window as any) + // .html2canvas(this._marqueeref, { + // x: 100, + // y: 100, + // width: 100, + // height: 100, + // }) + // .then((canvas: HTMLCanvasElement) => { + // const img = canvas.toDataURL('image/png'); + + // const link = document.createElement('a'); + // link.href = img; + // link.download = 'screenshot.png'; + + // document.body.appendChild(link); + // link.click(); + // link.remove(); + // }); + + // var range = window.getSelection()?.getRangeAt(0); + // var selectionContents = range?.extractContents(); + // var div = document.createElement("div"); + // div.style.color = "yellow"; + // div.appendChild(selectionContents!); + // range!.insertNode(div); + + // const canvas = document.createElement('canvas'); + // const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + // const w = AnchorMenu.Instance.marqueeWidth * scaling; + // const h = AnchorMenu.Instance.marqueeHeight * scaling; + // canvas.width = w; + // canvas.height = h; + // const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions + // if (ctx) { + // this._marqueeref && ctx.drawImage(div, NumCast(this._marqueeref.current?.left) * scaling, NumCast(this._marqueeref.current?.top) * scaling, w, h, 0, 0, w, h); + // } + this._loading = true; + try { + if (this._selectionText === '') { + } + const res = await gptAPICall(queryText, GPTCallType.FLASHCARD); + + console.log(res); + AnchorMenu.Instance.transferToFlashcard(res || 'Something went wrong', NumCast(this._props.layoutDoc['x']), NumCast(this._props.layoutDoc['y'])); + this._selectionText = ''; + } catch (err) { + console.error(err); + } + this._loading = false; + }; + @action finishMarquee = (/* x?: number, y?: number */) => { this._getAnchor = AnchorMenu.Instance?.GetAnchor; @@ -411,8 +571,10 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { document.removeEventListener('pointerup', this.onSelectEnd); const sel = window.getSelection(); + if (sel) { AnchorMenu.Instance.setSelectedText(sel.toString()); + AnchorMenu.Instance.setLocation(NumCast(this._props.layoutDoc['x']), NumCast(this._props.layoutDoc['y'])); } if (sel?.type === 'Range') { @@ -424,6 +586,7 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { GPTPopup.Instance.addDoc = this._props.sidebarAddDoc; // allows for creating collection AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; + AnchorMenu.Instance.gptFlashcards = this.gptPDFFlashcards; }; @action @@ -451,6 +614,7 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { this._mainCont.current!.style.transform = ''; } this._selectionContent = selRange.cloneContents(); + this._selectionText = this._selectionContent?.textContent || ''; // clear selection @@ -612,6 +776,11 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { /> )} </div> + {this._loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={80} width={80} color={'blue'} /> + </div> + ) : null} </div> ); } diff --git a/src/client/views/webcam/WebCamLogic.js b/src/client/views/webcam/WebCamLogic.js new file mode 100644 index 000000000..5f6202bc8 --- /dev/null +++ b/src/client/views/webcam/WebCamLogic.js @@ -0,0 +1,292 @@ +'use strict'; +import io from "socket.io-client"; + +var socket; +var isChannelReady = false; +var isInitiator = false; +var isStarted = false; +var localStream; +var pc; +var remoteStream; +var turnReady; +var room; + +export function initialize(roomName, handlerUI) { + + var pcConfig = { + 'iceServers': [{ + 'urls': 'stun:stun.l.google.com:19302' + }] + }; + + // Set up audio and video regardless of what devices are present. + var sdpConstraints = { + offerToReceiveAudio: true, + offerToReceiveVideo: true + }; + + ///////////////////////////////////////////// + + room = roomName; + + socket = io.connect(`${window.location.protocol}//${window.location.hostname}:4321`); + + if (room !== '') { + socket.emit('create or join', room); + console.log('Attempted to create or join room', room); + } + + socket.on('created', function (room) { + console.log('Created room ' + room); + isInitiator = true; + }); + + socket.on('full', function (room) { + console.log('Room ' + room + ' is full'); + }); + + socket.on('join', function (room) { + console.log('Another peer made a request to join room ' + room); + console.log('This peer is the initiator of room ' + room + '!'); + isChannelReady = true; + }); + + socket.on('joined', function (room) { + console.log('joined: ' + room); + isChannelReady = true; + }); + + socket.on('log', function (array) { + console.log.apply(console, array); + }); + + //////////////////////////////////////////////// + + + // This client receives a message + socket.on('message', function (message) { + console.log('Client received message:', message); + if (message === 'got user media') { + maybeStart(); + } else if (message.type === 'offer') { + if (!isInitiator && !isStarted) { + maybeStart(); + } + pc.setRemoteDescription(new RTCSessionDescription(message)); + doAnswer(); + } else if (message.type === 'answer' && isStarted) { + pc.setRemoteDescription(new RTCSessionDescription(message)); + } else if (message.type === 'candidate' && isStarted) { + var candidate = new RTCIceCandidate({ + sdpMLineIndex: message.label, + candidate: message.candidate + }); + pc.addIceCandidate(candidate); + } else if (message === 'bye' && isStarted) { + handleRemoteHangup(); + } + }); + + //////////////////////////////////////////////////// + + var localVideo = document.querySelector('#localVideo'); + var remoteVideo = document.querySelector('#remoteVideo'); + + const gotStream = (stream) => { + console.log('Adding local stream.'); + localStream = stream; + localVideo.srcObject = stream; + sendMessage('got user media'); + if (isInitiator) { + maybeStart(); + } + } + + + navigator.mediaDevices.getUserMedia({ + audio: true, + video: true + }) + .then(gotStream) + .catch(function (e) { + alert('getUserMedia() error: ' + e.name); + }); + + + + var constraints = { + video: true + }; + + console.log('Getting user media with constraints', constraints); + + const requestTurn = (turnURL) => { + var turnExists = false; + for (var i in pcConfig.iceServers) { + if (pcConfig.iceServers[i].urls.substr(0, 5) === 'turn:') { + turnExists = true; + turnReady = true; + break; + } + } + if (!turnExists) { + console.log('Getting TURN server from ', turnURL); + // No TURN server. Get one from computeengineondemand.appspot.com: + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function () { + if (xhr.readyState === 4 && xhr.status === 200) { + var turnServer = JSON.parse(xhr.responseText); + console.log('Got TURN server: ', turnServer); + pcConfig.iceServers.push({ + 'urls': 'turn:' + turnServer.username + '@' + turnServer.turn, + 'credential': turnServer.password + }); + turnReady = true; + } + }; + xhr.open('GET', turnURL, true); + xhr.send(); + } + } + + + + + if (location.hostname !== 'localhost') { + requestTurn( + `${window.location.origin}/corsProxy/${encodeURIComponent("https://computeengineondemand.appspot.com/turn?username=41784574&key=4080218913")}` + ); + } + + const maybeStart = () => { + console.log('>>>>>>> maybeStart() ', isStarted, localStream, isChannelReady); + if (!isStarted && typeof localStream !== 'undefined' && isChannelReady) { + console.log('>>>>>> creating peer connection'); + createPeerConnection(); + pc.addStream(localStream); + isStarted = true; + console.log('isInitiator', isInitiator); + if (isInitiator) { + doCall(); + } + } + }; + + window.onbeforeunload = function () { + sendMessage('bye'); + }; + + ///////////////////////////////////////////////////////// + + const createPeerConnection = () => { + try { + pc = new RTCPeerConnection(null); + pc.onicecandidate = handleIceCandidate; + pc.onaddstream = handleRemoteStreamAdded; + pc.onremovestream = handleRemoteStreamRemoved; + console.log('Created RTCPeerConnnection'); + } catch (e) { + console.log('Failed to create PeerConnection, exception: ' + e.message); + alert('Cannot create RTCPeerConnection object.'); + return; + } + } + + const handleIceCandidate = (event) => { + console.log('icecandidate event: ', event); + if (event.candidate) { + sendMessage({ + type: 'candidate', + label: event.candidate.sdpMLineIndex, + id: event.candidate.sdpMid, + candidate: event.candidate.candidate + }); + } else { + console.log('End of candidates.'); + } + } + + const handleCreateOfferError = (event) => { + console.log('createOffer() error: ', event); + } + + const doCall = () => { + console.log('Sending offer to peer'); + pc.createOffer(setLocalAndSendMessage, handleCreateOfferError); + } + + const doAnswer = () => { + console.log('Sending answer to peer.'); + pc.createAnswer().then( + setLocalAndSendMessage, + onCreateSessionDescriptionError + ); + } + + const setLocalAndSendMessage = (sessionDescription) => { + pc.setLocalDescription(sessionDescription); + console.log('setLocalAndSendMessage sending message', sessionDescription); + sendMessage(sessionDescription); + } + + const onCreateSessionDescriptionError = (error) => { + trace('Failed to create session description: ' + error.toString()); + } + + + + const handleRemoteStreamAdded = (event) => { + console.log('Remote stream added.'); + remoteStream = event.stream; + remoteVideo.srcObject = remoteStream; + handlerUI(); + + }; + + const handleRemoteStreamRemoved = (event) => { + console.log('Remote stream removed. Event: ', event); + } +} + +export function hangup() { + console.log('Hanging up.'); + stop(); + sendMessage('bye'); + if (localStream) { + localStream.getTracks().forEach(track => track.stop()); + } +} + +function stop() { + isStarted = false; + if (pc) { + pc.close(); + } + pc = null; +} + +function handleRemoteHangup() { + console.log('Session terminated.'); + stop(); + isInitiator = false; + if (localStream) { + localStream.getTracks().forEach(track => track.stop()); + } +} + +function sendMessage(message) { + console.log('Client sending message: ', message); + socket.emit('message', message, room); +}; + +export function refreshVideos() { + var localVideo = document.querySelector('#localVideo'); + var remoteVideo = document.querySelector('#remoteVideo'); + if (localVideo) { + localVideo.srcObject = localStream; + } + if (remoteVideo) { + remoteVideo.srcObject = remoteStream; + } + +}
\ No newline at end of file |