import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; import axios from 'axios'; import { IReactionDisposer, action, computed, makeObservable, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import ReactLoading from 'react-loading'; import { imageUrlToBase64, returnFalse, returnNone, returnTrue, returnZero, setupMoveUpEvents } from '../../../ClientUtils'; import { emptyFunction } from '../../../Utils'; import { Doc, Opt } from '../../../fields/Doc'; import { Animation, DocData } from '../../../fields/DocSymbols'; import { RichTextField } from '../../../fields/RichTextField'; import { BoolCast, Cast, DocCast, NumCast, RTFCast, StrCast, toList } from '../../../fields/Types'; import { nullAudio } from '../../../fields/URLField'; import { GPTCallType, gptAPICall, gptImageLabel } from '../../apis/gpt/GPT'; import { DocUtils, FollowLinkScript } from '../../documents/DocUtils'; import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { DragManager } from '../../util/DragManager'; import { dropActionType } from '../../util/DropActionTypes'; import { undoable } from '../../util/UndoManager'; import { ContextMenu } from '../ContextMenu'; import { ViewBoxAnnotatableComponent } from '../DocComponent'; import { PinDocView, PinProps } from '../PinFuncs'; import { StyleProp } from '../StyleProp'; import { flashcardRevealOp, practiceMode } from '../collections/FlashcardPracticeUI'; import { CollectionFreeFormView } from '../collections/collectionFreeForm'; import '../pdf/GPTPopup/GPTPopup.scss'; import './ComparisonBox.scss'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { TraceMobx } from '../../../fields/util'; const API_URL = 'https://api.unsplash.com/search/photos'; /** * This view serves two distinct functions depending on the revealOp field ('slide' or 'flip) * 1) ('slide') - provides a before/after animated sliding transition between two Docs * 2) ('flip') - provides a question/answer flip between two Docs * And a third function that overrides the first two if the doc's container has its 'practiceMode' set to 'quiz' * 3) ('quiz') - it provides a quiz view that displays a question and a user answer that can be "scored" by GPT * NOTE: this should probably be changed to passing down a prop to the flashcard telling it to render as a quiz. * * In each case, the two docs are stored in the _front and _back fields * * For 'flip' and 'slide', the trigger can either be clicking, or hovering as determined by the revealOp_hover field. * For 'quiz' the data of both Docs are shown in a single-view quiz display. * * Users can create a stack of flashcards all at once (only) from an empty flashcard by entering a topic into the front card * and clicking on the flashcard stack button. This will convert the comparision box into a stack of comparison boxes * filled in by GPT about the topic. * */ @observer export class ComparisonBox extends ViewBoxAnnotatableComponent() { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ComparisonBox, fieldKey); } /** * Creates a flashcard (or fills in flashcard data to a specified Doc) from a control string containing a question and answer * @param tuple string containing Question:, Answer: and optionally a Keyword: * @param useDoc doc to fill in instead of creating a Doc * @returns the resulting flashcard Doc */ public static createFlashcard(tuple3: string, frontKey: string, backKey: string, useDoc?: Doc) { const [qtoken, ktoken, atoken] = [ComparisonBox.qtoken, ComparisonBox.ktoken, ComparisonBox.atoken]; const [title, tuple] = tuple3.split(qtoken); const question = (tuple.includes(ktoken) ? tuple.split(ktoken)[0] : tuple).split(atoken)[0]; const rest = tuple.replace(question, ''); // prettier-ignore const answer = rest.startsWith(ktoken) ? // if keyword comes first, tuple.includes(atoken) ? tuple.split(atoken)[1] : "" : //if tuple includes answer, split at answer and take what's left, otherwise there's no answer rest.includes(ktoken) ? // otherwise if keyword is present it must come after answer, rest.split(ktoken)[0].split(atoken)[1] : // split at keyword and take what comes first and split that at answer and take what's left rest.replace(atoken,""); // finally if there's no keyword, just get rid of answer token and take what's left const keyword = rest.replace(atoken, '').replace(answer, '').replace(ktoken, '').trim(); const fillInFlashcard = (img?: Doc) => { const front = Docs.Create.CenteredTextCreator('question', question, {}, img); const back = Docs.Create.CenteredTextCreator('answer', answer, {}); if (useDoc) { useDoc['$' + frontKey] = front; useDoc['$' + backKey] = back; return useDoc; } return Docs.Create.FlashcardDocument(title, front, back, { _width: 300, _height: 300 }); }; return keyword && keyword.toLowerCase() !== 'none' ? ComparisonBox.fetchImages(keyword).then(img => fillInFlashcard(img)) : fillInFlashcard(); } /** * Create a carousel of flashcards from a GPT response string where questions and answers are given in a format loosely defined by: * Question: ... Answer: ... Keyword: ... * Note that Keyword or Answer may not be present, or their orders may be reversed. */ public static createFlashcardDeck(text: string, width: number, height: number, front: string, back: string) { return Promise.all( text .toLowerCase() .split(ComparisonBox.ttoken) .filter(t => t) .map(tuple => ComparisonBox.createFlashcard(tuple, front, back)) ).then(docs => { return Docs.Create.CarouselDocument(docs, { title: text, _width: width, _height: height, _layout_fitWidth: false, _layout_autoHeight: true, _xMargin: 5, _yMargin: 5, }); }); } private SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; static qtoken = 'question: '; static ktoken = 'keyword: '; static atoken = 'answer: '; static ttoken = 'title: '; private _slideTiming = 200; private _sideBtnWidth = 35; private _closeRef = React.createRef(); private _disposers: { [key: string]: DragManager.DragDropDisposer | undefined } = {}; private _reactDisposer: { [key: string]: IReactionDisposer } = {}; @observable private _inputValue = ''; @observable private _outputValue = ''; @observable private _loading = false; @observable private _childActive = false; @observable private _animating = ''; @observable private _listening = false; @observable private _renderSide = this.frontKey; @observable private _recognition = new this.SpeechRecognition(); constructor(props: FieldViewProps) { super(props); makeObservable(this); } componentDidMount() { this._props.setContentViewBox?.(this); this._reactDisposer.select = reaction( () => this._props.isSelected(), selected => { if (selected) { switch (this.revealOp) { default: case flashcardRevealOp.FLIP: this.activateContent(); break; case flashcardRevealOp.SLIDE: break; } // prettier-ignore } else { this._childActive = false; } }, // what it should update to { fireImmediately: true } ); this._reactDisposer.inactive = reaction( () => !this._props.isContentActive(), inactive => { if (inactive) { switch (this.revealOp) { case flashcardRevealOp.FLIP: this.animateFlipping(this.frontKey); break; case flashcardRevealOp.SLIDE: this.animateSliding(this._props.PanelWidth() - 3); break; } // prettier-ignore } }, { fireImmediately: true } ); } componentWillUnmount() { Object.values(this._reactDisposer).forEach(disposer => disposer?.()); } protected createDropTarget = (ele: HTMLDivElement | null, fieldKey: string) => { this._disposers[fieldKey]?.(); if (ele) { this._disposers[fieldKey] = DragManager.MakeDropTarget(ele, (e, dropEvent) => this.internalDrop(e, dropEvent, fieldKey), this.layoutDoc); } }; private internalDrop = undoable((e: Event, dropEvent: DragManager.DropEvent, fieldKey: string) => { if (dropEvent.complete.docDragData) { const { droppedDocuments } = dropEvent.complete.docDragData; const added = dropEvent.complete.docDragData.moveDocument?.(droppedDocuments, this.Document, (doc: Doc | Doc[]) => this.addDoc(toList(doc).lastElement(), fieldKey)); Doc.SetContainer(droppedDocuments.lastElement(), this.dataDoc); !added && e.preventDefault(); e.stopPropagation(); // prevent parent Doc from registering new position so that it snaps back into place // this.childActive = false; return added; } return undefined; }, 'internal drop'); @computed get containerDoc() { return this._props.docViewPath().slice(-2)[0]?.Document; } // prettier-ignore @computed get isQuizMode() { return this.containerDoc?.practiceMode === practiceMode.QUIZ; } // prettier-ignore @computed get isFlashcard() { return StrCast(this.Document.layout_flashcardType); } // prettier-ignore @computed get frontKey() { return this._props.fieldKey + '_front'; } // prettier-ignore @computed get backKey() { return this._props.fieldKey + '_back'; } // prettier-ignore @computed get frontText() { return RTFCast(DocCast(this.dataDoc[this.frontKey]).text)?.Text; } // prettier-ignore @computed get backText() { return RTFCast(DocCast(this.dataDoc[this.backKey]).text)?.Text; } // prettier-ignore @computed get revealOpKey() { return `_${this._props.fieldKey}_revealOp`; } // prettier-ignore @computed get clipHeightKey() { return `_${this._props.fieldKey}_clipHeight`; } // prettier-ignore @computed get clipWidthKey() { return `_${this._props.fieldKey}_clipWidth`; } // prettier-ignore @computed get clipWidth() { return NumCast(this.layoutDoc[this.clipWidthKey], this.isFlashcard ? 100: 50); } // prettier-ignore @computed get clipHeight() { return NumCast(this.layoutDoc[this.clipHeightKey], 200); } // prettier-ignore @computed get revealOp() { return StrCast(this.layoutDoc[this.revealOpKey], StrCast(this.containerDoc?.revealOp, this.isFlashcard ? flashcardRevealOp.FLIP : flashcardRevealOp.SLIDE)) as flashcardRevealOp; } // prettier-ignore set revealOp(op:flashcardRevealOp) { this.layoutDoc[this.revealOpKey] = op; } // prettier-ignore @computed get revealOpHover() { return BoolCast(this.layoutDoc[this.revealOpKey+"_hover"], BoolCast(this.containerDoc?.revealOp_hover)); } // prettier-ignore set revealOpHover(on:boolean) { this.layoutDoc[this.revealOpKey+"_hover"] = on; } // prettier-ignore @computed get loading() { return this._loading; } // prettier-ignore set loading(value) { runInAction(() => { this._loading = value; })} // prettier-ignore @computed get overlayAlternateIcon() { return ( flip}>
setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => { if (!this.revealOp || this.revealOp === flashcardRevealOp.FLIP) { this.animateFlipping(); } }) } style={{ background: this.revealOpHover ? 'gray' : this._renderSide === this.backKey ? 'white' : 'black', color: this.revealOpHover ? 'black' : this._renderSide === this.backKey ? 'black' : 'white', display: 'inline-block', }}>
); } /** * How much the content of the view is being scaled based on its nesting and its fit-to-width settings */ @computed get viewScaling() { return this.ScreenToLocalBoxXf().Scale; } // prettier-ignore /** * The maximum size a UI widget can be scaled so that it won't be bigger in screen pixels than its normal 35 pixel size. */ @computed get maxWidgetSize() { return Math.min(this._sideBtnWidth * this.viewScaling, 0.25 * Math.min(NumCast(this.Document.width), NumCast(this.Document.height))); } // prettier-ignore /** * How much to reactively scale a UI element so that it is as big as it can be (up to its normal 35pixel size) without being too big for the Doc content */ @computed get uiBtnScaling() { return Math.max(this.maxWidgetSize / this._sideBtnWidth, 1) * Math.min(1, this.viewScaling)* (this._props.NativeDimScaling?.() ?? 1); } // prettier-ignore @computed get flashcardMenu() { return (
{this.revealOpHover || !this._props.isSelected() ? null : this.overlayAlternateIcon} {!this._props.isSelected() || this._renderSide === this.frontKey ? null : ( Ask GPT to create an answer for the question on the front
}>
this.askGPT(GPTCallType.CHATCARD)}>
)} {!this._props.isSelected() || this._renderSide === this.backKey || !CollectionFreeFormView.from(this.DocumentView?.()) || (this.dataDoc[this.backKey] && !DocCast(this.dataDoc[this.backKey])?.text_placeholder) ? null : ( Create new flashcard stack based on text}>
this.askGPT(GPTCallType.STACK).then(async text => { const newCol = await ComparisonBox.createFlashcardDeck(text, NumCast(this.layoutDoc._width, 250) + 50, NumCast(this.layoutDoc._height, 200), this.frontKey, this.backKey); newCol.x = NumCast(this.layoutDoc.x); newCol.y = NumCast(this.layoutDoc.y); this._props.DocumentView?.()._props.addDocument?.(newCol); this._props.removeDocument?.(this.Document); }) }>
)} ); } @action activateContent = () => { this._childActive = true; }; @action handleRenderGPTClick = () => { if (this._inputValue) this.askGPT(GPTCallType.QUIZDOC); }; onPointerMove = ({ movementX }: PointerEvent) => { const width = movementX * this.ScreenToLocalBoxXf().Scale + (this.clipWidth / 100) * this._props.PanelWidth(); if (width && width > 5 && width < this._props.PanelWidth()) { this.layoutDoc[this.clipWidthKey] = (width * 100) / this._props.PanelWidth(); } return false; }; getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => { const anchor = Docs.Create.ConfigDocument({ title: 'CompareAnchor:' + this.Document.title, // set presentation timing properties for restoring view presentation_transition: 1000, annotationOn: this.Document, }); if (anchor) { if (!addAsAnnotation) anchor.backgroundColor = 'transparent'; /* addAsAnnotation && */ this.addDocument(anchor); PinDocView(anchor, { pinDocLayout: pinProps?.pinDocLayout, pinData: { ...(pinProps?.pinData ?? {}), clippable: true } }, this.Document); return anchor; } return this.Document; }; clearDoc = undoable((fieldKey: string) => { this.dataDoc[fieldKey] = undefined; }, 'clear doc'); moveDoc = (doc: Doc, addDocument: (document: Doc | Doc[]) => boolean, which: string) => this.remDoc(doc, which) && addDocument(doc); addDoc = (doc: Doc, which: string) => { this.dataDoc[which] = doc; return true; }; remDoc = (doc: Doc, which: string) => { if (this.dataDoc[which] === doc) { this.dataDoc[which] = undefined; return true; } return false; }; closeDown = (e: React.PointerEvent, which: string) => { setupMoveUpEvents( this, e, moveEv => { const de = new DragManager.DocumentDragData([DocCast(this.dataDoc[which])], dropActionType.move); de.moveDocument = (doc: Doc | Doc[], targetCollection: Doc | undefined, addDocument: (doc: Doc | Doc[]) => boolean): boolean => addDocument(doc); de.canEmbed = true; DragManager.StartDocumentDrag([this._closeRef.current!], de, moveEv.clientX, moveEv.clientY); return true; }, emptyFunction, () => this.clearDoc(which) ); }; docStyleProvider = (doc: Opt, props: Opt, property: string) => { switch (property) { case StyleProp.PointerEvents: return 'none'; default: return this._props.styleProvider?.(doc, props, property); } // prettier-ignore }; moveDocFront = (docs: Doc | Doc[], targetCol: Doc | undefined, addDoc: (doc: Doc | Doc[]) => boolean) => toList(docs).reduce((res, doc: Doc) => res && this.moveDoc(doc, addDoc, this.frontKey), true); moveDocBack = (docs: Doc | Doc[], targetCol: Doc | undefined, addDoc: (doc: Doc | Doc[]) => boolean) => toList(docs).reduce((res, doc: Doc) => res && this.moveDoc(doc, addDoc, this.backKey), true); remDocFront = (docs: Doc | Doc[]) => toList(docs).reduce((res, doc) => res && this.remDoc(doc, this.frontKey), true); remDocBack = (docs: Doc | Doc[]) => toList(docs).reduce((res, doc) => res && this.remDoc(doc, this.backKey), true); animateSliding = action((targetWidth: number) => { this._animating = `all ${this._slideTiming}ms`; // on click, animate slider movement to the targetWidth this.layoutDoc[this.clipWidthKey] = (targetWidth * 100) / this._props.PanelWidth(); setTimeout(action(() => {this._animating = ''; }), this._slideTiming); // prettier-ignore }); _flipAnim: NodeJS.Timeout | undefined; animateFlipping = action((side?: string) => { if (side !== this._renderSide) { this._renderSide = side ?? (this._renderSide === this.frontKey ? this.backKey : this.frontKey); // switches to new front this._animating = '0'; // reveals old front on the bottom layer by making top layer transparent setTimeout( action(() => { this._animating = `all ${this._slideTiming * 5}ms`; // makes new front fade in clearTimeout(this._flipAnim); this._flipAnim = setTimeout( action(() => { this._animating = ''; }), this._slideTiming * 5 ); // prettier-ignore }) ); } }); registerSliding = (e: React.PointerEvent, targetWidth: number) => { if (e.button !== 2) { setupMoveUpEvents( this, e, this.onPointerMove, emptyFunction, action((moveEv, doubleTap) => { if (doubleTap) { this._childActive = true; if (!this.dataDoc[this.frontKey] && !this.dataDoc[this.fieldKey]) this.dataDoc[this.frontKey] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); if (!this.dataDoc[this.backKey] && !this.dataDoc[this.fieldKey + '_alternate']) this.dataDoc[this.backKey] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); } }), false, undefined, action(() => !this._childActive && this.animateSliding(targetWidth)) ); } }; /** * Set up speech to text tool. */ setListening = () => { if (this.SpeechRecognition) { this._recognition.continuous = true; this._recognition.interimResults = true; this._recognition.lang = 'en-US'; this._recognition.onresult = this.handleResult.bind(this); } ContextMenu.Instance.setLangIndex(0); }; startListening = () => { this._recognition.start(); this._listening = true; }; stopListening = () => { this._recognition.stop(); this._listening = false; }; setLanguage = (language: string, ind: number) => { this._recognition.lang = language; ContextMenu.Instance.setLangIndex(ind); }; /** * Determine which language the speech to text tool is in. * @returns */ convertAbr = () => { switch (this._recognition.lang) { case 'en-US': return 'English'; //prettier-ignore case 'es-ES': return 'Spanish'; //prettier-ignore case 'fr-FR': return 'French'; //prettier-ignore case 'it-IT': return 'Italian'; //prettier-ignore case 'zh-CH': return 'Mandarin Chinese'; //prettier-ignore case 'ja': return 'Japanese'; //prettier-ignore default: return 'Korean'; //prettier-ignore } }; openContextMenu = (x: number, y: number, evalu: boolean) => { ContextMenu.Instance.clearItems(); ContextMenu.Instance.addItem({ description: 'English', event: () => this.setLanguage('en-US', 0), icon: 'question' }); //prettier-ignore ContextMenu.Instance.addItem({ description: 'Spanish', event: () => this.setLanguage('es-ES', 1 ), icon: 'question'}); //prettier-ignore ContextMenu.Instance.addItem({ description: 'French', event: () => this.setLanguage('fr-FR', 2), icon: 'question' }); //prettier-ignore ContextMenu.Instance.addItem({ description: 'Italian', event: () => this.setLanguage('it-IT', 3), icon: 'question' }); //prettier-ignore if (!evalu) ContextMenu.Instance.addItem({ description: 'Mandarin Chinese', event: () => this.setLanguage('zh-CH', 4), icon: 'question' }); //prettier-ignore ContextMenu.Instance.addItem({ description: 'Japanese', event: () => this.setLanguage('ja', 5), icon: 'question' }); //prettier-ignore ContextMenu.Instance.addItem({ description: 'Korean', event: () => this.setLanguage('ko', 6), icon: 'question' }); //prettier-ignore ContextMenu.Instance.displayMenu(x, y); }; /** * Creates an AudioBox to record a user's audio. */ evaluatePronunciation = () => { const newAudio = Docs.Create.AudioDocument(nullAudio, { _width: 200, _height: 100 }); this.Document.audio = newAudio[DocData]; this._props.DocumentView?.()._props.addDocument?.(newAudio); }; /** * Calls GPT for each flashcard type. */ askGPT = async (callType: GPTCallType) => { const questionText = this.frontText; const queryText = questionText + (callType == GPTCallType.QUIZDOC ? ' UserAnswer: ' + this._inputValue + '. ' + ' Rubric: ' + this.backText : ''); this.loading = true; const res = !this.frontText ? '' : await gptAPICall(queryText, callType).then( action(resp => { switch (resp && callType) { case GPTCallType.CHATCARD: DocCast(this.dataDoc[this.backKey]).$text = resp; break; case GPTCallType.QUIZDOC: this._renderSide = this.backKey; this._outputValue = resp.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); break; case GPTCallType.FLASHCARD: default: } return resp; }) ); this.loading = false; if (!res) console.error('GPT call failed'); return res; }; layoutWidth = () => NumCast(this.layoutDoc.width, 200); layoutHeight = () => NumCast(this.layoutDoc.height, 200); /** * Display a user's speech to text result. * @param e */ handleResult = (e: SpeechRecognitionEvent) => { let finalTranscript = ''; for (let i = e.resultIndex; i < e.results.length; i++) { const transcript = e.results[i][0].transcript; if (e.results[i].isFinal) { finalTranscript += transcript; } } this._inputValue += finalTranscript; }; /** * Get images from unsplash api and place that will be placed inside generated flashcard. * @param selection * @returns Image Document */ public static async fetchImages(selection: string) { try { const { data } = await axios.get(`${API_URL}?query=${selection}&page=1&per_page=${1}&client_id=Q4zruu6k6lum2kExiGhLNBJIgXDxD6NNj0SRHH_XXU0`); const imageSnapshot = Docs.Create.ImageDocument(data.results[0].urls.small, { onClick: FollowLinkScript(), _width: 150, _height: 150, title: selection, }); return imageSnapshot; } catch (error) { console.log(error); } } getImageDesc = async (u: string) => { try { const hrefBase64 = await imageUrlToBase64(u); const response = await gptImageLabel(hrefBase64, 'Answer the following question as a short flashcard response. Do not include a label.' + (this.dataDoc.text as RichTextField)?.Text); DocCast(this.dataDoc[this.backKey]).$text = response; } catch (error) { console.log('Error', error); } }; flashcardContextMenu = () => { const appearance = ContextMenu.Instance.findByDescription('Appearance...'); const appearanceItems = appearance?.subitems ?? []; appearanceItems.push({ description: 'Create ChatCard', event: () => this.askGPT(GPTCallType.CHATCARD), icon: 'id-card' }); appearanceItems.push({ description: 'Reveal by ' + (this.revealOp === flashcardRevealOp.FLIP ? 'Sliding' : 'Flipping'), event: () => (this.revealOp = this.revealOp === flashcardRevealOp.FLIP ? flashcardRevealOp.SLIDE : flashcardRevealOp.FLIP), icon: 'id-card', }); appearanceItems.push({ description: (this.revealOpHover ? 'Click ' : 'Hover ') + ' to reveal', event: () => (this.revealOpHover = !this.revealOpHover), icon: 'id-card' }); !appearance && ContextMenu.Instance.addItem({ description: 'Appearance...', subitems: appearanceItems, icon: 'eye' }); }; childActiveFunc = () => this._childActive; contentScreenToLocalXf = () => this._props.ScreenToLocalTransform().scale(this._props.NativeDimScaling?.() || 1); clearButton = (which: string) => ( remove}>
this.closeDown(e, which)} // prevent triggering slider movement in registerSliding >
); childFitWidth = () => Cast(this.Document.childLayoutFitWidth, 'boolean') ?? Cast(this.Document.childLayoutFitWidth, 'boolean'); displayDoc = (whichSlot: string) => { const whichDoc = DocCast(this.dataDoc[whichSlot]); const targetDoc = DocCast(whichDoc?.annotationOn, whichDoc); return targetDoc ? ( <> {!this.isFlashcard ? this.clearButton(whichSlot) : null} ) : (
); }; displayBox = (which: string, cover: number) => (
this.registerSliding(e, cover)} ref={ele => this.createDropTarget(ele, which)}> {this.displayDoc(which)}
); /* renders front(qustion) and back(answer) at the same time, then on user input replaces the answer with a GPT analysis of the answer */ renderAsQuiz = (text: string) => (

{text}

Return to all flashcards and add text to both sides.