merged with master and cleaned up outpainting a bit.

1 files changed, 7 insertions, 132 deletions

diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx
index c0c6db4d3..6d5891003 100644
--- a/src/client/views/nodes/ComparisonBox.tsx
+++ b/src/client/views/nodes/ComparisonBox.tsx
@@ -29,7 +29,6 @@ import '../pdf/GPTPopup/GPTPopup.scss';
 import './ComparisonBox.scss';
 import { DocumentView } from './DocumentView';
 import { FieldView, FieldViewProps } from './FieldView';
-import { FormattedTextBox } from './formattedText/FormattedTextBox';
 import { TraceMobx } from '../../../fields/util';
 
 const API_URL = 'https://api.unsplash.com/search/photos';
@@ -80,8 +79,8 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
             const front = Docs.Create.CenteredTextCreator('question', question, {}, img);
             const back = Docs.Create.CenteredTextCreator('answer', answer, {});
             if (useDoc) {
-                useDoc[DocData][frontKey] = front;
-                useDoc[DocData][backKey] = back;
+                useDoc['$' + frontKey] = front;
+                useDoc['$' + backKey] = back;
                 return useDoc;
             }
             return Docs.Create.FlashcardDocument(title, front, back, { _width: 300, _height: 300 });
@@ -285,13 +284,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
     };
 
     @action handleRenderGPTClick = () => {
-        const phonTrans = DocCast(this.Document.audio) ? DocCast(this.Document.audio).phoneticTranscription : undefined;
-        if (phonTrans) {
-            this._inputValue = StrCast(phonTrans);
-            this.askGPTPhonemes(this._inputValue);
-            this._renderSide = this.backKey;
-            this._outputValue = '';
-        } else if (this._inputValue) this.askGPT(GPTCallType.QUIZDOC);
+        if (this._inputValue) this.askGPT(GPTCallType.QUIZDOC);
     };
 
     onPointerMove = ({ movementX }: PointerEvent) => {
@@ -468,45 +461,6 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
     };
 
     /**
-     * Gets the transcription of an audio recording by sending the
-     * recording to backend.
-     */
-    pushInfo = () =>
-        axios
-            .post(
-                'http://localhost:105/recognize/', //
-                { file: DocCast(this.Document.audio)[DocData].url },
-                { headers: { 'Content-Type': 'application/json' } }
-            )
-            .then(response => {
-                this.Document.phoneticTranscription = response.data.transcription;
-            });
-
-    /**
-     * Extracts the id of the youtube video url.
-     * @param url
-     * @returns
-     */
-    getYouTubeVideoId = (url: string) => {
-        const regExp = /^.*(youtu.be\/|v\/|u\/\w\/|embed\/|watch\?v=|&v=|\?v=)([^#&?]*).*/;
-        const match = url.match(regExp);
-        return match && match[2].length === 11 ? match[2] : null;
-    };
-
-    /**
-     * Gets the transcript of a youtube video by sending the video url to the backend.
-     * @returns transcription of youtube recording
-     */
-    youtubeUpload = async () =>
-        axios
-            .post(
-                'http://localhost:105/youtube/', //
-                { file: this.getYouTubeVideoId(this.frontText) },
-                { headers: { 'Content-Type': 'application/json' } }
-            )
-            .then(response => response.data.transcription);
-
-    /**
      * Calls GPT for each flashcard type.
      */
     askGPT = async (callType: GPTCallType) => {
@@ -520,7 +474,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
                   action(resp => {
                       switch (resp && callType) {
                           case GPTCallType.CHATCARD:
-                              DocCast(this.dataDoc[this.backKey])[DocData].text = resp;
+                              DocCast(this.dataDoc[this.backKey]).$text = resp;
                               break;
                           case GPTCallType.QUIZDOC:
                               this._renderSide = this.backKey;
@@ -540,45 +494,6 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
     layoutHeight = () => NumCast(this.layoutDoc.height, 200);
 
     /**
-     * Ask GPT for advice on how to improve speech by comparing the phonetic transcription of
-     * a users audio recording with the phonetic transcription of their intended sentence.
-     * @param phonemes
-     */
-    askGPTPhonemes = async (phonemes: string) => {
-        const sentence = this.frontText;
-        const phon6 = 'huː ɑɹ juː tədeɪ';
-        const phon4 = 'kamo estas hɔi';
-        const promptEng =
-            'Consider all possible phonetic transcriptions of the intended sentence "' +
-            sentence +
-            '" that is standard in American speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' +
-            phon6 +
-            '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in American speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Only note the difference if they are not allophones of the same phoneme and if they are far away on the vowel chart. The goal is to be understood, not sound like a native speaker. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Provide a response like this: "Lets work on improving the pronunciation of "coffee." You said "ceeffee," which is close, but we need to adjust the vowel sound. In American English, "coffee" is pronounced /ˈkɔːfi/, with a long "aw" sound. Try saying "kah-fee." Your intonation is good, but try putting a bit more stress on "like" in the sentence "I would like a coffee with milk." This will make your speech sound more natural. Keep practicing, and lets try saying the whole sentence again!"';
-        const promptSpa =
-            'Consider all possible phonetic transcriptions of the intended sentence "' +
-            'como estás hoy' +
-            '" that is standard in Spanish speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' +
-            phon4 +
-            '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in Spanish speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound.  Only note the difference if they are not allophones of the same phoneme and if they are far away on the vowel chart; say good job if it would be understood by a native Spanish speaker. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". Do not make "θ" and "f" interchangable. Do not make "n" and "ɲ" interchangable. Do not make "e" and "i" interchangable. If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Identify "ɔi" sounds like "oy". Ignore accents and do not say anything to the user about this.';
-        const promptAll =
-            'Consider all possible phonetic transcriptions of the intended sentence "' +
-            sentence +
-            '" that is standard in ' +
-            this.convertAbr() +
-            ' speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' +
-            phonemes +
-            '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in ' +
-            this.convertAbr() +
-            ' speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". Do not make "θ" and "f" interchangable. Do not make "n" and "ɲ" interchangable. Do not make "e" and "i" interchangable. If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Provide a response like this: "Lets work on improving the pronunciation of "coffee." You said "cawffee," which is close, but we need to adjust the vowel sound. In American English, "coffee" is pronounced /ˈkɔːfi/, with a long "aw" sound. Try saying "kah-fee." Your intonation is good, but try putting a bit more stress on "like" in the sentence "I would like a coffee with milk." This will make your speech sound more natural. Keep practicing, and lets try saying the whole sentence again!"';
-
-        switch (this._recognition.lang) {
-            case 'en-US': this._outputValue = await gptAPICall(promptEng, GPTCallType.PRONUNCIATION); break;
-            case 'es-ES': this._outputValue = await gptAPICall(promptSpa, GPTCallType.PRONUNCIATION); break;
-            default:      this._outputValue = await gptAPICall(promptAll, GPTCallType.PRONUNCIATION); break;
-        } // prettier-ignore
-    };
-
-    /**
      * Display a user's speech to text result.
      * @param e
      */
@@ -618,7 +533,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
             const hrefBase64 = await imageUrlToBase64(u);
             const response = await gptImageLabel(hrefBase64, 'Answer the following question as a short flashcard response. Do not include a label.' + (this.dataDoc.text as RichTextField)?.Text);
 
-            DocCast(this.dataDoc[this.backKey])[DocData].text = response;
+            DocCast(this.dataDoc[this.backKey]).$text = response;
         } catch (error) {
             console.log('Error', error);
         }
@@ -637,31 +552,6 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
         !appearance && ContextMenu.Instance.addItem({ description: 'Appearance...', subitems: appearanceItems, icon: 'eye' });
     };
 
-    testForTextFields = (whichSlot: string) => {
-        const slotData = Doc.Get(this.dataDoc, whichSlot, true);
-        const slotHasText = slotData instanceof RichTextField || typeof slotData === 'string';
-        const subjectText = RTFCast(this.Document[this.fieldKey])?.Text.trim();
-        const altText = RTFCast(this.Document[this.fieldKey + '_alternate'])?.Text.trim();
-        const layoutTemplateString =
-            slotHasText ? FormattedTextBox.LayoutString(whichSlot):
-            whichSlot === this.frontKey ? (subjectText !== undefined ? FormattedTextBox.LayoutString(this.fieldKey) : undefined) :
-                                          altText !== undefined     ? FormattedTextBox.LayoutString(this.fieldKey + '_alternate'): undefined; // prettier-ignore
-
-        // A bit hacky to try out the concept of using GPT to fill in flashcards
-        //   If the second slot doesn't have anything in it, but the fieldKey slot has text (e.g., this.text is a string)
-        // and the fieldKey + "_alternate" has text that includes a GPT query (indicated by (( && )) ) that is parameterized (optionally) by the fieldKey text (this) or other metadata (this.<field>).
-        // eg., this.text_alternate is
-        //     "((Provide a one sentence definition for (this) that doesn't use any word in (this.excludeWords) ))"
-        //     where (this) is replaced by the text in the fieldKey slot abd this.excludeWords is repalced by the conetnts of the excludeWords field
-        // The GPT call will put the "answer"  in the second slot of the comparison (eg., text_0)
-        if (whichSlot === this.backKey && !layoutTemplateString?.includes(whichSlot)) {
-            const queryText = altText?.replace('(this)', subjectText); // TODO: this should be done in Doc.setField but it doesn't know about the fieldKey ...
-            if (queryText?.match(/\(\(.*\)\)/)) {
-                Doc.SetField(this.Document, whichSlot, ':=' + queryText, false); // make the second slot be a computed field on the data doc that calls ChatGpt
-            }
-        }
-        return layoutTemplateString;
-    };
     childActiveFunc = () => this._childActive;
 
     contentScreenToLocalXf = () => this._props.ScreenToLocalTransform().scale(this._props.NativeDimScaling?.() || 1);
@@ -682,24 +572,21 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
     displayDoc = (whichSlot: string) => {
         const whichDoc = DocCast(this.dataDoc[whichSlot]);
         const targetDoc = DocCast(whichDoc?.annotationOn, whichDoc);
-        const layoutString = targetDoc ? '' : this.testForTextFields(whichSlot);
 
-        return targetDoc || layoutString ? (
+        return targetDoc ? (
             <>
                 <DocumentView
                     {...this._props}
-                    Document={layoutString ? this.Document : targetDoc}
+                    Document={targetDoc}
                     NativeWidth={returnZero}
                     NativeHeight={returnZero}
                     renderDepth={this.props.renderDepth + 1}
-                    LayoutTemplateString={layoutString}
                     containerViewPath={this._props.docViewPath}
                     ScreenToLocalTransform={this.contentScreenToLocalXf}
                     isDocumentActive={returnFalse}
                     isContentActive={this.childActiveFunc}
                     showTags={undefined}
                     fitWidth={this.childFitWidth} // set to returnTrue to make images fill the comparisonBox-- should be a user option
-                    ignoreUsePath={layoutString ? true : undefined}
                     moveDocument={whichSlot === this.frontKey ? this.moveDocFront : this.moveDocBack}
                     removeDocument={whichSlot === this.frontKey ? this.remDocFront : this.remDocBack}
                     dontSelect={returnTrue}
@@ -745,18 +632,6 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
             </div>
             <div>
                 <div className="submit-button">
-                    {/* <div className="submit-buttonschema-header-button" onPointerDown={e => this.openContextMenu(e.clientX, e.clientY, false)}>
-                        <FontAwesomeIcon color="white" icon="caret-down" />
-                    </div> */}
-                    {/* <button className="submit-buttonrecord" onClick={this._listening ? this.stopListening : this.startListening} style={{ background: this._listening ? 'lightgray' : '' }}>
-                        {<FontAwesomeIcon icon="microphone" size="lg" />}
-                    </button> */}
-                    {/* <div className="submit-buttonschema-header-button" onPointerDown={e => this.openContextMenu(e.clientX, e.clientY, true)} style={{ left: '50px', zIndex: '100' }}>
-                        <FontAwesomeIcon color="white" icon="caret-down" />
-                    </div> */}
-                    {/* <button className="submit-buttonpronunciation" onClick={this.evaluatePronunciation}>
-                        Evaluate Pronunciation
-                    </button> */}
                     <button className="submit-buttonsubmit" type="button" onClick={this._renderSide === this.backKey ? () => this.animateFlipping(this.frontKey) : this.handleRenderGPTClick}>
                         {this._renderSide === this.backKey ? 'Redo the Question' : 'Submit'}
                     </button>