palette, extracting image colors

7 files changed, 178 insertions, 15 deletions

diff --git a/package-lock.json b/package-lock.json
index e84f33046..419ce40c4 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -13766,6 +13766,11 @@
                 }
             }
         },
+        "extract-colors": {
+            "version": "4.0.2",
+            "resolved": "https://registry.npmjs.org/extract-colors/-/extract-colors-4.0.2.tgz",
+            "integrity": "sha512-G7v2C3LJqW38U+yRUPD6nJCjBRfdLD7y8efEHn+1qONt1mhj+OZBpzFmiaS+dZiHU/k0dvgK2kgIkbAllHVbRw=="
+        },
         "extract-zip": {
             "version": "2.0.1",
             "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
diff --git a/package.json b/package.json
index d8996f57e..faa4553bb 100644
--- a/package.json
+++ b/package.json
@@ -207,6 +207,7 @@
         "express-session": "^1.17.2",
         "express-validator": "^5.3.1",
         "expressjs": "^1.0.1",
+        "extract-colors": "^4.0.2",
         "ffmpeg": "0.0.4",
         "file-saver": "^2.0.5",
         "find-in-files": "^0.5.0",
diff --git a/src/client/apis/gpt/customization.ts b/src/client/apis/gpt/customization.ts
index 20dac0a4e..fec0a50f9 100644
--- a/src/client/apis/gpt/customization.ts
+++ b/src/client/apis/gpt/customization.ts
@@ -4,6 +4,16 @@ export enum CustomizationType {
     PRES_TRAIL_SLIDE = 'trails',
 }
 
+export interface GeneratedResponse {
+    collectionBackgroundColor: string;
+    documentsWithColors: DocumentWithColor[];
+}
+
+export interface DocumentWithColor {
+    id: number;
+    color: string;
+}
+
 interface PromptInfo {
     description: string;
     features: { name: string; description: string; values?: string[] }[];
@@ -65,14 +75,14 @@ export const generatePalette = async (inputData: any) => {
     let prompt = 'Dash is a hypermedia web application that allows users to organize documents of different media types into collections. I want you to come up with a cohesive color palette for a collection.';
     prompt +=
         'I am going to give you a json object of this format:' +
-        JSON.stringify({ collectionDescription: 'string', documents: 'Document[]' }) +
-        '. collectionDescription is the title of the collection, which you should create a color palette based on. This is the document format:' +
+        JSON.stringify({ collectionDescription: 'string', documents: 'Document[]', imageColors: 'string[]' }) +
+        '. collectionDescription is the title of the collection, which you should create color palettes based on. This is the document format:' +
         JSON.stringify({
             id: 'number',
             textSize: 'number',
             textContent: 'string',
         }) +
-        '. You are going to create a color palette based mostly on collectionDescription, and loosely on the text content and text size of the documents. Return a json object in this format:' +
+        '. Finally, imageColors are the main hex colors of the images in the collection. You are going to generate three distinct variants of color palettes for the user to choose from based mostly on collectionDescription, and loosely on the text content and text size of the documents. You should also take the imageColors into account, but mostly rely on the text content. The variants should start with a light palette and grow increasingly more intense and vibrant. Return a json array of three objects in this format:' +
         JSON.stringify({
             collectionBackgroundColor: 'string',
             documentsWithColors: 'DocumentWithColor[]',
@@ -82,7 +92,7 @@ export const generatePalette = async (inputData: any) => {
             id: 'number',
             color: 'string',
         }) +
-        ", and each element’s color is based on the theme of the overall color palette and also by its document’s textContent. Please pay attention to aesthetics of how each document's color complement the background and each other and choose a variety of colors when appropriate. Respond with only the JSON object.";
+        ", and each element’s color is based on the theme of the overall color palette and also by its document’s textContent. Please pay attention to aesthetics of how each document's color complement the background and each other and choose a variety of colors when appropriate. Respond with only the JSON array.";
 
     // console.log('Prompt', prompt);
     try {
@@ -95,8 +105,12 @@ export const generatePalette = async (inputData: any) => {
             temperature: 0.1,
             max_tokens: 2000,
         });
-        console.log(response.data.choices[0].message?.content);
-        return response.data.choices[0].message?.content;
+        const content = response.data.choices[0].message?.content;
+        console.log(content);
+        if (content) {
+            return content;
+        }
+        return 'Malformed response.';
     } catch (err) {
         console.log(err);
         return 'Error connecting with API.';
diff --git a/src/client/views/ExtractColors.ts b/src/client/views/ExtractColors.ts
new file mode 100644
index 000000000..f78d9a355
--- /dev/null
+++ b/src/client/views/ExtractColors.ts
@@ -0,0 +1,57 @@
+import { extractColors } from 'extract-colors';
+
+// Manages image color extraction
+export class ExtractColors {
+    // loads all images into img elements
+    static loadImages = async (imageFiles: File[]): Promise<HTMLImageElement[]> => {
+        try {
+            const imageElements = await Promise.all(imageFiles.map(file => this.loadImage(file)));
+            return imageElements;
+        } catch (error) {
+            console.error(error);
+            return [];
+        }
+    };
+
+    // loads a single img into an img element
+    static loadImage = (file: File): Promise<HTMLImageElement> => {
+        return new Promise((resolve, reject) => {
+            const img = new Image();
+
+            img.onload = () => resolve(img);
+            img.onerror = error => reject(error);
+
+            const url = URL.createObjectURL(file);
+            img.src = url;
+        });
+    };
+
+    // loads all images into img elements
+    static loadImagesUrl = async (imageUrls: string[]): Promise<HTMLImageElement[]> => {
+        try {
+            const imageElements = await Promise.all(imageUrls.map(url => this.loadImageUrl(url)));
+            return imageElements;
+        } catch (error) {
+            console.error(error);
+            return [];
+        }
+    };
+
+    // loads a single img into an img element
+    static loadImageUrl = (url: string): Promise<HTMLImageElement> => {
+        return new Promise((resolve, reject) => {
+            const img = new Image();
+
+            img.onload = () => resolve(img);
+            img.onerror = error => reject(error);
+
+            img.src = url;
+        });
+    };
+
+    // extracts a list of collors from an img element
+    static getImgColors = async (img: HTMLImageElement) => {
+        const colors = await extractColors(img, { distance: 0.35 });
+        return colors;
+    };
+}
diff --git a/src/client/views/PropertiesView.scss b/src/client/views/PropertiesView.scss
index 2da2ec568..ffcad0e7e 100644
--- a/src/client/views/PropertiesView.scss
+++ b/src/client/views/PropertiesView.scss
@@ -7,6 +7,15 @@
     position: absolute;
     right: 4;
 }
+.propertiesView-palette {
+    cursor: pointer;
+    padding: 8px;
+    border-radius: 4px;
+    transition: all 0.2s ease;
+    &:hover {
+        background-color: #3b3c3e;
+    }
+}
 .propertiesView {
     height: 100%;
     width: 250;
diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx
index 64b2a9d65..fb2d811f4 100644
--- a/src/client/views/PropertiesView.tsx
+++ b/src/client/views/PropertiesView.tsx
@@ -40,6 +40,7 @@ import { PropertiesSection } from './PropertiesSection';
 import './PropertiesView.scss';
 import { DefaultStyleProvider } from './StyleProvider';
 import { FaFillDrip } from 'react-icons/fa';
+import { GeneratedResponse } from '../apis/gpt/customization';
 const _global = (window /* browser */ || global) /* node */ as any;
 
 interface PropertiesViewProps {
@@ -90,6 +91,16 @@ export class PropertiesView extends React.Component<PropertiesViewProps> {
     @observable openAppearance: boolean = true;
     @observable openTransform: boolean = true;
     @observable openFilters: boolean = false;
+    @observable openStyling: boolean = true;
+
+    // GPT styling
+    @observable generatedStyles: GeneratedResponse[] = [];
+    @observable inputDocs: Doc[] = [];
+    @observable selectedStyle: number = -1;
+
+    @action
+    setGeneratedStyles = (responses: GeneratedResponse[]) => (this.generatedStyles = responses);
+    setInputDocs = (docs: Doc[]) => (this.inputDocs = docs);
 
     //Pres Trails booleans:
     @observable openPresTransitions: boolean = true;
@@ -1161,6 +1172,36 @@ export class PropertiesView extends React.Component<PropertiesViewProps> {
         }
     };
 
+    @action
+    styleCollection = (i: number) => {
+        this.selectedStyle = i;
+        const resObj = this.generatedStyles[i];
+        if (this.selectedDoc && this.selectedDoc.type === 'collection') {
+            this.selectedDoc.backgroundColor = resObj.collectionBackgroundColor;
+            resObj.documentsWithColors.forEach((elem, i) => (this.inputDocs[i].backgroundColor = elem.color));
+        }
+    };
+
+    // GPT styling
+    @computed get stylingSubMenu() {
+        return (
+            <PropertiesSection title="Styling" isOpen={this.openStyling} setIsOpen={bool => (this.openStyling = bool)} onDoubleClick={() => this.CloseAll()}>
+                <div className="propertiesView-content" style={{ position: 'relative', height: 'auto', display: 'flex', flexDirection: 'column', alignItems: 'center', gap: '4px' }}>
+                    {this.generatedStyles.length > 0
+                        ? this.generatedStyles.map((style, i) => (
+                              <div className="propertiesView-palette" style={{ display: 'flex', gap: '4px', backgroundColor: this.selectedStyle === i ? StrCast(Doc.UserDoc().userVariantColor) : '#00000000' }} onClick={() => this.styleCollection(i)}>
+                                  <div style={{ width: '24px', height: '24px', backgroundColor: style.collectionBackgroundColor, borderRadius: '2px' }}></div>
+                                  {style.documentsWithColors.map(c => (
+                                      <div style={{ width: '24px', height: '24px', backgroundColor: c.color, borderRadius: '2px' }}></div>
+                                  ))}
+                              </div>
+                          ))
+                        : 'Generating styles...'}
+                </div>
+            </PropertiesSection>
+        );
+    }
+
     @computed get filtersSubMenu() {
         return (
             <PropertiesSection title="Filters" isOpen={this.openFilters} setIsOpen={bool => (this.openFilters = bool)} onDoubleClick={() => this.CloseAll()}>
@@ -1739,6 +1780,7 @@ export class PropertiesView extends React.Component<PropertiesViewProps> {
 
                         <div className="propertiesView-name">{this.editableTitle}</div>
                         <div className="propertiesView-type"> {this.currentType} </div>
+                        {this.stylingSubMenu}
                         {this.optionsSubMenu}
                         {this.linksSubMenu}
                         {!LinkManager.currentLink || !this.openLinks ? null : this.linkProperties}
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 002ebf1ae..fb2de5647 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -27,7 +27,7 @@ import { FollowLinkScript } from '../../../util/LinkFollower';
 import { ReplayMovements } from '../../../util/ReplayMovements';
 import { ScriptingGlobals } from '../../../util/ScriptingGlobals';
 import { SelectionManager } from '../../../util/SelectionManager';
-import { freeformScrollMode } from '../../../util/SettingsManager';
+import { freeformScrollMode, SettingsManager } from '../../../util/SettingsManager';
 import { SnappingManager } from '../../../util/SnappingManager';
 import { Transform } from '../../../util/Transform';
 import { undoBatch, UndoManager } from '../../../util/UndoManager';
@@ -52,7 +52,11 @@ import { CollectionFreeFormRemoteCursors } from './CollectionFreeFormRemoteCurso
 import './CollectionFreeFormView.scss';
 import { MarqueeView } from './MarqueeView';
 import React = require('react');
-import { generatePalette } from '../../../apis/gpt/customization';
+import { DocumentWithColor, GeneratedResponse, generatePalette } from '../../../apis/gpt/customization';
+import { PropertiesView } from '../../PropertiesView';
+import { MainView } from '../../MainView';
+import { ExtractColors } from '../../ExtractColors';
+import { extname } from 'path';
 
 export type collectionFreeformViewProps = {
     NativeWidth?: () => number;
@@ -1796,22 +1800,50 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
         });
     };
 
+    @action
+    openProperties = () => {
+        SettingsManager.propertiesWidth = 300;
+    };
+
+    choosePath(url: URL) {
+        if (!url?.href) return '';
+        const lower = url.href.toLowerCase();
+        if (url.protocol === 'data') return url.href;
+        if (url.href.indexOf(window.location.origin) === -1 && url.href.indexOf('dashblobstore') === -1) return Utils.CorsProxy(url.href);
+        if (!/\.(png|jpg|jpeg|gif|webp)$/.test(lower)) return `/assets/unknown-file-icon-hi.png`;
+
+        const ext = extname(url.href);
+        return url.href.replace(ext, '_m' + ext);
+    }
+
     // gpt styling
     @action
     gptStyling = async () => {
+        this.openProperties();
         console.log('Title', this.rootDoc.title);
         console.log('bgcolor', this.layoutDoc._backgroundColor);
         // doc.backgroundColor
         console.log('styling');
         const inputDocs = this.childDocs.filter(doc => doc.type == 'rich text');
+        const imgDocs = this.childDocs.filter(doc => doc.type == 'image');
+        const imgUrls = imgDocs.map(doc => this.choosePath((doc.data as ImageField).url));
+
+        const imageElements = await ExtractColors.loadImagesUrl(imgUrls);
+        const colors = await Promise.all(imageElements.map(elem => ExtractColors.getImgColors(elem)));
+        let colorHexes = colors
+            .reduce((acc, curr) => acc.concat(curr), [])
+            .map(color => color.hex)
+            .slice(0, 10);
+        console.log('Hexes', colorHexes);
+
+        PropertiesView.Instance?.setInputDocs(inputDocs);
+        // also pass it colors
         const gptInput = inputDocs.map((doc, i) => ({
             id: i,
             textContent: (doc.text as RichTextField)?.Text,
             textSize: 16,
         }));
 
-        // inputDocs[0].backgroundColor = '#3392ff';
-
         const collectionDescription = StrCast(this.rootDoc.title);
 
         console.log({
@@ -1823,13 +1855,16 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
             const res = await generatePalette({
                 collectionDescription,
                 documents: gptInput,
+                imageColors: colorHexes,
             });
-            console.log('done');
             if (typeof res === 'string') {
-                const resObj = JSON.parse(res);
-                console.log('Result ', resObj);
-                this.rootDoc.backgroundColor = resObj.collectionBackgroundColor;
-                (resObj.documentsWithColors as any[]).forEach((elem, i) => (inputDocs[i].backgroundColor = elem.color));
+                console.log(res);
+                const resObj = JSON.parse(res) as GeneratedResponse[];
+                PropertiesView.Instance?.setGeneratedStyles(resObj);
+                // const resObj = JSON.parse(res) as GeneratedResponse;
+                // console.log('Result ', resObj);
+                // this.rootDoc.backgroundColor = resObj.collectionBackgroundColor;
+                // (resObj.documentsWithColors).forEach((elem, i) => (inputDocs[i].backgroundColor = elem.color));
             }
         } catch (err) {
             console.error(err);