Merge branch 'master' into alyssa-starter

4 files changed, 772 insertions, 76 deletions

diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts
index 8447a4934..af25722a4 100644
--- a/src/server/ApiManagers/AssistantManager.ts
+++ b/src/server/ApiManagers/AssistantManager.ts
@@ -15,14 +15,17 @@ import * as fs from 'fs';
 import { writeFile } from 'fs';
 import { google } from 'googleapis';
 import { JSDOM } from 'jsdom';
+import OpenAI from 'openai';
 import * as path from 'path';
 import * as puppeteer from 'puppeteer';
 import { promisify } from 'util';
 import * as uuid from 'uuid';
 import { AI_Document } from '../../client/views/nodes/chatbot/types/types';
+import { DashUploadUtils } from '../DashUploadUtils';
 import { Method } from '../RouteManager';
 import { filesDirectory, publicDirectory } from '../SocketData';
 import ApiManager, { Registration } from './ApiManager';
+import { env } from 'process';
 
 // Enumeration of directories where different file types are stored
 export enum Directory {
@@ -87,6 +90,7 @@ export default class AssistantManager extends ApiManager {
     protected initialize(register: Registration): void {
         // Initialize Google Custom Search API
         const customsearch = google.customsearch('v1');
+        const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY });
 
         // Register Wikipedia summary API route
         register({
@@ -115,29 +119,86 @@ export default class AssistantManager extends ApiManager {
             },
         });
 
-        // Register Google Web Search Results API route
+        // Register an API route to retrieve web search results using Google Custom Search
+        // This route filters results by checking their x-frame-options headers for security purposes
         register({
             method: Method.POST,
             subscription: '/getWebSearchResults',
             secureHandler: async ({ req, res }) => {
                 const { query, max_results } = req.body;
-                try {
-                    // Fetch search results using Google Custom Search API
-                    const response = await customsearch.cse.list({
+                const MIN_VALID_RESULTS_RATIO = 0.75; // 3/4 threshold
+                let startIndex = 1; // Start at the first result initially
+                const fetchSearchResults = async (start: number) => {
+                    return customsearch.cse.list({
                         q: query,
                         cx: process.env._CLIENT_GOOGLE_SEARCH_ENGINE_ID,
                         key: process.env._CLIENT_GOOGLE_API_KEY,
                         safe: 'active',
                         num: max_results,
+                        start, // This controls which result index the search starts from
                     });
+                };
+
+                const filterResultsByXFrameOptions = async (
+                    results: {
+                        url: string | null | undefined;
+                        snippet: string | null | undefined;
+                    }[]
+                ) => {
+                    const filteredResults = await Promise.all(
+                        results
+                            .filter(result => result.url)
+                            .map(async result => {
+                                try {
+                                    const urlResponse = await axios.head(result.url!, { timeout: 5000 });
+                                    const xFrameOptions = urlResponse.headers['x-frame-options'];
+                                    if (xFrameOptions && xFrameOptions.toUpperCase() === 'SAMEORIGIN') {
+                                        return result;
+                                    }
+                                } catch (error) {
+                                    console.error(`Error checking x-frame-options for URL: ${result.url}`, error);
+                                }
+                                return null; // Exclude the result if it doesn't match
+                            })
+                    );
+                    return filteredResults.filter(result => result !== null); // Remove null results
+                };
 
-                    const results =
+                try {
+                    // Fetch initial search results
+                    let response = await fetchSearchResults(startIndex);
+                    const initialResults =
                         response.data.items?.map(item => ({
                             url: item.link,
                             snippet: item.snippet,
                         })) || [];
 
-                    res.send({ results });
+                    // Filter the initial results
+                    let validResults = await filterResultsByXFrameOptions(initialResults);
+
+                    // If valid results are less than 3/4 of max_results, fetch more results
+                    while (validResults.length < max_results * MIN_VALID_RESULTS_RATIO) {
+                        // Increment the start index by the max_results to fetch the next set of results
+                        startIndex += max_results;
+                        response = await fetchSearchResults(startIndex);
+
+                        const additionalResults =
+                            response.data.items?.map(item => ({
+                                url: item.link,
+                                snippet: item.snippet,
+                            })) || [];
+
+                        const additionalValidResults = await filterResultsByXFrameOptions(additionalResults);
+                        validResults = [...validResults, ...additionalValidResults]; // Combine valid results
+
+                        // Break if no more results are available
+                        if (additionalValidResults.length === 0 || response.data.items?.length === 0) {
+                            break;
+                        }
+                    }
+
+                    // Return the filtered valid results
+                    res.send({ results: validResults.slice(0, max_results) }); // Limit the results to max_results
                 } catch (error) {
                     console.error('Error performing web search:', error);
                     res.status(500).send({
@@ -147,6 +208,187 @@ export default class AssistantManager extends ApiManager {
             },
         });
 
+        /**
+         * Converts a video file to audio format using ffmpeg.
+         * @param videoPath The path to the input video file.
+         * @param outputAudioPath The path to the output audio file.
+         * @returns A promise that resolves when the conversion is complete.
+         */
+        function convertVideoToAudio(videoPath: string, outputAudioPath: string): Promise<void> {
+            return new Promise((resolve, reject) => {
+                const ffmpegProcess = spawn('ffmpeg', [
+                    '-i',
+                    videoPath, // Input file
+                    '-vn', // No video
+                    '-acodec',
+                    'pcm_s16le', // Audio codec
+                    '-ac',
+                    '1', // Number of audio channels
+                    '-ar',
+                    '16000', // Audio sampling frequency
+                    '-f',
+                    'wav', // Output format
+                    outputAudioPath, // Output file
+                ]);
+
+                ffmpegProcess.on('error', error => {
+                    console.error('Error running ffmpeg:', error);
+                    reject(error);
+                });
+
+                ffmpegProcess.on('close', code => {
+                    if (code === 0) {
+                        console.log('Audio extraction complete:', outputAudioPath);
+                        resolve();
+                    } else {
+                        reject(new Error(`ffmpeg exited with code ${code}`));
+                    }
+                });
+            });
+        }
+
+        // Register an API route to process a media file (audio or video)
+        // Extracts audio from video files, transcribes the audio using OpenAI Whisper, and provides a summary
+        register({
+            method: Method.POST,
+            subscription: '/processMediaFile',
+            secureHandler: async ({ req, res }) => {
+                const { fileName } = req.body;
+
+                // Ensure the filename is provided
+                if (!fileName) {
+                    res.status(400).send({ error: 'Filename is required' });
+                    return;
+                }
+
+                try {
+                    // Determine the file type and location
+                    const isAudio = fileName.toLowerCase().endsWith('.mp3');
+                    const directory = isAudio ? Directory.audio : Directory.videos;
+                    const filePath = serverPathToFile(directory, fileName);
+
+                    // Check if the file exists
+                    if (!fs.existsSync(filePath)) {
+                        res.status(404).send({ error: 'File not found' });
+                        return;
+                    }
+
+                    console.log(`Processing ${isAudio ? 'audio' : 'video'} file: ${fileName}`);
+
+                    // Step 1: Extract audio if it's a video
+                    let audioPath = filePath;
+                    if (!isAudio) {
+                        const audioFileName = `${path.basename(fileName, path.extname(fileName))}.wav`;
+                        audioPath = path.join(pathToDirectory(Directory.audio), audioFileName);
+
+                        console.log('Extracting audio from video...');
+                        await convertVideoToAudio(filePath, audioPath);
+                    }
+
+                    // Step 2: Transcribe audio using OpenAI Whisper
+                    console.log('Transcribing audio...');
+                    const transcription = await openai.audio.transcriptions.create({
+                        file: fs.createReadStream(audioPath),
+                        model: 'whisper-1',
+                        response_format: 'verbose_json',
+                        timestamp_granularities: ['segment'],
+                    });
+
+                    console.log('Audio transcription complete.');
+
+                    // Step 3: Extract concise JSON
+                    console.log('Extracting concise JSON...');
+                    const originalSegments = transcription.segments?.map((segment, index) => ({
+                        index: index.toString(),
+                        text: segment.text,
+                        start: segment.start,
+                        end: segment.end,
+                    }));
+
+                    interface ConciseSegment {
+                        text: string;
+                        indexes: string[];
+                        start: number | null;
+                        end: number | null;
+                    }
+
+                    const combinedSegments = [];
+                    let currentGroup: ConciseSegment = { text: '', indexes: [], start: null, end: null };
+                    let currentDuration = 0;
+
+                    originalSegments?.forEach(segment => {
+                        const segmentDuration = segment.end - segment.start;
+
+                        if (currentDuration + segmentDuration <= 4000) {
+                            // Add segment to the current group
+                            currentGroup.text += (currentGroup.text ? ' ' : '') + segment.text;
+                            currentGroup.indexes.push(segment.index);
+                            if (currentGroup.start === null) {
+                                currentGroup.start = segment.start;
+                            }
+                            currentGroup.end = segment.end;
+                            currentDuration += segmentDuration;
+                        } else {
+                            // Push the current group and start a new one
+                            combinedSegments.push({ ...currentGroup });
+                            currentGroup = {
+                                text: segment.text,
+                                indexes: [segment.index],
+                                start: segment.start,
+                                end: segment.end,
+                            };
+                            currentDuration = segmentDuration;
+                        }
+                    });
+
+                    // Push the final group if it has content
+                    if (currentGroup.text) {
+                        combinedSegments.push({ ...currentGroup });
+                    }
+                    const lastSegment = combinedSegments[combinedSegments.length - 1];
+
+                    // Check if the last segment is too short and combine it with the second last
+                    if (combinedSegments.length > 1 && lastSegment.end && lastSegment.start) {
+                        const secondLastSegment = combinedSegments[combinedSegments.length - 2];
+                        const lastDuration = lastSegment.end - lastSegment.start;
+
+                        if (lastDuration < 30) {
+                            // Combine the last segment with the second last
+                            secondLastSegment.text += (secondLastSegment.text ? ' ' : '') + lastSegment.text;
+                            secondLastSegment.indexes = secondLastSegment.indexes.concat(lastSegment.indexes);
+                            secondLastSegment.end = lastSegment.end;
+
+                            // Remove the last segment from the array
+                            combinedSegments.pop();
+                        }
+                    }
+
+                    console.log('Segments combined successfully.');
+
+                    console.log('Generating summary using GPT-4...');
+                    const combinedText = combinedSegments.map(segment => segment.text).join(' ');
+
+                    let summary = '';
+                    try {
+                        const completion = await openai.chat.completions.create({
+                            messages: [{ role: 'system', content: `Summarize the following text in a concise paragraph:\n\n${combinedText}` }],
+                            model: 'gpt-4o',
+                        });
+                        console.log('Summary generation complete.');
+                        summary = completion.choices[0].message.content ?? 'Summary could not be generated.';
+                    } catch (summaryError) {
+                        console.error('Error generating summary:', summaryError);
+                        summary = 'Summary could not be generated.';
+                    }
+                    // Step 5: Return the JSON result
+                    res.send({ full: originalSegments, condensed: combinedSegments, summary });
+                } catch (error) {
+                    console.error('Error processing media file:', error);
+                    res.status(500).send({ error: 'Failed to process media file' });
+                }
+            },
+        });
+
         // Axios instance with custom headers for scraping
         const axiosInstance = axios.create({
             headers: {
@@ -181,7 +423,38 @@ export default class AssistantManager extends ApiManager {
             }
         };
 
-        // Register a proxy fetch API route
+        // Register an API route to generate an image using OpenAI's DALL-E model
+        // Uploads the generated image to the server and provides a URL for access
+        register({
+            method: Method.POST,
+            subscription: '/generateImage',
+            secureHandler: async ({ req, res }) => {
+                const { image_prompt } = req.body;
+
+                if (!image_prompt) {
+                    res.status(400).send({ error: 'No prompt provided' });
+                    return;
+                }
+
+                try {
+                    const image = await openai.images.generate({ model: 'dall-e-3', prompt: image_prompt, response_format: 'url' });
+                    console.log(image);
+                    const result = await DashUploadUtils.UploadImage(image.data[0].url!);
+
+                    const url = image.data[0].url;
+
+                    res.send({ result, url });
+                } catch (error) {
+                    console.error('Error fetching the URL:', error);
+                    res.status(500).send({
+                        error: 'Failed to fetch the URL',
+                    });
+                }
+            },
+        });
+
+        // Register an API route to fetch data from a URL using a proxy with retry logic
+        // Useful for bypassing rate limits or scraping inaccessible data
         register({
             method: Method.POST,
             subscription: '/proxyFetch',
@@ -206,6 +479,7 @@ export default class AssistantManager extends ApiManager {
         });
 
         // Register an API route to scrape website content using Puppeteer and JSDOM
+        // Extracts and returns readable content from a given URL
         register({
             method: Method.POST,
             subscription: '/scrapeWebsite',
@@ -246,6 +520,8 @@ export default class AssistantManager extends ApiManager {
             },
         });
 
+        // Register an API route to create a document and start a background job for processing
+        // Uses Python scripts to process files and generate document chunks for further use
         register({
             method: Method.POST,
             subscription: '/createDocument',
@@ -263,7 +539,7 @@ export default class AssistantManager extends ApiManager {
 
                     // Spawn the Python process and track its progress/output
                     // eslint-disable-next-line no-use-before-define
-                    spawnPythonProcess(jobId, file_name, file_data);
+                    spawnPythonProcess(jobId, public_path);
 
                     // Send the job ID back to the client for tracking
                     res.send({ jobId });
@@ -277,6 +553,7 @@ export default class AssistantManager extends ApiManager {
         });
 
         // Register an API route to check the progress of a document creation job
+        // Returns the current step and progress percentage
         register({
             method: Method.GET,
             subscription: '/getProgress/:jobId',
@@ -294,59 +571,30 @@ export default class AssistantManager extends ApiManager {
             },
         });
 
-        // Register an API route to get the final result of a document creation job
+        // Register an API route to retrieve the final result of a document creation job
+        // Returns the processed data or an error status if the job is incomplete
         register({
             method: Method.GET,
             subscription: '/getResult/:jobId',
             secureHandler: async ({ req, res }) => {
-                const { jobId } = req.params; // Get the job ID from the URL parameters
-                // Check if the job result is available
+                const { jobId } = req.params;
                 if (jobResults[jobId]) {
                     const result = jobResults[jobId] as AI_Document & { status: string };
 
-                    // If the result contains image or table chunks, save the base64 data as image files
                     if (result.chunks && Array.isArray(result.chunks)) {
-                        await Promise.all(
-                            result.chunks.map(chunk => {
-                                if (chunk.metadata && (chunk.metadata.type === 'image' || chunk.metadata.type === 'table')) {
-                                    const files_directory = '/files/chunk_images/';
-                                    const directory = path.join(publicDirectory, files_directory);
-
-                                    // Ensure the directory exists or create it
-                                    if (!fs.existsSync(directory)) {
-                                        fs.mkdirSync(directory);
-                                    }
-
-                                    const fileName = path.basename(chunk.metadata.file_path); // Get the file name from the path
-                                    const filePath = path.join(directory, fileName); // Create the full file path
-
-                                    // Check if the chunk contains base64 encoded data
-                                    if (chunk.metadata.base64_data) {
-                                        // Decode the base64 data and write it to a file
-                                        const buffer = Buffer.from(chunk.metadata.base64_data, 'base64');
-                                        fs.promises.writeFile(filePath, buffer).then(() => {
-                                            // Update the file path in the chunk's metadata
-                                            chunk.metadata.file_path = path.join(files_directory, fileName);
-                                            chunk.metadata.base64_data = undefined; // Remove the base64 data from the metadata
-                                        });
-                                    } else {
-                                        console.warn(`No base64_data found for chunk: ${fileName}`);
-                                    }
-                                }
-                            })
-                        );
                         result.status = 'completed';
                     } else {
                         result.status = 'pending';
                     }
-                    res.json(result); // Send the result back to the client
+                    res.json(result);
                 } else {
                     res.status(202).send({ status: 'pending' });
                 }
             },
         });
 
-        // Register an API route to format chunks (e.g., text or image chunks) for display
+        // Register an API route to format chunks of text or images for structured display
+        // Converts raw chunk data into a structured format for frontend consumption
         register({
             method: Method.POST,
             subscription: '/formatChunks',
@@ -367,7 +615,8 @@ export default class AssistantManager extends ApiManager {
                         // If the chunk is an image or table, read the corresponding file and encode it as base64
                         if (chunk.metadata.type === 'image' || chunk.metadata.type === 'table') {
                             try {
-                                const filePath = serverPathToFile(Directory.chunk_images, chunk.metadata.file_path); // Get the file path
+                                const filePath = path.join(pathToDirectory(Directory.chunk_images), chunk.metadata.file_path); // Get the file path
+                                console.log(filePath);
                                 readFileAsync(filePath).then(imageBuffer => {
                                     const base64Image = imageBuffer.toString('base64'); // Convert the image to base64
 
@@ -401,6 +650,7 @@ export default class AssistantManager extends ApiManager {
         });
 
         // Register an API route to create and save a CSV file on the server
+        // Writes the CSV content to a unique file and provides a URL for download
         register({
             method: Method.POST,
             subscription: '/createCSV',
@@ -440,15 +690,23 @@ export default class AssistantManager extends ApiManager {
     }
 }
 
-function spawnPythonProcess(jobId: string, file_name: string, file_data: string) {
+/**
+ * Spawns a Python process to handle file processing tasks.
+ * @param jobId The job ID for tracking progress.
+ * @param file_name The name of the file to process.
+ * @param file_path The filepath of the file to process.
+ */
+function spawnPythonProcess(jobId: string, file_path: string) {
     const venvPath = path.join(__dirname, '../chunker/venv');
     const requirementsPath = path.join(__dirname, '../chunker/requirements.txt');
     const pythonScriptPath = path.join(__dirname, '../chunker/pdf_chunker.py');
 
+    const outputDirectory = pathToDirectory(Directory.chunk_images);
+
     function runPythonScript() {
         const pythonPath = process.platform === 'win32' ? path.join(venvPath, 'Scripts', 'python') : path.join(venvPath, 'bin', 'python3');
 
-        const pythonProcess = spawn(pythonPath, [pythonScriptPath, jobId, file_name, file_data]);
+        const pythonProcess = spawn(pythonPath, [pythonScriptPath, jobId, file_path, outputDirectory]);
 
         let pythonOutput = '';
         let stderrOutput = '';
@@ -460,23 +718,30 @@ function spawnPythonProcess(jobId: string, file_name: string, file_data: string)
         pythonProcess.stderr.on('data', data => {
             stderrOutput += data.toString();
             const lines = stderrOutput.split('\n');
+            stderrOutput = lines.pop() || ''; // Save the last partial line back to stderrOutput
             lines.forEach(line => {
                 if (line.trim()) {
-                    try {
-                        const parsedOutput = JSON.parse(line);
-                        if (parsedOutput.job_id && parsedOutput.progress !== undefined) {
-                            jobProgress[parsedOutput.job_id] = {
-                                step: parsedOutput.step,
-                                progress: parsedOutput.progress,
-                            };
-                        } else if (parsedOutput.progress !== undefined) {
-                            jobProgress[jobId] = {
-                                step: parsedOutput.step,
-                                progress: parsedOutput.progress,
-                            };
+                    if (line.startsWith('PROGRESS:')) {
+                        const jsonString = line.substring('PROGRESS:'.length);
+                        try {
+                            const parsedOutput = JSON.parse(jsonString);
+                            if (parsedOutput.job_id && parsedOutput.progress !== undefined) {
+                                jobProgress[parsedOutput.job_id] = {
+                                    step: parsedOutput.step,
+                                    progress: parsedOutput.progress,
+                                };
+                            } else if (parsedOutput.progress !== undefined) {
+                                jobProgress[jobId] = {
+                                    step: parsedOutput.step,
+                                    progress: parsedOutput.progress,
+                                };
+                            }
+                        } catch (err) {
+                            console.error('Error parsing progress JSON:', jsonString, err);
                         }
-                    } catch (err) {
-                        console.error('Progress log from Python:', line, err);
+                    } else {
+                        // Log other stderr output
+                        console.error('Python stderr:', line);
                     }
                 }
             });
@@ -490,10 +755,24 @@ function spawnPythonProcess(jobId: string, file_name: string, file_data: string)
                     jobProgress[jobId] = { step: 'Complete', progress: 100 };
                 } catch (err) {
                     console.error('Error parsing final JSON result:', err);
+                    jobResults[jobId] = { error: 'Failed to parse final result' };
                 }
             } else {
                 console.error(`Python process exited with code ${code}`);
-                jobResults[jobId] = { error: 'Python process failed' };
+                // Check if there was an error message in stderr
+                if (stderrOutput) {
+                    // Try to parse the last line as JSON
+                    const lines = stderrOutput.trim().split('\n');
+                    const lastLine = lines[lines.length - 1];
+                    try {
+                        const errorOutput = JSON.parse(lastLine);
+                        jobResults[jobId] = errorOutput;
+                    } catch {
+                        jobResults[jobId] = { error: 'Python process failed' };
+                    }
+                } else {
+                    jobResults[jobId] = { error: 'Python process failed' };
+                }
             }
         });
     }
diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts
index 88f22992d..d2028f23b 100644
--- a/src/server/ApiManagers/DataVizManager.ts
+++ b/src/server/ApiManagers/DataVizManager.ts
@@ -9,7 +9,7 @@ export default class DataVizManager extends ApiManager {
         register({
             method: Method.GET,
             subscription: '/csvData',
-            secureHandler: async ({ req, res }) => {
+            secureHandler: ({ req, res }) => {
                 const uri = req.query.uri as string;
 
                 return new Promise<void>(resolve => {
diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
new file mode 100644
index 000000000..e75ede9df
--- /dev/null
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -0,0 +1,410 @@
+import axios from 'axios';
+import { Dropbox } from 'dropbox';
+import * as fs from 'fs';
+import * as multipart from 'parse-multipart-data';
+import * as path from 'path';
+import { DashUserModel } from '../authentication/DashUserModel';
+import { DashUploadUtils } from '../DashUploadUtils';
+import { _error, _invalid, _success, Method } from '../RouteManager';
+import { Directory, filesDirectory } from '../SocketData';
+import ApiManager, { Registration } from './ApiManager';
+
+export default class FireflyManager extends ApiManager {
+    getBearerToken = () =>
+        fetch('https://ims-na1.adobelogin.com/ims/token/v3', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+            },
+            body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`,
+        }).catch(error => {
+            console.error('Error:', error);
+            return undefined;
+        });
+
+    generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[], styleUrl: string | undefined) =>
+        this.getBearerToken().then(response =>
+            response?.json().then((data: { access_token: string }) =>
+                //prettier-ignore
+                fetch('https://firefly-api.adobe.io/v3/images/generate', {
+                    method: 'POST',
+                    headers: [
+                        ['Content-Type', 'application/json'],
+                        ['Accept', 'application/json'],
+                        ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                        ['Authorization', `Bearer ${data.access_token}`],
+                    ],
+                    body: JSON.stringify({
+                        prompt,
+                        numVariations: 4,
+                        detailLevel: 'preview', 
+                        modelVersion: 'image3_fast',
+                        size: { width, height },
+                        structure: !structureUrl
+                            ? undefined
+                            : {
+                                  strength,
+                                  imageReference: {
+                                      source: { url: structureUrl },
+                                  },
+                              },
+                        // prettier-ignore
+                        style: {
+                            presets: styles,
+                            imageReference : !styleUrl 
+                                ? undefined
+                                : {
+                                    source: { url: styleUrl },
+                                }
+                            }
+                    }),
+                })
+                .then(response2 => response2.json().then(json => 
+                {
+                    if (json.outputs?.length)
+                        return (json.outputs as {image: {url:string }}[]).map(output => output.image);
+                    throw new Error(JSON.stringify(json));
+                })
+            )
+            )
+        );
+
+    uploadImageToDropbox = (fileUrl: string, user: DashUserModel | undefined, dbx = new Dropbox({ accessToken: user?.dropboxToken || '' })) =>
+        new Promise<string | Error>((res, rej) =>
+            fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => {
+                if (err) {
+                    console.log('Error: ', err);
+                    rej();
+                } else {
+                    dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents })
+                        .then(response => {
+                            dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' })
+                                .then(link => res(link.result.link))
+                                .catch(e => res(new Error(e.toString())));
+                        })
+                        .catch(e => {
+                            if (user?.dropboxRefresh) {
+                                console.log('*********** try refresh dropbox for: ' + user.email + ' ***********');
+                                this.refreshDropboxToken(user).then(token => {
+                                    if (!token) {
+                                        console.log('Dropbox error: cannot refresh token');
+                                        res(new Error(e.toString()));
+                                    } else {
+                                        const dbxNew = new Dropbox({ accessToken: user.dropboxToken || '' });
+                                        dbxNew
+                                            .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents })
+                                            .then(response => {
+                                                dbxNew
+                                                    .filesGetTemporaryLink({ path: response.result.path_display ?? '' })
+                                                    .then(link => res(link.result.link))
+                                                    .catch(linkErr => res(new Error(linkErr.toString())));
+                                            })
+                                            .catch(uploadErr => {
+                                                console.log('Dropbox error:', uploadErr);
+                                                res(new Error(uploadErr.toString()));
+                                            });
+                                    }
+                                });
+                            } else {
+                                console.log('Dropbox error:', e);
+                                res(new Error(e.toString()));
+                            }
+                        });
+                }
+            })
+        );
+
+    generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => {
+        let body = `{  "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`;
+        if (seed) {
+            console.log('RECEIVED SEED', seed);
+            body = `{  "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`;
+        }
+        const fetched = this.getBearerToken().then(response =>
+            response?.json().then((data: { access_token: string }) =>
+                fetch('https://firefly-api.adobe.io/v3/images/generate', {
+                    method: 'POST',
+                    headers: [
+                        ['Content-Type', 'application/json'],
+                        ['Accept', 'application/json'],
+                        ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                        ['Authorization', `Bearer ${data.access_token}`],
+                    ],
+                    body: body,
+                })
+                    .then(response2 => response2.json())
+                    .then(json => (json.error_code ? json : { seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url }))
+                    .catch(error => {
+                        console.error('Error:', error);
+                        return undefined;
+                    })
+            )
+        );
+        return fetched;
+    };
+    expandImage = (imgUrl: string, prompt?: string) => {
+        const dropboxImgUrl = imgUrl;
+        const fetched = this.getBearerToken().then(response =>
+            response
+                ?.json()
+                .then((data: { access_token: string }) => {
+                    return fetch('https://firefly-api.adobe.io/v3/images/expand', {
+                        method: 'POST',
+                        headers: [
+                            ['Content-Type', 'application/json'],
+                            ['Accept', 'application/json'],
+                            ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                            ['Authorization', `Bearer ${data.access_token}`],
+                        ],
+                        body: JSON.stringify({
+                            image: {
+                                source: {
+                                    url: dropboxImgUrl,
+                                },
+                            },
+                            numVariations: 1,
+                            seeds: [0],
+                            size: {
+                                width: 3048,
+                                height: 2048,
+                            },
+                            prompt: prompt ?? 'cloudy skies',
+                            placement: {
+                                inset: {
+                                    left: 0,
+                                    top: 0,
+                                    right: 0,
+                                    bottom: 0,
+                                },
+                                alignment: {
+                                    horizontal: 'center',
+                                    vertical: 'center',
+                                },
+                            },
+                        }),
+                    });
+                })
+                .then(resp => resp.json())
+        );
+        return fetched;
+    };
+    getImageText = (imageBlob: Blob) => {
+        const inputFileVarName = 'infile';
+        const outputVarName = 'result';
+        const fetched = this.getBearerToken().then(response =>
+            response?.json().then((data: { access_token: string }) => {
+                return fetch('https://sensei.adobe.io/services/v2/predict', {
+                    method: 'POST',
+                    headers: [
+                        ['Prefer', 'respond-async, wait=59'],
+                        ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                        // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string
+                        ['Authorization', `Bearer ${data.access_token}`],
+                    ],
+                    body: ((form: FormData) => {
+                        form.set(inputFileVarName, imageBlob);
+                        form.set(
+                            'contentAnalyzerRequests',
+                            JSON.stringify({
+                                'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                'sensei:invocation_mode': 'asynchronous',
+                                'sensei:invocation_batch': false,
+                                'sensei:engines': [
+                                    {
+                                        'sensei:execution_info': {
+                                            'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                        },
+                                        'sensei:inputs': {
+                                            documents: [
+                                                {
+                                                    'sensei:multipart_field_name': inputFileVarName,
+                                                    'dc:format': 'image/png',
+                                                },
+                                            ],
+                                        },
+                                        'sensei:params': {
+                                            correct_with_dictionary: true,
+                                        },
+                                        'sensei:outputs': {
+                                            result: {
+                                                'sensei:multipart_field_name': outputVarName,
+                                                'dc:format': 'application/json',
+                                            },
+                                        },
+                                    },
+                                ],
+                            })
+                        );
+                        return form;
+                    })(new FormData()),
+                }).then(response2 => {
+                    const contentType = response2.headers.get('content-type') ?? '';
+                    if (contentType.includes('application/json')) {
+                        return response2.json().then((json: object) => JSON.stringify(json));
+                    }
+                    if (contentType.includes('multipart')) {
+                        return response2
+                            .arrayBuffer()
+                            .then(arrayBuffer =>
+                                multipart
+                                    .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? ''))
+                                    .filter(part => part.name === outputVarName)
+                                    .map(part => JSON.parse(part.data.toString())[0])
+                                    .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '')
+                            )
+                            .catch(error => {
+                                console.error('Error:', error);
+                                return '';
+                            });
+                    }
+                    return response2.text();
+                });
+            })
+        );
+        return fetched;
+    };
+
+    refreshDropboxToken = (user: DashUserModel) =>
+        axios
+            .post(
+                'https://api.dropbox.com/oauth2/token',
+                new URLSearchParams({
+                    refresh_token: user.dropboxRefresh || '',
+                    grant_type: 'refresh_token',
+                    client_id: process.env._CLIENT_DROPBOX_CLIENT_ID || '',
+                    client_secret: process.env._CLIENT_DROPBOX_SECRET || '',
+                }).toString()
+            )
+            .then(refresh => {
+                console.log('***** dropbox token refreshed for ' + user?.email + ' ******* ');
+                user.dropboxToken = refresh.data.access_token;
+                user.save();
+                return user.dropboxToken;
+            })
+            .catch(e => {
+                console.log(e);
+                return undefined;
+            });
+
+    protected initialize(register: Registration): void {
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImageFromStructure',
+            secureHandler: ({ req, res }) =>
+                new Promise<void>(resolver => {
+                    (req.body.styleUrl ? this.uploadImageToDropbox(req.body.styleUrl, req.user as DashUserModel) : Promise.resolve(undefined))
+                        .then(styleUrl => {
+                            if (styleUrl instanceof Error) {
+                                _invalid(res, styleUrl.message);
+                                throw new Error('Error uploading images to dropbox');
+                            }
+                            this.uploadImageToDropbox(req.body.structureUrl, req.user as DashUserModel)
+                                .then(dropboxStructureUrl => {
+                                    if (dropboxStructureUrl instanceof Error) {
+                                        _invalid(res, dropboxStructureUrl.message);
+                                        throw new Error('Error uploading images to dropbox');
+                                    }
+                                    return { styleUrl, structureUrl: dropboxStructureUrl };
+                                })
+                                .then(uploads =>
+                                    this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploads.structureUrl, req.body.strength, req.body.presets, uploads.styleUrl)
+                                        .then(images => {
+                                            Promise.all((images ?? [new Error('no images were generated')]).map(fire => (fire instanceof Error ? fire : DashUploadUtils.UploadImage(fire.url))))
+                                                .then(dashImages => {
+                                                    if (dashImages.every(img => img instanceof Error)) _invalid(res, dashImages[0]!.message);
+                                                    else _success(res, JSON.stringify(dashImages.filter(img => !(img instanceof Error))));
+                                                })
+                                                .then(resolver);
+                                        })
+                                        .catch(e => {
+                                            _invalid(res, e.message);
+                                            resolver();
+                                        })
+                                );
+                        })
+                        .catch(() => {
+                            /* do nothing */
+                            resolver();
+                        });
+                }),
+        });
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImage',
+            secureHandler: ({ req, res }) =>
+                this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img =>
+                    img.error_code
+                        ? _invalid(res, img.message)
+                        : DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => {
+                              if (info instanceof Error) _invalid(res, info.message);
+                              else _success(res, info);
+                          })
+                ),
+        });
+
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImageText',
+            secureHandler: ({ req, res }) =>
+                fetch(req.body.file).then(json =>
+                    json.blob().then(file =>
+                        this.getImageText(file).then(text => {
+                            _success(res, text);
+                        })
+                    )
+                ),
+        });
+        register({
+            method: Method.POST,
+            subscription: '/expandImage',
+            secureHandler: ({ req, res }) =>
+                this.uploadImageToDropbox(req.body.file, req.user as DashUserModel).then(uploadUrl =>
+                    uploadUrl instanceof Error
+                        ? _invalid(res, uploadUrl.message)
+                        : this.expandImage(uploadUrl, req.body.prompt).then(text => {
+                              if (text.error_code) _error(res, text.message);
+                              else
+                                  DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => {
+                                      if (info instanceof Error) _invalid(res, info.message);
+                                      else _success(res, info);
+                                  });
+                          })
+                ),
+        });
+
+        // construct this url and send user to it.  It will allow them to authorize their dropbox account and will send the resulting token to our endpoint /refreshDropbox
+        // https://www.dropbox.com/oauth2/authorize?client_id=DROPBOX_CLIENT_ID&response_type=code&token_access_type=offline&redirect_uri=http://localhost:1050/refreshDropbox
+        // see: https://dropbox.tech/developers/using-oauth-2-0-with-offline-access
+        //
+        register({
+            method: Method.GET,
+            subscription: '/refreshDropbox',
+            secureHandler: ({ req, res }) => {
+                const user = req.user as DashUserModel;
+                console.log(`******************* dropbox authorized for ${user?.email} ******************`);
+                _success(res, 'dropbox authorized for ' + user?.email);
+
+                const data = new URLSearchParams({
+                    code: req.query.code as string,
+                    grant_type: 'authorization_code',
+                    client_id: process.env._CLIENT_DROPBOX_CLIENT_ID ?? '',
+                    client_secret: process.env._CLIENT_DROPBOX_SECRET ?? '',
+                    redirect_uri: 'http://localhost:1050/refreshDropbox',
+                });
+                axios
+                    .post('https://api.dropbox.com/oauth2/token', data.toString())
+                    .then(response => {
+                        console.log('***** dropbox token (and refresh) received for ' + user?.email + ' ******* ');
+                        user.dropboxToken = response.data.access_token;
+                        user.dropboxRefresh = response.data.refresh_token;
+                        user.save();
+
+                        setTimeout(() => this.refreshDropboxToken(user), response.data.expires_in - 600);
+                    })
+                    .catch(e => {
+                        console.log(e);
+                    });
+            },
+        });
+    }
+}
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 868373474..c9d5df547 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -70,10 +70,16 @@ export default class UploadManager extends ApiManager {
                             ]);
                         } else {
                             fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`));
+                            // original filenames with '.'s, such as a Macbook screenshot, can be a problem - their extension is not kept in formidable's newFilename.
+                            // This makes sure that the extension is preserved in the newFilename.
+                            const fixNewFilename = (f: formidable.File) => {
+                                if (path.extname(f.originalFilename ?? '') !== path.extname(f.newFilename)) f.newFilename = f.newFilename + path.extname(f.originalFilename ?? '');
+                                return f;
+                            };
                             const results = (
                                 await Promise.all(
                                     Array.from(Object.keys(files)).map(
-                                        async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress.
+                                        async key => (!files[key] ? undefined : DashUploadUtils.upload(fixNewFilename(files[key][0]) /* , key */)) // key is the guid used by the client to track upload progress.
                                     )
                                 )
                             ).filter(result => result && !(result.result instanceof Error));
@@ -147,13 +153,10 @@ export default class UploadManager extends ApiManager {
                     if (doc.id) {
                         doc.id = getId(doc.id);
                     }
-                    // eslint-disable-next-line no-restricted-syntax
                     for (const key in doc.fields) {
-                        // eslint-disable-next-line no-continue
                         if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue;
 
                         const field = doc.fields[key];
-                        // eslint-disable-next-line no-continue
                         if (field === undefined || field === null) continue;
 
                         if (field.__type === 'Doc') {
@@ -182,11 +185,9 @@ export default class UploadManager extends ApiManager {
                         let docids: string[] = [];
                         let linkids: string[] = [];
                         try {
-                            // eslint-disable-next-line no-restricted-syntax
                             for (const name in files) {
                                 if (Object.prototype.hasOwnProperty.call(files, name)) {
                                     const f = files[name];
-                                    // eslint-disable-next-line no-continue
                                     if (!f) continue;
                                     const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set?
                                     const zip = new AdmZip(path2.filepath);
@@ -273,14 +274,20 @@ export default class UploadManager extends ApiManager {
                         .filter(f => regex.test(f))
                         .map(f => fs.unlinkSync(serverPath + f));
                 }
-                imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => {
-                    const ext = path.extname(savedName).toLowerCase();
-                    const outputPath = serverPathToFile(Directory.images, filename + ext);
-                    if (AcceptableMedia.imageFormats.includes(ext)) {
-                        workerResample(savedName, outputPath, origSuffix, false);
-                    }
-                    res.send(clientPathToFile(Directory.images, filename + ext));
-                });
+                imageDataUri
+                    .outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix)))
+                    .then((savedName: string) => {
+                        const ext = path.extname(savedName).toLowerCase();
+                        const outputPath = serverPathToFile(Directory.images, filename + ext);
+                        if (AcceptableMedia.imageFormats.includes(ext)) {
+                            workerResample(savedName, outputPath, origSuffix, false);
+                        }
+                        res.send(clientPathToFile(Directory.images, filename + ext));
+                    })
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    .catch((e: any) => {
+                        res.status(404).json({ error: e.toString() });
+                    });
             },
         });
     }