diff options
23 files changed, 206 insertions, 935 deletions
diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index c644308b7..e3c4609c0 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -277,11 +277,11 @@ export namespace DocServer { const fetchDocPromises: Map<string, Promise<Opt<Doc>>> = new Map(); // { p: Promise<Doc>; id: string }[] = []; // promises to fetch the value for a requested Doc // Determine which requested documents need to be fetched - // eslint-disable-next-line no-restricted-syntax for (const id of ids.filter(filterid => filterid)) { if (_cache[id] === undefined) { // EMPTY CACHE - make promise that we resolve after all batch-requested Docs have been fetched and deserialized and we know we have this Doc - const fetchPromise = new Promise<Opt<Doc>>(res => + // eslint-disable-next-line no-loop-func + _cache[id] = new Promise<Opt<Doc>>(res => allCachesFilledPromise.then(() => { // if all Docs have been cached, then we can be sure the fetched Doc has been found and cached. So return it to anyone who had been awaiting it. const cache = _cache[id]; @@ -289,8 +289,7 @@ export namespace DocServer { res(cache instanceof Doc ? cache : undefined); }) ); - // eslint-disable-next-line no-loop-func - fetchDocPromises.set(id, (_cache[id] = fetchPromise)); + fetchDocPromises.set(id, _cache[id]); uncachedRequestedIds.push(id); // add to list of Doc requests from server } // else CACHED => do nothing, Doc or promise of Doc is already in cache @@ -307,11 +306,11 @@ export namespace DocServer { let processed = 0; console.log('Retrieved ' + serializedFields.length + ' fields'); // After the serialized Docs have been received, deserialize them into objects. - // eslint-disable-next-line no-restricted-syntax for (const field of serializedFields) { - // eslint-disable-next-line no-await-in-loop ++processed % 150 === 0 && + // eslint-disable-next-line no-await-in-loop (await new Promise<number>( + // eslint-disable-next-line no-loop-func res => setTimeout(action(() => res(FieldLoader.ServerLoadStatus.retrieved = processed))) // prettier-ignore )); // force loading to yield to splash screen rendering to update progress diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 8488c5293..113250a99 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -50,9 +50,7 @@ interface GestureOverlayProps { * drew or perform the gesture's action */ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChildren<GestureOverlayProps>> { - // eslint-disable-next-line no-use-before-define static Instance: GestureOverlay; - // eslint-disable-next-line no-use-before-define static Instances: GestureOverlay[] = []; @observable public SavedColor?: string = undefined; diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index 06463b2a2..f7c4b464c 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -58,7 +58,6 @@ interface PropertiesViewProps { export class PropertiesView extends ObservableReactComponent<PropertiesViewProps> { private _widthUndo?: UndoManager.Batch; - // eslint-disable-next-line no-use-before-define public static Instance: PropertiesView; constructor(props: PropertiesViewProps) { super(props); @@ -1187,6 +1186,7 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps setFinalNumber = () => { this._sliderKey = ''; this._sliderBatch?.end(); + this._sliderBatch = undefined; }; getNumber = (label: string, unit: string, min: number, max: number, number: number, setNumber: (val: number) => void, autorange?: number, autorangeMinVal?: number) => { @@ -1198,7 +1198,6 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps key={key} onPointerDown={() => { this._sliderKey = key; - this._sliderBatch = UndoManager.StartBatch('slider ' + label); }} multithumb={false} color={this.color} @@ -1211,7 +1210,10 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps unit={unit} decimals={1} setFinalNumber={this.setFinalNumber} - setNumber={setNumber} + setNumber={e => { + if (!this._sliderBatch) this._sliderBatch = UndoManager.StartBatch('slider ' + label); + setNumber(e); + }} fillWidth /> </div> diff --git a/src/client/views/collections/CollectionStackingView.scss b/src/client/views/collections/CollectionStackingView.scss index 2cf361847..ec81eb305 100644 --- a/src/client/views/collections/CollectionStackingView.scss +++ b/src/client/views/collections/CollectionStackingView.scss @@ -18,7 +18,7 @@ .collectionStackingView-columnDragger { width: 28px; height: 28px; - position: relative; + position: absolute; margin-left: -5px; z-index: 10; > svg { diff --git a/src/client/views/collections/CollectionStackingView.tsx b/src/client/views/collections/CollectionStackingView.tsx index fbdd23315..7ff90ee53 100644 --- a/src/client/views/collections/CollectionStackingView.tsx +++ b/src/client/views/collections/CollectionStackingView.tsx @@ -417,8 +417,8 @@ export class CollectionStackingView extends CollectionSubView<Partial<collection className="collectionStackingView-columnDragger" onPointerDown={this.columnDividerDown} ref={this._draggerRef} - style={{ cursor: this._cursor, color: SettingsManager.userColor, left: `${NumCast(this.Document._layout_columnWidth) + this.xMargin}px` }}> - <FontAwesomeIcon icon="arrows-alt-h" /> + style={{ cursor: this._cursor, color: SettingsManager.userColor, left: `${NumCast(this.Document._layout_columnWidth)}px` }}> + <FontAwesomeIcon icon="arrows-alt-h" size="sm" /> </div> ); } @@ -563,7 +563,7 @@ export class CollectionStackingView extends CollectionSubView<Partial<collection const rows = () => (!this.isStackingView ? 1 : Math.max(1, Math.min(docList.length, Math.floor((this._props.PanelWidth() - 2 * this.xMargin) / (this.columnWidth + this.gridGap))))); return ( <div key={(heading?.heading ?? '') + 'head'}> - {this._props.isContentActive() && !this.isStackingView && !this.chromeHidden ? this.columnDragger : null} + {!this.isStackingView && !this.chromeHidden ? this.columnDragger : null} <div style={{ position: 'relative' }}> <CollectionMasonryViewFieldRow showHandle={first} diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 76b9fd8db..3e6aa777f 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -107,7 +107,6 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection /** * The Freeformview below the cursor at the start of a gesture (that receives the pointerDown event). Used by GestureOverlay to determine the doc a gesture should apply to. */ - // eslint-disable-next-line no-use-before-define public static DownFfview: CollectionFreeFormView | undefined; // the first DocView that receives a pointerdown event. used by GestureOverlay to determine the doc a gesture should apply to. private _clusters = new CollectionFreeFormClusters(this); @@ -2204,10 +2203,6 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection @observable private _regenerateLoading = false; @observable private _regenInput = ''; - @observable private _canInteract = true; - @observable private _drawingFillInput = ''; - @observable private _regenLoading = false; - @observable private _drawingFillLoading = false; @observable private _fireflyRefStrength = 0; componentAIView = () => { @@ -2220,7 +2215,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection aria-label="Edit instructions input" type="text" value={this._regenInput || StrCast(this.Document.title)} - onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} + onChange={action(e => (this._regenInput = e.target.value))} placeholder={this._regenInput || StrCast(this.Document.title)} /> <div className="imageBox-aiView-regenerate-createBtn"> @@ -2253,10 +2248,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection max={100} number={this._fireflyRefStrength} size={Size.XXSMALL} - setNumber={undoable( - action(val => this._canInteract && (this._fireflyRefStrength = val as number)), - `${this.Document.title} button set from list` - )} + setNumber={undoable(action(val => (this._fireflyRefStrength = val as number)),`${this.Document.title} button set from list` )} fillWidth /> </div> diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index bd71115db..90edab3a7 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -743,6 +743,12 @@ export class DocumentViewInternal extends DocComponent<DocumentViewProps & Field widgetOverlayFunc = () => (this.widgetDecorations ? this.widgetOverlay : null); viewingAiEditor = () => (this._props.showAIEditor && this._componentView?.componentAIView?.() !== undefined ? this.aiEditor : null); @observable _contentsRef: DocumentContentsView | undefined = undefined; + screenToContentsXf = () => + this.viewingAiEditor() + ? this.layoutDoc.layout_reflowHorizontal + ? this._props.ScreenToLocalTransform().scale(Math.min(this.aiContentsWidth() / this._props.PanelWidth(), this.aiContentsHeight() / this._props.PanelHeight())) + : this._props.ScreenToLocalTransform().translate((this._props.PanelWidth() - this.aiContentsWidth()) / 2, 0) + : this._props.ScreenToLocalTransform(); @computed get viewBoxContents() { TraceMobx(); const isInk = this.layoutDoc._layout_isSvg && !this._props.LayoutTemplateString; @@ -762,6 +768,7 @@ export class DocumentViewInternal extends DocComponent<DocumentViewProps & Field layoutFieldKey={StrCast(this.Document.layout_fieldKey, 'layout')} pointerEvents={this.contentPointerEvents} setContentViewBox={this.setContentView} + ScreenToLocalTransform={this.screenToContentsXf} childFilters={this.childFilters} PanelWidth={this.viewingAiEditor() ? this.aiContentsWidth : this._props.PanelWidth} PanelHeight={this.viewingAiEditor() ? this.aiContentsHeight : this.panelHeight} diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index fb2346bd1..617a09ed5 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -99,7 +99,6 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // variables for AI Image Editor @observable private _regenInput = ''; - @observable private _canInteract = true; @observable private _regenerateLoading = false; // Add these observable properties to the ImageBox class @@ -716,7 +715,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @computed get nativeSize() { TraceMobx(); - if (this.paths.length && this.paths[0].includes(DefaultPath)) return { nativeWidth: NumCast(this.layoutDoc._width), nativeHeight: NumCast(this.layoutDoc._height), nativeOrientation: 0 }; + if (this.paths[0]?.includes(DefaultPath) && this.layoutDoc._height) return { nativeWidth: NumCast(this.layoutDoc._width), nativeHeight: NumCast(this.layoutDoc._height), nativeOrientation: 0 }; const { nativeWidth, nativeHeight } = this.imgNativeSize; const nativeOrientation = NumCast(this.dataDoc[this.fieldKey + '_nativeOrientation'], 1); return { nativeWidth, nativeHeight, nativeOrientation }; @@ -902,7 +901,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { aria-label="Edit instructions input" type="text" value={this._regenInput || StrCast(this.Document.title)} - onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} + onChange={action(e => (this._regenInput = e.target.value))} placeholder={this._regenInput || StrCast(this.Document.title)} /> <div className="imageBox-aiView-regenerate-createBtn"> @@ -936,7 +935,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { number={this._fireflyRefStrength} size={Size.XXSMALL} setNumber={undoable( - action(val => this._canInteract && (this._fireflyRefStrength = val as number)), + action(val => (this._fireflyRefStrength = val as number)), `${this.Document.title} button set from list` )} fillWidth diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index 198b8e713..abe235ad5 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -286,8 +286,8 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc const maskBlob = await ImageUtility.canvasToBlob(canvasMask); const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); const res = await ImageUtility.getEdit(imgBlob, maskBlob, input || 'Fill in the image in the same style', 2); - if ((res as any).status == 'error') { - alert((res as any).message); + if (res.status == 'error') { + alert(res.message); } // create first image diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index f773957e7..e9e3335f9 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -62,7 +62,7 @@ export class DrawingFillHandler { ai_prompt: newPrompt, tags: new List<string>(['@ai']), title: newPrompt, - _data_usePath: 'alternate:hover', + //_data_usePath: 'alternate:hover', data_alternates: new List<Doc>([drawing]), _width: 500, data_nativeWidth: info.nativeWidth, diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index b7ff5fff7..a017dd7eb 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -450,7 +450,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { }, }} checked={this._generateDrawing} - onChange={() => this._canInteract && (this._generateDrawing = !this._generateDrawing)} + onChange={action(() => this._canInteract && (this._generateDrawing = !this._generateDrawing))} /> </div> <div className="image-checkbox"> diff --git a/src/fields/ObjectField.ts b/src/fields/ObjectField.ts index c533cb596..1e1dbcd62 100644 --- a/src/fields/ObjectField.ts +++ b/src/fields/ObjectField.ts @@ -3,10 +3,10 @@ import { Copy, FieldChanged, Parent, ToJavascriptString, ToScriptString, ToStrin import { RefField } from './RefField'; export type serializedFieldType = { fieldId: string; heading?: string; __type: string }; -export type serializedFieldsType = { [key: string]: { fields: serializedFieldType[] } }; +export type serializedFieldsType = { [key: string]: { fields: serializedFieldType[] } | string }; export interface serializedDoctype { readonly id: string; - readonly fields?: serializedFieldsType; + readonly fields?: serializedFieldsType | serializedFieldType; } export type serverOpType = { @@ -18,12 +18,10 @@ export type serverOpType = { export abstract class ObjectField { // prettier-ignore public [FieldChanged]?: (diff?: { op: '$addToSet' | '$remFromSet' | '$set'; - // eslint-disable-next-line no-use-before-define items: FieldType[] | undefined; length: number | undefined; hint?: { deleteCount: number, start: number} }, serverOp?: serverOpType) => void; - // eslint-disable-next-line no-use-before-define public [Parent]?: RefField | ObjectField; abstract [Copy](): ObjectField; diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 07c970a4e..b917f555c 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -39,7 +39,6 @@ export enum Directory { csv = 'csv', chunk_images = 'chunk_images', scrape_images = 'scrape_images', - vectorstore = 'vectorstore', } // In-memory job tracking @@ -93,132 +92,6 @@ export default class AssistantManager extends ApiManager { const customsearch = google.customsearch('v1'); const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY }); - // Register an endpoint to retrieve file summaries from the json file - register({ - method: Method.GET, - subscription: '/getFileSummaries', - secureHandler: async ({ req, res }) => { - try { - // Read the file summaries JSON file - const filePath = path.join(filesDirectory, Directory.vectorstore, 'file_summaries.json'); - - if (!fs.existsSync(filePath)) { - res.status(404).send({ error: 'File summaries not found' }); - return; - } - - const data = fs.readFileSync(filePath, 'utf8'); - res.send(data); - } catch (error) { - console.error('Error retrieving file summaries:', error); - res.status(500).send({ - error: 'Failed to retrieve file summaries', - }); - } - }, - }); - - // Register an endpoint to retrieve file names from the file_summaries.json file - register({ - method: Method.GET, - subscription: '/getFileNames', - secureHandler: async ({ res }) => { - const filePath = path.join(filesDirectory, Directory.vectorstore, 'file_summaries.json'); - const data = fs.readFileSync(filePath, 'utf8'); - console.log(Object.keys(JSON.parse(data))); - - res.send(Object.keys(JSON.parse(data))); - }, - }); - - // Register an endpoint to retrieve file content from the content json file - register({ - method: Method.POST, - subscription: '/getFileContent', - secureHandler: async ({ req, res }) => { - const { filepath } = req.body; - - if (!filepath) { - res.status(400).send({ error: 'Filepath is required' }); - return; - } - - try { - // Read the file content JSON file - const filePath = path.join(filesDirectory, Directory.vectorstore, 'file_content.json'); - - if (!fs.existsSync(filePath)) { - res.status(404).send({ error: 'File content database not found' }); - return; - } - - console.log(`[DEBUG] Retrieving content for: ${filepath}`); - - // Read the JSON file in chunks to handle large files - const readStream = fs.createReadStream(filePath, { encoding: 'utf8' }); - let jsonData = ''; - - readStream.on('data', chunk => { - jsonData += chunk; - }); - - readStream.on('end', () => { - try { - // Parse the JSON - const contentMap = JSON.parse(jsonData); - - // Check if the filepath exists in the map - if (!contentMap[filepath]) { - console.log(`[DEBUG] Content not found for: ${filepath}`); - res.status(404).send({ error: `Content not found for filepath: ${filepath}` }); - return; - } - - // Return the file content as is, not as JSON - console.log(`[DEBUG] Found content for: ${filepath} (${contentMap[filepath].length} chars)`); - res.send(contentMap[filepath]); - } catch (parseError) { - console.error('Error parsing file_content.json:', parseError); - res.status(500).send({ - error: 'Failed to parse file content database', - }); - } - }); - - readStream.on('error', streamError => { - console.error('Error reading file_content.json:', streamError); - res.status(500).send({ - error: 'Failed to read file content database', - }); - }); - } catch (error) { - console.error('Error retrieving file content:', error); - res.status(500).send({ - error: 'Failed to retrieve file content', - }); - } - }, - }); - - // Register an endpoint to search file summaries - register({ - method: Method.POST, - subscription: '/searchFileSummaries', - secureHandler: async ({ req, res }) => { - const { query, topK } = req.body; - - if (!query) { - res.status(400).send({ error: 'Search query is required' }); - return; - } - - // This endpoint will be called by the client-side Vectorstore to perform the search - // The actual search is implemented in the Vectorstore class - - res.send({ message: 'This endpoint should be called through the Vectorstore class' }); - }, - }); - // Register Wikipedia summary API route register({ method: Method.POST, @@ -566,9 +439,9 @@ export default class AssistantManager extends ApiManager { try { const image = await openai.images.generate({ model: 'dall-e-3', prompt: image_prompt, response_format: 'url' }); console.log(image); - const result = await DashUploadUtils.UploadImage(image.data[0].url!); + const url = image.data?.[0].url; - const url = image.data[0].url; + const result = url ? await DashUploadUtils.UploadImage(url) : { error: 'Image generation failed' }; res.send({ result, url }); } catch (error) { @@ -612,76 +485,36 @@ export default class AssistantManager extends ApiManager { subscription: '/scrapeWebsite', secureHandler: async ({ req, res }) => { const { url } = req.body; - let browser = null; try { - // Set a longer timeout for slow-loading pages - const navigationTimeout = 60000; // 60 seconds - // Launch Puppeteer browser to navigate to the webpage - browser = await puppeteer.launch({ - args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], + const browser = await puppeteer.launch({ + args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'); - - // Set timeout for navigation - page.setDefaultNavigationTimeout(navigationTimeout); - - // Navigate with timeout and wait for content to load - await page.goto(url, { - waitUntil: 'networkidle2', - timeout: navigationTimeout, - }); - - // Wait a bit longer to ensure dynamic content loads - await new Promise(resolve => setTimeout(resolve, 2000)); + await page.goto(url, { waitUntil: 'networkidle2' }); // Extract HTML content const htmlContent = await page.content(); await browser.close(); - browser = null; - let extractedText = ''; + // Parse HTML content using JSDOM + const dom = new JSDOM(htmlContent, { url }); - // First try with Readability - try { - // Parse HTML content using JSDOM - const dom = new JSDOM(htmlContent, { url }); - - // Extract readable content using Mozilla's Readability API - const reader = new Readability(dom.window.document, { - // Readability configuration to focus on text content - charThreshold: 100, - keepClasses: false, - }); - const article = reader.parse(); + // Extract readable content using Mozilla's Readability API + const reader = new Readability(dom.window.document); + const article = reader.parse(); - if (article && article.textContent) { - extractedText = article.textContent; - } else { - // If Readability doesn't return useful content, try alternate method - extractedText = await extractEnhancedContent(htmlContent); - } - } catch (parsingError) { - console.error('Error parsing website content with Readability:', parsingError); - // Fallback to enhanced content extraction - extractedText = await extractEnhancedContent(htmlContent); + if (article) { + const plainText = article.textContent; + res.send({ website_plain_text: plainText }); + } else { + res.status(500).send({ error: 'Failed to extract readable content' }); } - - // Clean up the extracted text - extractedText = cleanupText(extractedText); - - res.send({ website_plain_text: extractedText }); } catch (error) { console.error('Error scraping website:', error); - - // Clean up browser if still open - if (browser) { - await browser.close().catch(e => console.error('Error closing browser:', e)); - } - res.status(500).send({ - error: 'Failed to scrape website: ' + ((error as Error).message || 'Unknown error'), + error: 'Failed to scrape website', }); } }, @@ -693,20 +526,20 @@ export default class AssistantManager extends ApiManager { method: Method.POST, subscription: '/createDocument', secureHandler: async ({ req, res }) => { - const { file_path, doc_id } = req.body; + const { file_path } = req.body; const public_path = path.join(publicDirectory, file_path); // Resolve the file path in the public directory const file_name = path.basename(file_path); // Extract the file name from the path try { // Read the file data and encode it as base64 - const file_data: string = fs.readFileSync(public_path, { encoding: 'base64' }); + const file_data = fs.readFileSync(public_path, { encoding: 'base64' }); // Generate a unique job ID for tracking const jobId = uuid.v4(); // Spawn the Python process and track its progress/output // eslint-disable-next-line no-use-before-define - spawnPythonProcess(jobId, public_path, doc_id); + spawnPythonProcess(jobId, public_path); // Send the job ID back to the client for tracking res.send({ jobId }); @@ -854,193 +687,6 @@ export default class AssistantManager extends ApiManager { } }, }); - - // Register an API route to capture a screenshot of a webpage using Puppeteer - // and return the image URL for display in the WebBox component - register({ - method: Method.POST, - subscription: '/captureWebScreenshot', - secureHandler: async ({ req, res }) => { - const { url, width, height, fullPage } = req.body; - - if (!url) { - res.status(400).send({ error: 'URL is required' }); - return; - } - - let browser = null; - try { - // Increase timeout for websites that load slowly - const navigationTimeout = 60000; // 60 seconds - - // Launch a headless browser with additional options to improve stability - browser = await puppeteer.launch({ - headless: true, // Use headless mode - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - '--disable-accelerated-2d-canvas', - '--disable-gpu', - '--window-size=1200,800', - '--disable-web-security', // Helps with cross-origin issues - '--disable-features=IsolateOrigins,site-per-process', // Helps with frames - ], - timeout: navigationTimeout, - }); - - const page = await browser.newPage(); - - // Set a larger viewport to capture more content - await page.setViewport({ - width: Number(width) || 1200, - height: Number(height) || 800, - deviceScaleFactor: 1, - }); - - // Enable request interception to speed up page loading - await page.setRequestInterception(true); - page.on('request', request => { - // Skip unnecessary resources to speed up loading - const resourceType = request.resourceType(); - if (resourceType === 'font' || resourceType === 'media' || resourceType === 'websocket' || request.url().includes('analytics') || request.url().includes('tracker')) { - request.abort(); - } else { - request.continue(); - } - }); - - // Set navigation and timeout options - console.log(`Navigating to URL: ${url}`); - - // Navigate to the URL and wait for the page to load - await page.goto(url, { - waitUntil: ['networkidle2'], - timeout: navigationTimeout, - }); - - // Wait for a short delay after navigation to allow content to render - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Take a screenshot - console.log('Taking screenshot...'); - const screenshotPath = `./src/server/public/files/images/webpage_${Date.now()}.png`; - const screenshotOptions = { - path: screenshotPath, - fullPage: fullPage === true, - omitBackground: false, - type: 'png' as 'png', - clip: - fullPage !== true - ? { - x: 0, - y: 0, - width: Number(width) || 1200, - height: Number(height) || 800, - } - : undefined, - }; - - await page.screenshot(screenshotOptions); - - // Get the full height of the page - const fullHeight = await page.evaluate(() => { - return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight, document.body.offsetHeight, document.documentElement.offsetHeight, document.body.clientHeight, document.documentElement.clientHeight); - }); - - console.log(`Screenshot captured successfully with height: ${fullHeight}px`); - - // Return the URL to the screenshot - const screenshotUrl = `/files/images/webpage_${Date.now()}.png`; - res.json({ - screenshotUrl, - fullHeight, - }); - } catch (error: any) { - console.error('Error capturing screenshot:', error); - res.status(500).send({ - error: `Failed to capture screenshot: ${error.message}`, - details: error.stack, - }); - } finally { - // Ensure browser is closed to free resources - if (browser) { - try { - await browser.close(); - console.log('Browser closed successfully'); - } catch (error) { - console.error('Error closing browser:', error); - } - } - } - }, - }); - - // Register an endpoint to retrieve raw file content as plain text (no JSON parsing) - register({ - method: Method.POST, - subscription: '/getRawFileContent', - secureHandler: async ({ req, res }) => { - const { filepath } = req.body; - - if (!filepath) { - res.status(400).send('Filepath is required'); - return; - } - - try { - // Read the file content JSON file - const filePath = path.join(filesDirectory, Directory.vectorstore, 'file_content.json'); - - if (!fs.existsSync(filePath)) { - res.status(404).send('File content database not found'); - return; - } - - console.log(`[DEBUG] Retrieving raw content for: ${filepath}`); - - // Read the JSON file - const readStream = fs.createReadStream(filePath, { encoding: 'utf8' }); - let jsonData = ''; - - readStream.on('data', chunk => { - jsonData += chunk; - }); - - readStream.on('end', () => { - try { - // Parse the JSON - const contentMap = JSON.parse(jsonData); - - // Check if the filepath exists in the map - if (!contentMap[filepath]) { - console.log(`[DEBUG] Content not found for: ${filepath}`); - res.status(404).send(`Content not found for filepath: ${filepath}`); - return; - } - - // Set content type to plain text to avoid JSON parsing - res.setHeader('Content-Type', 'text/plain'); - - // Return the file content as plain text - console.log(`[DEBUG] Found content for: ${filepath} (${contentMap[filepath].length} chars)`); - res.send(contentMap[filepath]); - } catch (parseError) { - console.error('Error parsing file_content.json:', parseError); - res.status(500).send('Failed to parse file content database'); - } - }); - - readStream.on('error', streamError => { - console.error('Error reading file_content.json:', streamError); - res.status(500).send('Failed to read file content database'); - }); - } catch (error) { - console.error('Error retrieving file content:', error); - res.status(500).send('Failed to retrieve file content'); - } - }, - }); } } @@ -1050,7 +696,7 @@ export default class AssistantManager extends ApiManager { * @param file_name The name of the file to process. * @param file_path The filepath of the file to process. */ -function spawnPythonProcess(jobId: string, file_path: string, doc_id: string) { +function spawnPythonProcess(jobId: string, file_path: string) { const venvPath = path.join(__dirname, '../chunker/venv'); const requirementsPath = path.join(__dirname, '../chunker/requirements.txt'); const pythonScriptPath = path.join(__dirname, '../chunker/pdf_chunker.py'); @@ -1060,7 +706,7 @@ function spawnPythonProcess(jobId: string, file_path: string, doc_id: string) { function runPythonScript() { const pythonPath = process.platform === 'win32' ? path.join(venvPath, 'Scripts', 'python') : path.join(venvPath, 'bin', 'python3'); - const pythonProcess = spawn(pythonPath, [pythonScriptPath, jobId, file_path, outputDirectory, doc_id]); + const pythonProcess = spawn(pythonPath, [pythonScriptPath, jobId, file_path, outputDirectory]); let pythonOutput = ''; let stderrOutput = ''; @@ -1135,6 +781,7 @@ function spawnPythonProcess(jobId: string, file_path: string, doc_id: string) { console.log('Virtual environment not found. Creating and setting up...'); // Create venv + // const createVenvProcess = spawn('python', ['-m', 'venv', venvPath]); const createVenvProcess = spawn('python3.10', ['-m', 'venv', venvPath]); createVenvProcess.on('close', code => { @@ -1183,121 +830,3 @@ function spawnPythonProcess(jobId: string, file_path: string, doc_id: string) { runPythonScript(); } } - -/** - * Enhanced content extraction that focuses on meaningful text content. - * @param html The HTML content to process - * @returns Extracted and cleaned text content - */ -async function extractEnhancedContent(html: string): Promise<string> { - try { - // Create DOM to extract content - const dom = new JSDOM(html, { runScripts: 'outside-only' }); - const document = dom.window.document; - - // Remove all non-content elements - const elementsToRemove = [ - 'script', - 'style', - 'iframe', - 'noscript', - 'svg', - 'canvas', - 'header', - 'footer', - 'nav', - 'aside', - 'form', - 'button', - 'input', - 'select', - 'textarea', - 'meta', - 'link', - 'img', - 'video', - 'audio', - '.ad', - '.ads', - '.advertisement', - '.banner', - '.cookie', - '.popup', - '.modal', - '.newsletter', - '[role="banner"]', - '[role="navigation"]', - '[role="complementary"]', - ]; - - elementsToRemove.forEach(selector => { - const elements = document.querySelectorAll(selector); - elements.forEach(el => el.remove()); - }); - - // Get all text paragraphs with meaningful content - const contentElements = [ - ...Array.from(document.querySelectorAll('p')), - ...Array.from(document.querySelectorAll('h1')), - ...Array.from(document.querySelectorAll('h2')), - ...Array.from(document.querySelectorAll('h3')), - ...Array.from(document.querySelectorAll('h4')), - ...Array.from(document.querySelectorAll('h5')), - ...Array.from(document.querySelectorAll('h6')), - ...Array.from(document.querySelectorAll('li')), - ...Array.from(document.querySelectorAll('td')), - ...Array.from(document.querySelectorAll('article')), - ...Array.from(document.querySelectorAll('section')), - ...Array.from(document.querySelectorAll('div:not([class]):not([id])')), - ]; - - // Extract text from content elements that have meaningful text - let contentParts: string[] = []; - contentElements.forEach(el => { - const text = el.textContent?.trim(); - // Only include elements with substantial text (more than just a few characters) - if (text && text.length > 10 && !contentParts.includes(text)) { - contentParts.push(text); - } - }); - - // If no significant content found with selective approach, fallback to body - if (contentParts.length < 3) { - return document.body.textContent || ''; - } - - return contentParts.join('\n\n'); - } catch (error) { - console.error('Error extracting enhanced content:', error); - return 'Failed to extract content from the webpage.'; - } -} - -/** - * Cleans up extracted text to improve readability and focus on useful content. - * @param text The raw extracted text - * @returns Cleaned and formatted text - */ -function cleanupText(text: string): string { - if (!text) return ''; - - return ( - text - // Remove excessive whitespace and normalize line breaks - .replace(/\s+/g, ' ') - .replace(/\n\s*\n\s*\n+/g, '\n\n') - // Remove common boilerplate phrases - .replace(/cookie policy|privacy policy|terms of service|all rights reserved|copyright ©/gi, '') - // Remove email addresses - .replace(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, '') - // Remove URLs - .replace(/https?:\/\/[^\s]+/g, '') - // Remove social media handles - .replace(/@[a-zA-Z0-9_]+/g, '') - // Clean up any remaining HTML tags that might have been missed - .replace(/<[^>]*>/g, '') - // Fix spacing issues after cleanup - .replace(/ +/g, ' ') - .trim() - ); -} diff --git a/src/server/ApiManagers/AzureManager.ts b/src/server/ApiManagers/AzureManager.ts index 2d0ab3aa6..d54b15810 100644 --- a/src/server/ApiManagers/AzureManager.ts +++ b/src/server/ApiManagers/AzureManager.ts @@ -1,37 +1,38 @@ -import { ContainerClient, BlobServiceClient } from "@azure/storage-blob"; -import * as fs from "fs"; -import { Readable, Stream } from "stream"; -import * as path from "path"; +import { ContainerClient, BlobServiceClient } from '@azure/storage-blob'; +import * as fs from 'fs'; +import { Readable } from 'stream'; +import * as path from 'path'; const AZURE_STORAGE_CONNECTION_STRING = process.env.AZURE_STORAGE_CONNECTION_STRING; const extToType: { [suffix: string]: string } = { - ".jpeg" : "image/jpeg", - ".jpg" : "image/jpeg", - ".png" : "image/png", - ".svg" : "image/svg+xml", - ".webp" : "image/webp", - ".gif" : "image/gif" -} + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.png': 'image/png', + '.svg': 'image/svg+xml', + '.webp': 'image/webp', + '.gif': 'image/gif', +}; export class AzureManager { private _containerClient: ContainerClient; private _blobServiceClient: BlobServiceClient; private static _instance: AzureManager | undefined; - public static CONTAINER_NAME = "dashmedia"; - public static STORAGE_ACCOUNT_NAME = "dashblobstore"; + public static CONTAINER_NAME = 'dashmedia'; + public static STORAGE_ACCOUNT_NAME = 'dashblobstore'; public static BASE_STRING = `https://${AzureManager.STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AzureManager.CONTAINER_NAME}`; constructor() { if (!AZURE_STORAGE_CONNECTION_STRING) { - throw new Error("Azure Storage Connection String Not Found"); + throw new Error('Azure Storage Connection String Not Found'); } this._blobServiceClient = BlobServiceClient.fromConnectionString(AZURE_STORAGE_CONNECTION_STRING); this._containerClient = this.BlobServiceClient.getContainerClient(AzureManager.CONTAINER_NAME); } public static get Instance() { - return this._instance = this._instance ?? new AzureManager(); + this._instance = this._instance ?? new AzureManager(); + return this._instance; } public get BlobServiceClient() { @@ -44,14 +45,14 @@ export class AzureManager { public static UploadBlob(filename: string, filepath: string, filetype: string) { const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); - const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }}; + const blobOptions = { blobHTTPHeaders: { blobContentType: filetype } }; const stream = fs.createReadStream(filepath); return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions); } public static UploadBase64ImageBlob(filename: string, data: string, filetype?: string) { const confirmedFiletype = filetype ? filetype : extToType[path.extname(filename)]; - const buffer = Buffer.from(data, "base64"); + const buffer = Buffer.from(data, 'base64'); const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); const blobOptions = { blobHTTPHeaders: { blobContentType: confirmedFiletype } }; return blockBlobClient.upload(buffer, buffer.length, blobOptions); @@ -59,7 +60,7 @@ export class AzureManager { public static UploadBlobStream(stream: Readable, filename: string, filetype: string) { const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); - const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }}; + const blobOptions = { blobHTTPHeaders: { blobContentType: filetype } }; return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions); } @@ -74,9 +75,9 @@ export class AzureManager { console.log(`${blob.name}`); const blobItem = { - url : `https://${AzureManager.STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AzureManager.CONTAINER_NAME}/${blob.name}`, - name : blob.name - } + url: `https://${AzureManager.STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AzureManager.CONTAINER_NAME}/${blob.name}`, + name: blob.name, + }; foundBlobs.push(blobItem); } diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts deleted file mode 100644 index 5ee21fb44..000000000 --- a/src/server/ApiManagers/DownloadManager.ts +++ /dev/null @@ -1,262 +0,0 @@ -import * as Archiver from 'archiver'; -import * as express from 'express'; -import * as path from 'path'; -import { URL } from 'url'; -import { DashUploadUtils, SizeSuffix } from '../DashUploadUtils'; -import { Method } from '../RouteManager'; -import RouteSubscriber from '../RouteSubscriber'; -import { Directory, publicDirectory, serverPathToFile } from '../SocketData'; -import { Database } from '../database'; -import ApiManager, { Registration } from './ApiManager'; - -export type Hierarchy = { [id: string]: string | Hierarchy }; -export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>; -export interface DocumentElements { - data: string | any[]; - title: string; -} - -/** - * This is a very specific utility method to help traverse the database - * to parse data and titles out of images and collections alone. - * - * We don't know if the document id given to is corresponds to a view document or a data - * document. If it's a data document, the response from the database will have - * a data field. If not, call recursively on the proto, and resolve with *its* data - * - * @param targetId the id of the Dash document whose data is being requests - * @returns the data of the document, as well as its title - */ -async function getData(targetId: string): Promise<DocumentElements> { - return new Promise<DocumentElements>((resolve, reject) => { - Database.Instance.getDocument(targetId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); - } - } else if (proto) { - getData(proto.fieldId).then(resolve, reject); - } else { - reject(); - } - }); - }); -} - -/** - * This function starts with a single document id as a seed, - * typically that of a collection, and then descends the entire tree - * of image or collection documents that are reachable from that seed. - * @param seedId the id of the root of the subtree we're trying to capture, interesting only if it's a collection - * @param hierarchy the data structure we're going to use to record the nesting of the collections and images as we descend - -Below is an example of the JSON hierarchy built from two images contained inside a collection titled 'a nested collection', -following the general recursive structure shown immediately below -{ - "parent folder name":{ - "first child's fild name":"first child's url" - ... - "nth child's fild name":"nth child's url" - } -} -{ - "a nested collection (865c4734-c036-4d67-a588-c71bb43d1440)":{ - "an image of a cat (ace99ffd-8ed8-4026-a5d5-a353fff57bdd).jpg":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", - "1*SGJw31T5Q9Zfsk24l2yirg.gif (9321cc9b-9b3e-4cb6-b99c-b7e667340f05).gif":"https://cdn-media-1.freecodecamp.org/images/1*SGJw31T5Q9Zfsk24l2yirg.gif" - } -} -*/ -async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Promise<void> { - const { title, data } = await getData(seedId); - const label = `${title} (${seedId})`; - // is the document a collection? - if (Array.isArray(data)) { - // recurse over all documents in the collection. - const local: Hierarchy = {}; // create a child hierarchy for this level, which will get passed in as the parent of the recursive call - hierarchy[label] = local; // store it at the index in the parent, so we'll end up with a map of maps of maps - await Promise.all(data.map(proxy => buildHierarchyRecursive(proxy.fieldId, local))); - } else { - // now, data can only be a string, namely the url of the image - const filename = label + path.extname(data); // this is the file name under which the output image will be stored - hierarchy[filename] = data; - } -} - -/** - * This utility function factors out the process - * of creating a zip file and sending it back to the client - * by piping it into a response. - * - * Learn more about piping and readable / writable streams here! - * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/ - * - * @param res the writable stream response object that will transfer the generated zip file - * @param mutator the callback function used to actually modify and insert information into the zip instance - */ -export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise<void> { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', 'application/zip'); - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); -} - -/** - * - * @param file the zip file to which we write the files - * @param hierarchy the data structure from which we read, defining the nesting of the documents in the zip - * @param prefix lets us create nested folders in the zip file by continually appending to the end - * of the prefix with each layer of recursion. - * - * Function Call #1 => "Dash Export" - * Function Call #2 => "Dash Export/a nested collection" - * Function Call #3 => "Dash Export/a nested collection/lowest level collection" - * ... - */ -async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = 'Dash Export'): Promise<void> { - // eslint-disable-next-line no-restricted-syntax - for (const documentTitle in hierarchy) { - if (Object.prototype.hasOwnProperty.call(hierarchy, documentTitle)) { - const result = hierarchy[documentTitle]; - // base case or leaf node, we've hit a url (image) - if (typeof result === 'string') { - let fPath: string; - const matches = /:\d+\/files\/images\/(upload_[\da-z]{32}.*)/g.exec(result); - if (matches !== null) { - // image already exists on our server - fPath = serverPathToFile(Directory.images, matches[1]); - } else { - // the image doesn't already exist on our server (may have been dragged - // and dropped in the browser and thus hosted remotely) so we upload it - // to our server and point the zip file to it, so it can bundle up the bytes - // eslint-disable-next-line no-await-in-loop - const information = await DashUploadUtils.UploadImage(result); - fPath = information instanceof Error ? '' : information.accessPaths[SizeSuffix.Original].server; - } - // write the file specified by the path to the directory in the - // zip file given by the prefix. - if (fPath) { - file.file(fPath, { name: documentTitle, prefix }); - } - } else { - // we've hit a collection, so we have to recurse - // eslint-disable-next-line no-await-in-loop - await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); - } - } - } -} - -async function getDocs(docId: string) { - const files = new Set<string>(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const { id } = doc; - if (typeof id === 'string' && id.endsWith('Proto')) { - // Skip protos - return []; - } - const ids: string[] = []; - // eslint-disable-next-line no-restricted-syntax - for (const key in doc.fields) { - // eslint-disable-next-line no-continue - if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue; - - const field = doc.fields[key]; - // eslint-disable-next-line no-continue - if (field === undefined || field === null) continue; - - if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') { - ids.push(field.fieldId); - } else if (field.__type === 'script' || field.__type === 'computed') { - field.captures && ids.push(field.captures.fieldId); - } else if (field.__type === 'list') { - ids.push(...fn(field)); - } else if (typeof field === 'string') { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w-]*)"/g; - for (let match = re.exec(field); match !== null; match = re.exec(field)) { - ids.push(match[1]); - } - } else if (field.__type === 'RichTextField') { - const re = /"href"\s*:\s*"(.*?)"/g; - for (let match = re.exec(field.data); match !== null; match = re.exec(field.Data)) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split('doc/'); - if (split.length > 1) { - ids.push(split[split.length - 1]); - } - } - const re2 = /"src"\s*:\s*"(.*?)"/g; - for (let match = re2.exec(field.Data); match !== null; match = re2.exec(field.Data)) { - const urlString = match[1]; - const { pathname } = new URL(urlString); - files.add(pathname); - } - } else if (['audio', 'image', 'video', 'pdf', 'web', 'map'].includes(field.__type)) { - const { pathname } = new URL(field.url); - files.add(pathname); - } - } - - if (doc.id) { - docs[doc.id] = doc; - } - return ids; - }; - await Database.Instance.visit([docId], fn); - return { id: docId, docs, files }; -} - -export default class DownloadManager extends ApiManager { - protected initialize(register: Registration): void { - /** - * Let's say someone's using Dash to organize images in collections. - * This lets them export the hierarchy they've built to their - * own file system in a useful format. - * - * This handler starts with a single document id (interesting only - * if it's that of a collection). It traverses the database, captures - * the nesting of only nested images or collections, writes - * that to a zip file and returns it to the client for download. - */ - register({ - method: Method.GET, - subscription: new RouteSubscriber('imageHierarchyExport').add('docId'), - secureHandler: async ({ req, res }) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await buildHierarchyRecursive(id, hierarchy); - return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); - }, - }); - - register({ - method: Method.GET, - subscription: new RouteSubscriber('downloadId').add('docId'), - secureHandler: async ({ req, res }) => - BuildAndDispatchZip(res, async zip => { - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - zip.append(docString, { name: 'doc.json' }); - files.forEach(val => { - zip.file(publicDirectory + val, { name: val.substring(1) }); - }); - }), - }); - - register({ - method: Method.GET, - subscription: new RouteSubscriber('serializeDoc').add('docId'), - secureHandler: async ({ req, res }) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); - }, - }); - } -} diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 1e68a4e30..5e527281f 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -131,6 +131,9 @@ export default class UploadManager extends ApiManager { }, }); + type fieldstype = string | { __type: string; Data: string } | { __type: string; id: string; fieldId: string; fields: fieldstype[]; captures: { fieldId: string } }; + type doctype = { id: string; fields: fieldstype[] }; + register({ method: Method.POST, subscription: '/uploadDoc', @@ -145,7 +148,7 @@ export default class UploadManager extends ApiManager { ids[id] = uuid.v4(); return ids[id]; }; - const mapFn = (docIn: { id: string; fields: any[] }) => { + const mapFn = (docIn: doctype) => { const doc = docIn; if (doc.id) { doc.id = getId(doc.id); @@ -156,22 +159,20 @@ export default class UploadManager extends ApiManager { const field = doc.fields[key]; if (field === undefined || field === null) continue; - if (field.__type === 'Doc') { - mapFn(field); + if (typeof field === 'string') { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w-]*)"/g; + doc.fields[key] = field.replace(re, (match: string, p1: string, p2: string) => `${p1}${getId(p2)}"`); + } else if ('Data' in field) { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: string, p1: string, p2: string) => `${p1}${getId(p2)}"`); } else if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') { field.fieldId = getId(field.fieldId); } else if (field.__type === 'script' || field.__type === 'computed') { if (field.captures) { field.captures.fieldId = getId(field.captures.fieldId); } - } else if (field.__type === 'list') { + } else if (field.__type === 'list' || field.__type === 'Doc') { mapFn(field); - } else if (typeof field === 'string') { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w-]*)"/g; - doc.fields[key] = field.replace(re, (match: string, p1: string, p2: string) => `${p1}${getId(p2)}"`); - } else if (field.__type === 'RichTextField') { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: string, p1: string, p2: string) => `${p1}${getId(p2)}"`); } } }; diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index b587340e2..b42c974ac 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -22,6 +22,7 @@ export default class UserManager extends ApiManager { secureHandler: async ({ res }) => { const cursor = await Database.Instance.query({}, { email: 1, linkDatabaseId: 1, sharingDocumentId: 1 }, 'users'); const results = await cursor.toArray(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any res.send(results.map((user: any) => ({ email: user.email, linkDatabaseId: user.linkDatabaseId, sharingDocumentId: user.sharingDocumentId }))); }, }); @@ -31,9 +32,10 @@ export default class UserManager extends ApiManager { subscription: '/setCacheDocumentIds', secureHandler: async ({ user, req, res }) => { const userModel = user; - const result: any = {}; + const result: { error?: unknown } = {}; userModel.cacheDocumentIds = req.body.cacheDocumentIds; - userModel.save().then(undefined, (err: any) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + userModel.save?.().then(undefined, (err: any) => { if (err) { result.error = [{ msg: 'Error while caching documents' }]; } @@ -90,20 +92,19 @@ export default class UserManager extends ApiManager { subscription: '/internalResetPassword', secureHandler: async ({ user, req, res }) => { const userModel = user; - const result: any = {}; - // eslint-disable-next-line camelcase + const result: { error?: unknown } = {}; const { curr_pass, new_pass } = req.body; // perhaps should assert whether curr password is entered correctly const validated = await new Promise<Opt<boolean>>(resolve => { - bcrypt.compare(curr_pass, userModel.password, (err, passwordsMatch) => { - if (err || !passwordsMatch) { - result.error = [{ msg: 'Incorrect current password' }]; - res.send(result); - resolve(undefined); - } else { - resolve(passwordsMatch); - } - }); + userModel.password && + bcrypt.compare(curr_pass, userModel.password, (err, passwordsMatch) => { + if (err || !passwordsMatch) { + res.send({ error: [{ msg: 'Incorrect current password' }] }); + resolve(undefined); + } else { + resolve(passwordsMatch); + } + }); }); if (validated === undefined) { @@ -133,7 +134,7 @@ export default class UserManager extends ApiManager { userModel.passwordResetExpires = undefined; } - userModel.save().then(undefined, err => { + userModel.save?.().then(undefined, err => { if (err) { result.error = [{ msg: 'Error while saving new password' }]; } diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index c5d70da3d..ed51aea40 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -18,10 +18,11 @@ export interface CoreArguments { isRelease: boolean; } -export type AuthorizedCore = CoreArguments & { user: DashUserModel }; -export type SecureHandler = (core: AuthorizedCore) => any | Promise<any>; -export type PublicHandler = (core: CoreArguments) => any | Promise<any>; -export type ErrorHandler = (core: CoreArguments & { error: any }) => any | Promise<any>; +export type AuthorizedCore = CoreArguments & { user: Partial<DashUserModel> }; +export type SecureHandler = (core: AuthorizedCore) => unknown | Promise<unknown>; +export type PublicHandler = (core: CoreArguments) => unknown | Promise<unknown>; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type ErrorHandler = (core: CoreArguments & { error: any }) => unknown | Promise<unknown>; export const STATUS = { OK: 200, @@ -30,13 +31,14 @@ export const STATUS = { PERMISSION_DENIED: 403, }; +// eslint-disable-next-line @typescript-eslint/no-explicit-any export function _error(res: Response, message: string, error?: any) { console.error(message, error); res.statusMessage = message; res.status(STATUS.EXECUTION_ERROR).send(error); } -export function _success(res: Response, body: any) { +export function _success(res: Response, body: unknown) { res.status(STATUS.OK).send(body); } @@ -135,18 +137,6 @@ export default class RouteManager { user = { id: 'guest', email: 'guest', userDocumentId: Utils.GuestID() }; } const core = { req, res, isRelease }; - const tryExecute = async (toExecute: (args: any) => any | Promise<any>, args: any) => { - try { - await toExecute(args); - } catch (e) { - console.log(red(target), user && 'email' in user ? '<user logged out>' : undefined); - if (errorHandler) { - errorHandler({ ...core, error: e }); - } else { - _error(res, `The server encountered an internal error when serving ${target}.`, e); - } - } - }; if (user) { if (requireAdmin && isRelease && process.env.PASSWORD) { if (AdminPrivileges.get(user.id)) { @@ -156,11 +146,29 @@ export default class RouteManager { return; } } - await tryExecute(secureHandler, { ...core, user }); + try { + await secureHandler({ ...core, user }); + } catch (e) { + console.log(red(target), user && 'email' in user ? '<user logged out>' : undefined); + if (errorHandler) { + errorHandler({ ...core, error: e }); + } else { + _error(res, `The server encountered an internal error when serving ${target}.`, e); + } + } } // req.session!.target = target; else if (publicHandler) { - await tryExecute(publicHandler, core); + try { + await publicHandler(core); + } catch (e) { + console.log(red(target), user && 'email' in user ? '<user logged out>' : undefined); + if (errorHandler) { + errorHandler({ ...core, error: e }); + } else { + _error(res, `The server encountered an internal error when serving ${target}.`, e); + } + } if (!res.headersSent) { // res.redirect("/login"); } diff --git a/src/server/authentication/AuthenticationManager.ts b/src/server/authentication/AuthenticationManager.ts index 0cc1553c0..3c7858a72 100644 --- a/src/server/authentication/AuthenticationManager.ts +++ b/src/server/authentication/AuthenticationManager.ts @@ -26,21 +26,12 @@ export const getSignup = (req: Request, res: Response) => { return undefined; }; -const tryRedirectToTarget = (req: Request, res: Response) => { - const target = (req.session as any)?.target; - if (req.session && target) { - res.redirect(target); - } else { - res.redirect('/home'); - } -}; - /** * POST /signup * Create a new local account. */ export const postSignup = (req: Request, res: Response, next: NextFunction) => { - const email = req.body.email as String; + const email = req.body.email as string; check('email', 'Email is not valid').isEmail().run(req); check('password', 'Password must be at least 4 characters long').isLength({ min: 4 }).run(req); check('confirmPassword', 'Passwords do not match').equals(req.body.password).run(req); @@ -66,7 +57,7 @@ export const postSignup = (req: Request, res: Response, next: NextFunction) => { const user = new User(model); User.findOne({ email }) - .then((existingUser: any) => { + .then((existingUser: DashUserModel | null) => { if (existingUser) { return res.redirect('/login'); } @@ -74,13 +65,15 @@ export const postSignup = (req: Request, res: Response, next: NextFunction) => { .then(() => { req.logIn(user, err => { if (err) return next(err); - tryRedirectToTarget(req, res); + res.redirect('/home'); return undefined; }); }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any .catch((err: any) => next(err)); return undefined; }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any .catch((err: any) => next(err)); return undefined; }; @@ -108,7 +101,8 @@ export const getLogin = (req: Request, res: Response) => { export const postLogin = (req: Request, res: Response, next: NextFunction) => { if (req.body.email === '') { User.findOne({ email: 'guest' }) - .then((user: any) => !user && initializeGuest()) + .then((user: DashUserModel | null) => !user && initializeGuest()) + // eslint-disable-next-line @typescript-eslint/no-explicit-any .catch((err: any) => err); req.body.email = 'guest'; req.body.password = 'guest'; @@ -132,7 +126,7 @@ export const postLogin = (req: Request, res: Response, next: NextFunction) => { req.logIn(user, loginErr => { if (loginErr) { next(loginErr); - } else tryRedirectToTarget(req, res); + } else res.redirect('/home'); }); return undefined; }; @@ -163,15 +157,15 @@ export const postForgot = function (req: Request, res: Response, next: NextFunct const { email } = req.body; async.waterfall( [ - function (done: any) { - c.randomBytes(20, (err: any, buffer: Buffer) => { + function (done: (arg: null, token?: string) => void) { + c.randomBytes(20, (err: Error | null, buffer: Buffer) => { if (err) { done(null); } else done(null, buffer.toString('hex')); }); }, - function (token: string, done: any) { - User.findOne({ email }).then((user: any) => { + function (token: string, done: (arg: null, token: string, user: DashUserModel) => void) { + User.findOne({ email }).then((user: DashUserModel | null) => { if (!user) { // NO ACCOUNT WITH SUBMITTED EMAIL res.redirect('/forgotPassword'); @@ -182,7 +176,7 @@ export const postForgot = function (req: Request, res: Response, next: NextFunct user.save().then(() => done(null, token, user)); }); }, - function (token: Uint16Array, user: DashUserModel, done: any) { + function (token: Uint16Array, user: DashUserModel, done: (arg: null, token: Error | null, data: string) => void) { const smtpTransport = nodemailer.createTransport({ service: 'Gmail', auth: { @@ -220,7 +214,7 @@ export const postForgot = function (req: Request, res: Response, next: NextFunct export const getReset = function (req: Request, res: Response) { User.findOne({ passwordResetToken: req.params.token, passwordResetExpires: { $gt: Date.now() } }) - .then((user: any) => { + .then((user: DashUserModel | null) => { if (!user) return res.redirect('/forgotPassword'); res.render('reset.pug', { title: 'Reset Password', @@ -234,9 +228,9 @@ export const getReset = function (req: Request, res: Response) { export const postReset = function (req: Request, res: Response) { async.waterfall( [ - function (done: any) { + function (done: (args: null, user: DashUserModel) => void) { User.findOne({ passwordResetToken: req.params.token, passwordResetExpires: { $gt: Date.now() } }) - .then((user: any) => { + .then((user: DashUserModel | null) => { if (!user) return res.redirect('back'); check('password', 'Password must be at least 4 characters long').isLength({ min: 4 }).run(req); @@ -250,8 +244,8 @@ export const postReset = function (req: Request, res: Response) { user.save() .then( - () => (req as any).logIn(user), - (err: any) => err + // eslint-disable-next-line @typescript-eslint/no-explicit-any + () => req.logIn(user, (err: any) => err) ) .catch(() => res.redirect('/login')); done(null, user); @@ -259,7 +253,7 @@ export const postReset = function (req: Request, res: Response) { }) .catch(() => res.redirect('back')); }, - function (user: DashUserModel, done: any) { + function (user: DashUserModel, done: (args: null, error: Error | null) => void) { const smtpTransport = nodemailer.createTransport({ service: 'Gmail', auth: { diff --git a/src/server/authentication/DashUserModel.ts b/src/server/authentication/DashUserModel.ts index 6fd8dd593..7aa7f2598 100644 --- a/src/server/authentication/DashUserModel.ts +++ b/src/server/authentication/DashUserModel.ts @@ -3,30 +3,31 @@ import * as mongoose from 'mongoose'; import { Utils } from '../../Utils'; type comparePasswordFunction = (candidatePassword: string, cb: (err: Error, isMatch: boolean) => void) => void; -type mongooseDocument = { id: string }; // & mongoose.Document; -export type DashUserModel = mongooseDocument & { - email: string; - password: string; - passwordResetToken?: string; - passwordResetExpires?: Date; +export type DashUserModel = mongoose.Document & { + email?: string | null | undefined; + password?: string | null | undefined; + passwordResetToken?: string | null | undefined; + passwordResetExpires?: Date | null | undefined; - dropboxRefresh?: string; - dropboxToken?: string; + dropboxRefresh?: string | null | undefined; + dropboxToken?: string | null | undefined; - userDocumentId: string; - sharingDocumentId: string; - linkDatabaseId: string; - cacheDocumentIds: string; + userDocumentId?: string | null | undefined; + sharingDocumentId?: string | null | undefined; + linkDatabaseId?: string | null | undefined; + cacheDocumentIds?: string | null | undefined; + profile?: + | { + name?: string | null | undefined; + gender?: string | null | undefined; + location?: string | null | undefined; + website?: string | null | undefined; + picture?: string | null | undefined; + } + | null + | undefined; - profile: { - name: string; - gender: string; - location: string; - website: string; - picture: string; - }; - - comparePassword: comparePasswordFunction; + comparePassword?: comparePasswordFunction | null | undefined; }; export type AuthToken = { @@ -68,36 +69,38 @@ const userSchema = new mongoose.Schema( */ // eslint-disable-next-line @typescript-eslint/no-explicit-any userSchema.pre('save', function save(next: any) { - const user = this; - if (!user.isModified('password')) { + if (!this.isModified('password')) { return next(); } - bcrypt.genSalt(10, (err: Error, salt: string) => { - if (err) { - return next(err); - } - bcrypt.hash( - user.password ?? '', - salt, - () => {}, - (cryptErr: mongoose.Error, hash: string) => { - if (cryptErr) { - return next(cryptErr); - } - user.password = hash; - next(); - return undefined; + bcrypt.genSalt( + 10, + ((err: Error, salt: string) => { + if (err) { + return next(err); } - ); - return undefined; - }); + bcrypt.hash( + this.password ?? '', + salt, + () => {}, + (cryptErr: mongoose.Error, hash: string) => { + if (cryptErr) { + return next(cryptErr); + } + this.password = hash; + next(); + return undefined; + } + ); + return undefined; + }).bind(this) + ); return undefined; }); const comparePassword: comparePasswordFunction = function (this: DashUserModel, candidatePassword, cb) { // Choose one of the following bodies for authentication logic. // secure (expected, default) - bcrypt.compare(candidatePassword, this.password, cb); + this.password && bcrypt.compare(candidatePassword, this.password, cb); // bypass password (debugging) // cb(undefined, true); }; diff --git a/src/server/authentication/Passport.ts b/src/server/authentication/Passport.ts index a62d38e3e..38a99bd45 100644 --- a/src/server/authentication/Passport.ts +++ b/src/server/authentication/Passport.ts @@ -1,25 +1,28 @@ import * as passport from 'passport'; import * as passportLocal from 'passport-local'; import User, { DashUserModel } from './DashUserModel'; +import { IncomingMessage } from 'webpack-dev-middleware'; const LocalStrategy = passportLocal.Strategy; -passport.serializeUser<any, any>((req, user, done) => { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +passport.serializeUser<any, IncomingMessage & DashUserModel>((req, user, done) => { done(undefined, (user as DashUserModel)?.id); }); -passport.deserializeUser<any, any>((id, done) => { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +passport.deserializeUser<any, IncomingMessage & DashUserModel>((id, done) => { User.findById(id) .exec() - .then((user: DashUserModel) => done(undefined, user)); + .then((user: DashUserModel | null) => user && done(undefined, user)); }); // AUTHENTICATE JUST WITH EMAIL AND PASSWORD passport.use( new LocalStrategy({ usernameField: 'email', passReqToCallback: true }, (req, email, password, done) => { User.findOne({ email: email.toLowerCase() }) - .then((user: DashUserModel) => { - if (!user) { + .then((user: DashUserModel | null) => { + if (!user?.comparePassword) { done(undefined, false, { message: 'Invalid email or password' }); // invalid email } else { user.comparePassword(password, (error: Error, isMatch: boolean) => { diff --git a/src/server/chunker/requirements.txt b/src/server/chunker/requirements.txt index eceb56f97..586bbe505 100644 --- a/src/server/chunker/requirements.txt +++ b/src/server/chunker/requirements.txt @@ -7,14 +7,14 @@ # ─── LLM clients ───────────────────────────────────────────────────────────── openai==1.40.6 -httpx==0.27.2 # <0.28 → avoids “proxies=” crash +httpx==0.27.2 # <0.28 → avoids "proxies=" crash anthropic==0.34.0 cohere==5.8.0 # ─── Torch stack (CPU) ─────────────────────────────────────────────────────── -torch==2.5.1 -torchvision==0.20.1 # matches torch 2.5.x -torchaudio==2.5.1 +torch<=2.7.1 +torchvision<=0.22.1 # matches torch 2.5.x +torchaudio<=2.7.1 # ─── Vision / OCR / PDF processing ─────────────────────────────────────────── ultralyticsplus==0.0.28 @@ -33,4 +33,4 @@ scikit-learn==1.5.1 # ─── Utilities ────────────────────────────────────────────────────────────── tqdm==4.66.5 python-dotenv==1.0.1 -packaging==24.0 +packaging==24.0
\ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 887974ed8..ca3398ad7 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -8,7 +8,6 @@ import AssistantManager from './ApiManagers/AssistantManager'; import FlashcardManager from './ApiManagers/FlashcardManager'; import DataVizManager from './ApiManagers/DataVizManager'; import DeleteManager from './ApiManagers/DeleteManager'; -import DownloadManager from './ApiManagers/DownloadManager'; import FireflyManager from './ApiManagers/FireflyManager'; import GeneralGoogleManager from './ApiManagers/GeneralGoogleManager'; import SessionManager from './ApiManagers/SessionManager'; @@ -68,7 +67,6 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage new SessionManager(), new UserManager(), new UploadManager(), - new DownloadManager(), new DeleteManager(), new UtilManager(), new GeneralGoogleManager(), |