if same youtube video is uploaded, reuse the existing file without downloading again. removed failed uploads from parsed_files. change pinWithView to highlight the pinned view region. Give pinWithView collections an option to just pan or zoom to % of screen size.

10 files changed, 178 insertions, 84 deletions

diff --git a/src/client/views/DocComponent.tsx b/src/client/views/DocComponent.tsx
index 886dd974b..465bb40f0 100644
--- a/src/client/views/DocComponent.tsx
+++ b/src/client/views/DocComponent.tsx
@@ -3,7 +3,6 @@ import { DateField } from '../../fields/DateField';
 import { AclAdmin, AclAugment, AclEdit, AclPrivate, AclReadonly, AclSym, DataSym, Doc, DocListCast, Opt } from '../../fields/Doc';
 import { InkTool } from '../../fields/InkField';
 import { List } from '../../fields/List';
-import { ScriptField } from '../../fields/ScriptField';
 import { Cast, ScriptCast } from '../../fields/Types';
 import { denormalizeEmail, distributeAcls, GetEffectiveAcl, inheritParentAcls, SharingPermissions } from '../../fields/util';
 import { returnFalse } from '../../Utils';
diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx
index 0a06b6e00..ee3f46818 100644
--- a/src/client/views/collections/CollectionView.tsx
+++ b/src/client/views/collections/CollectionView.tsx
@@ -42,6 +42,7 @@ interface CollectionViewProps_ extends FieldViewProps {
     isAnnotationOverlayScrollable?: boolean; // whether the annotation overlay can be vertically scrolled (just for tree views, currently)
     layoutEngine?: () => string;
     setPreviewCursor?: (func: (x: number, y: number, drag: boolean, hide: boolean) => void) => void;
+    setBrushViewer?: (func?: (view: { width: number; height: number; panX: number; panY: number }) => void) => void;
 
     // property overrides for child documents
     childDocuments?: Doc[]; // used to override the documents shown by the sub collection to an explicit list (see LinkBox)
diff --git a/src/client/views/collections/TabDocView.tsx b/src/client/views/collections/TabDocView.tsx
index 7a075a7ff..a82bd2dc8 100644
--- a/src/client/views/collections/TabDocView.tsx
+++ b/src/client/views/collections/TabDocView.tsx
@@ -231,7 +231,7 @@ export class TabDocView extends React.Component<TabDocViewProps> {
             pinDoc.presentationTargetDoc = doc;
             pinDoc.title = doc.title + ' - Slide';
             pinDoc.data = new List<Doc>(); // the children of the alias' layout are the presentation slide children.  the alias' data field might be children of a collection, PDF data, etc -- in any case we don't want the tree view to "see" this data
-            pinDoc.presMovement = pinProps?.pinDocView ? PresMovement.None : PresMovement.Zoom;
+            pinDoc.presMovement = pinProps?.pinDocView && !pinProps?.pinWithView ? PresMovement.None : PresMovement.Zoom;
             pinDoc.groupWithUp = false;
             pinDoc.context = curPres;
             // these should potentially all be props passed down by the CollectionTreeView to the TreeView elements.  That way the PresBox could configure all of its children at render time
@@ -367,7 +367,7 @@ export class TabDocView extends React.Component<TabDocViewProps> {
     @action
     focusFunc = (doc: Doc, options?: DocFocusOptions) => {
         const shrinkwrap = options?.originalTarget === this._document && this.view?.ComponentView?.shrinkWrap;
-        if (shrinkwrap && this._document) {
+        if (options?.willZoom !== false && shrinkwrap && this._document) {
             const focusSpeed = NumCast(this._document.focusSpeed, 500);
             shrinkwrap();
             this._document._viewTransition = `transform ${focusSpeed}ms`;
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 6927d429d..3cc425745 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -88,6 +88,8 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
     private _cachedPool: Map<string, PoolData> = new Map();
     private _lastTap = 0;
     private _batch: UndoManager.Batch | undefined = undefined;
+    private _brushtimer: any;
+    private _brushtimer1: any;
 
     // private isWritingMode: boolean = true;
     // private writingModeDocs: Doc[] = [];
@@ -114,6 +116,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
     @observable _marqueeRef = React.createRef<HTMLDivElement>();
     @observable _marqueeViewRef = React.createRef<MarqueeView>();
     @observable ChildDrag: DocumentView | undefined; // child document view being dragged.  needed to update drop areas of groups when a group item is dragged.
+    @observable _brushedView = { width: 0, height: 0, panX: 0, panY: 0, opacity: 0 }; // highlighted region of freeform canvas used by presentations to indicate a region
 
     @computed get views() {
         const viewsMask = this._layoutElements.filter(ele => ele.bounds && !ele.bounds.z && ele.inkMask !== -1 && ele.inkMask !== undefined).map(ele => ele.ele);
@@ -1520,6 +1523,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
     componentDidMount() {
         super.componentDidMount?.();
         this.props.setContentView?.(this);
+        this.props.setBrushViewer?.(this.brushView);
         setTimeout(
             action(() => {
                 this._firstRender = false;
@@ -1881,6 +1885,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
                         </div>
                     ) : null}
                     <CollectionFreeFormViewPannableContents
+                        brushView={this._brushedView}
                         isAnnotationOverlay={this.isAnnotationOverlay}
                         isAnnotationOverlayScrollable={this.props.isAnnotationOverlayScrollable}
                         transform={this.contentTransform}
@@ -1916,9 +1921,25 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
         }
     };
 
+    @action
+    brushView = (viewport: { width: number; height: number; panX: number; panY: number }) => {
+        this._brushedView = { ...viewport, panX: viewport.panX - viewport.width / 2, panY: viewport.panY - viewport.height / 2, opacity: 1 };
+        this._brushtimer1 && clearTimeout(this._brushtimer1);
+        this._brushtimer && clearTimeout(this._brushtimer);
+        this._brushtimer1 = setTimeout(
+            action(() => {
+                this._brushedView.opacity = 0;
+                this._brushtimer = setTimeout(
+                    action(() => (this._brushedView = { width: 0, height: 0, panX: 0, panY: 0, opacity: 0 })),
+                    500
+                );
+            }),
+            1000
+        );
+    };
+
     render() {
         TraceMobx();
-        const clientRect = this._mainCont?.getBoundingClientRect();
         return (
             <div
                 className={'collectionfreeformview-container'}
@@ -2004,6 +2025,7 @@ interface CollectionFreeFormViewPannableContentsProps {
     presPinView?: boolean;
     isAnnotationOverlay: boolean | undefined;
     isAnnotationOverlayScrollable: boolean | undefined;
+    brushView: { panX: number; panY: number; width: number; height: number; opacity: number };
 }
 
 @observer
@@ -2123,6 +2145,21 @@ class CollectionFreeFormViewPannableContents extends React.Component<CollectionF
                     //willChange: "transform"
                 }}>
                 {this.props.children()}
+                {!this.props.brushView.width ? null : (
+                    <div
+                        className="collectionFreeFormView-brushView"
+                        style={{
+                            zIndex: 1000,
+                            opacity: this.props.brushView.opacity,
+                            border: 'orange solid 2px',
+                            position: 'absolute',
+                            transform: `translate(${this.props.brushView.panX}px, ${this.props.brushView.panY}px)`,
+                            width: this.props.brushView.width,
+                            height: this.props.brushView.height,
+                            transition: 'opacity 2s',
+                        }}
+                    />
+                )}
                 {this.presPaths}
                 {this.progressivize}
                 {this.zoomProgressivize}
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
index 58a00bbac..584c9690f 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
@@ -425,13 +425,11 @@ export class MarqueeView extends React.Component<SubCollectionViewProps & Marque
     @undoBatch
     @action
     pinWithView = async () => {
-        const scale = Math.min(this.props.PanelWidth() / this.Bounds.width, this.props.PanelHeight() / this.Bounds.height);
         const doc = this.props.Document;
         const viewOptions: PinViewProps = {
             bounds: this.Bounds,
-            scale: scale,
         };
-        TabDocView.PinDoc(doc, { pinWithView: viewOptions });
+        TabDocView.PinDoc(doc, { pinWithView: viewOptions, pinDocView: true });
         MarqueeOptionsMenu.Instance.fadeOut(true);
         this.hideMarquee();
     };
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 2966c2a22..f24ceb5ae 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -86,6 +86,7 @@ export interface DocComponentView {
     updateIcon?: () => void; // updates the icon representation of the document
     getAnchor?: () => Doc; // returns an Anchor Doc that represents the current state of the doc's componentview (e.g., the current playhead location of a an audio/video box)
     scrollFocus?: (doc: Doc, smooth: boolean) => Opt<number>; // returns the duration of the focus
+    brushView?: (view: { width: number; height: number; panX: number; panY: number }) => void;
     setViewSpec?: (anchor: Doc, preview: boolean) => void; // sets viewing information for a componentview, typically when following a link. 'preview' tells the view to use the values without writing to the document
     reverseNativeScaling?: () => boolean; // DocumentView's setup screenToLocal based on the doc having a nativeWidth/Height.  However, some content views (e.g., FreeFormView w/ fitContentsToBox set) may ignore the native dimensions so this flags the DocumentView to not do Nativre scaling.
     shrinkWrap?: () => void; // requests a document to display all of its contents with no white space.  currently only implemented (needed?) for freeform views
diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx
index 345407c2f..001d9a5a6 100644
--- a/src/client/views/nodes/PDFBox.tsx
+++ b/src/client/views/nodes/PDFBox.tsx
@@ -195,6 +195,9 @@ export class PDFBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
         );
     }
 
+    brushView = (view: { width: number; height: number; panX: number; panY: number }) => {
+        this._pdfViewer?.brushView(view);
+    };
     scrollFocus = (doc: Doc, smooth: boolean) => {
         let didToggle = false;
         if (DocListCast(this.props.Document[this.fieldKey + '-sidebar']).includes(doc) && !this.SidebarShown) {
diff --git a/src/client/views/nodes/trails/PresBox.tsx b/src/client/views/nodes/trails/PresBox.tsx
index 6a929ef80..ab59e6112 100644
--- a/src/client/views/nodes/trails/PresBox.tsx
+++ b/src/client/views/nodes/trails/PresBox.tsx
@@ -44,7 +44,6 @@ export interface PinProps {
 
 export interface PinViewProps {
     bounds: MarqueeViewBounds;
-    scale: number;
 }
 
 @observer
@@ -362,17 +361,32 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() {
         bestTarget._viewTransition = presTransitionTime;
         if (clippable) bestTarget._clipWidth = activeItem.presPinClipWidth;
         if (temporal) bestTarget._currentTimecode = activeItem.presStartTime;
-        if (scrollable) bestTarget._scrollTop = activeItem.presPinViewScroll;
+        if (scrollable) {
+            bestTarget._scrollTop = activeItem.presPinViewScroll;
+            const contentBounds = Cast(activeItem.presPinViewBounds, listSpec('number'));
+            if (contentBounds) {
+                const dv = DocumentManager.Instance.getDocumentView(bestTarget)?.ComponentView;
+                dv?.brushView?.({ panX: (contentBounds[0] + contentBounds[2]) / 2, panY: (contentBounds[1] + contentBounds[3]) / 2, width: contentBounds[2] - contentBounds[0], height: contentBounds[3] - contentBounds[1] });
+            }
+        }
         if (dataview) Doc.GetProto(bestTarget).data = activeItem.presData instanceof ObjectField ? activeItem.presData[Copy]() : activeItem.presData;
         if (textview) Doc.GetProto(bestTarget).text = activeItem.presData instanceof ObjectField ? activeItem.presData[Copy]() : activeItem.presData;
         if (pannable) {
-            bestTarget._panX = activeItem.presPinViewX;
-            bestTarget._panY = activeItem.presPinViewY;
-            bestTarget._viewScale = activeItem.presPinViewScale;
             const contentBounds = Cast(activeItem.presPinViewBounds, listSpec('number'));
             if (contentBounds) {
+                const viewport = { panX: (contentBounds[0] + contentBounds[2]) / 2, panY: (contentBounds[1] + contentBounds[3]) / 2, width: contentBounds[2] - contentBounds[0], height: contentBounds[3] - contentBounds[1] };
+                bestTarget._panX = viewport.panX;
+                bestTarget._panY = viewport.panY;
                 const dv = DocumentManager.Instance.getDocumentView(bestTarget);
-                dv && (bestTarget._viewScale = Math.min(dv.props.PanelHeight() / (contentBounds[3] - contentBounds[1]), dv.props.PanelWidth() / (contentBounds[2] - contentBounds[0])));
+                if (dv) {
+                    const computedScale = NumCast(activeItem.presZoom, 1) * Math.min(dv.props.PanelWidth() / viewport.width, dv.props.PanelHeight() / viewport.height);
+                    activeItem.presMovement === 'zoom' && (bestTarget._viewScale = activeItem.presZoom !== undefined ? computedScale : Math.min(computedScale, NumCast(bestTarget._viewScale)));
+                    dv.ComponentView?.brushView?.(viewport);
+                }
+            } else {
+                bestTarget._panX = activeItem.presPinViewX;
+                bestTarget._panY = activeItem.presPinViewY;
+                activeItem.presMovement === 'zoom' && (bestTarget._viewScale = activeItem.presPinViewScale);
             }
         }
         return setTimeout(() => (bestTarget._viewTransition = undefined), transTime + 10);
@@ -389,7 +403,6 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() {
             pinDoc.presPinView = true;
             pinDoc.presPinViewX = bounds.left + bounds.width / 2;
             pinDoc.presPinViewY = bounds.top + bounds.height / 2;
-            pinDoc.presPinViewScale = pinProps.pinWithView.scale;
             pinDoc.presPinViewBounds = new List<number>([bounds.left, bounds.top, bounds.left + bounds.width, bounds.top + bounds.height]);
         }
         if (pinProps?.pinDocView) {
@@ -401,12 +414,14 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() {
             pinDoc.presWidth = NumCast(targetDoc.width);
             pinDoc.presHeight = NumCast(targetDoc.height);
 
-            if (scrollable) pinDoc.presPinViewScroll = pinDoc._scrollTop;
+            if (scrollable) {
+                pinDoc.presPinViewScroll = pinDoc._scrollTop;
+            }
             if (clippable) pinDoc.presPinClipWidth = pinDoc._clipWidth;
             if (temporal) pinDoc.presEndTime = NumCast((pinDoc.presStartTime = pinDoc._currentTimecode)) + 0.1;
             if (textview) pinDoc.presData = targetDoc.text instanceof ObjectField ? targetDoc.text[Copy]() : targetDoc.text;
             if (dataview) pinDoc.presData = targetDoc.data instanceof ObjectField ? targetDoc.data[Copy]() : targetDoc.data;
-            if (pannable) {
+            if (pannable || scrollable) {
                 const panX = NumCast(pinDoc._panX);
                 const panY = NumCast(pinDoc._panY);
                 const pw = NumCast(pinProps.panelWidth);
@@ -1131,7 +1146,7 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() {
         let scale = Number(number) / 100;
         if (change) scale += change;
         if (scale < 0.01) scale = 0.01;
-        if (scale > 1.5) scale = 1.5;
+        if (scale > 1) scale = 1;
         this.selectedArray.forEach(doc => (doc.presZoom = scale));
     };
 
@@ -1246,28 +1261,22 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() {
                     })}>
                     <div className="ribbon-box">
                         Movement
-                        {isPresCollection || (isPresCollection && isPinWithView) ? (
-                            <div className="ribbon-property" style={{ marginLeft: 0, height: 25, textAlign: 'left', paddingLeft: 5, paddingRight: 5, fontSize: 10 }}>
-                                {this.scrollable ? 'Scroll to pinned view' : !isPinWithView ? 'No movement' : 'Pan & Zoom to pinned view'}
-                            </div>
-                        ) : (
-                            <div
-                                className="presBox-dropdown"
-                                onClick={action(e => {
-                                    e.stopPropagation();
-                                    this._openMovementDropdown = !this._openMovementDropdown;
-                                })}
-                                style={{ borderBottomLeftRadius: this._openMovementDropdown ? 0 : 5, border: this._openMovementDropdown ? `solid 2px ${Colors.MEDIUM_BLUE}` : 'solid 1px black' }}>
-                                {this.movementName(activeItem)}
-                                <FontAwesomeIcon className="presBox-dropdownIcon" style={{ gridColumn: 2, color: this._openMovementDropdown ? Colors.MEDIUM_BLUE : 'black' }} icon={'angle-down'} />
-                                <div className={'presBox-dropdownOptions'} id={'presBoxMovementDropdown'} onPointerDown={StopEvent} style={{ display: this._openMovementDropdown ? 'grid' : 'none' }}>
-                                    {presMovement(PresMovement.None)}
-                                    {presMovement(PresMovement.Zoom)}
-                                    {presMovement(PresMovement.Pan)}
-                                    {presMovement(PresMovement.Jump)}
-                                </div>
+                        <div
+                            className="presBox-dropdown"
+                            onClick={action(e => {
+                                e.stopPropagation();
+                                this._openMovementDropdown = !this._openMovementDropdown;
+                            })}
+                            style={{ borderBottomLeftRadius: this._openMovementDropdown ? 0 : 5, border: this._openMovementDropdown ? `solid 2px ${Colors.MEDIUM_BLUE}` : 'solid 1px black' }}>
+                            {this.movementName(activeItem)}
+                            <FontAwesomeIcon className="presBox-dropdownIcon" style={{ gridColumn: 2, color: this._openMovementDropdown ? Colors.MEDIUM_BLUE : 'black' }} icon={'angle-down'} />
+                            <div className={'presBox-dropdownOptions'} id={'presBoxMovementDropdown'} onPointerDown={StopEvent} style={{ display: this._openMovementDropdown ? 'grid' : 'none' }}>
+                                {isPresCollection || (isPresCollection && isPinWithView) ? null : presMovement(PresMovement.None)}
+                                {presMovement(PresMovement.Zoom)}
+                                {presMovement(PresMovement.Pan)}
+                                {isPresCollection || (isPresCollection && isPinWithView) ? null : presMovement(PresMovement.Jump)}
                             </div>
-                        )}
+                        </div>
                         <div className="ribbon-doubleButton" style={{ display: activeItem.presMovement === PresMovement.Zoom ? 'inline-flex' : 'none' }}>
                             <div className="presBox-subheading">Zoom (% screen filled)</div>
                             <div className="ribbon-property">
@@ -1282,7 +1291,7 @@ export class PresBox extends ViewBoxBaseComponent<FieldViewProps>() {
                                 </div>
                             </div>
                         </div>
-                        {inputter('0', '1', '150', zoom, activeItem.presMovement === PresMovement.Zoom, this.setZoom)}
+                        {inputter('0', '1', '100', zoom, activeItem.presMovement === PresMovement.Zoom, this.setZoom)}
                         <div className="ribbon-doubleButton" style={{ display: activeItem.presMovement === PresMovement.Pan || activeItem.presMovement === PresMovement.Zoom ? 'inline-flex' : 'none' }}>
                             <div className="presBox-subheading">Movement Speed</div>
                             <div className="ribbon-property">
diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx
index 53d969c0a..5925c0392 100644
--- a/src/client/views/pdf/PDFViewer.tsx
+++ b/src/client/views/pdf/PDFViewer.tsx
@@ -64,6 +64,7 @@ export class PDFViewer extends React.Component<IViewerProps> {
     private _styleRule: any; // stylesheet rule for making hyperlinks clickable
     private _retries = 0; // number of times tried to create the PDF viewer
     private _setPreviewCursor: undefined | ((x: number, y: number, drag: boolean, hide: boolean) => void);
+    private _setBrushViewer: undefined | ((view: { width: number; height: number; panX: number; panY: number }) => void);
     private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef();
     private _disposers: { [name: string]: IReactionDisposer } = {};
     private _viewer: React.RefObject<HTMLDivElement> = React.createRef();
@@ -174,9 +175,8 @@ export class PDFViewer extends React.Component<IViewerProps> {
         }
         return focusSpeed;
     };
-    crop = (region: Doc | undefined, addCrop?: boolean) => {
-        return this.props.crop(region, addCrop);
-    };
+    crop = (region: Doc | undefined, addCrop?: boolean) => this.props.crop(region, addCrop);
+    brushView = (view: { width: number; height: number; panX: number; panY: number }) => this._setBrushViewer?.(view);
 
     @action
     setupPdfJsViewer = async () => {
@@ -446,6 +446,7 @@ export class PDFViewer extends React.Component<IViewerProps> {
     };
 
     setPreviewCursor = (func?: (x: number, y: number, drag: boolean, hide: boolean) => void) => (this._setPreviewCursor = func);
+    setBrushViewer = (func?: (view: { width: number; height: number; panX: number; panY: number }) => void) => (this._setBrushViewer = func);
 
     @action
     onZoomWheel = (e: React.WheelEvent) => {
@@ -512,6 +513,7 @@ export class PDFViewer extends React.Component<IViewerProps> {
                 fieldKey={this.props.fieldKey + '-annotations'}
                 CollectionView={undefined}
                 setPreviewCursor={this.setPreviewCursor}
+                setBrushViewer={this.setBrushViewer}
                 PanelHeight={this.panelHeight}
                 PanelWidth={this.panelWidth}
                 ScreenToLocalTransform={this.overlayTransform}
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index c38f603a4..c28ae686b 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -2,7 +2,7 @@ import { green, red } from 'colors';
 import { ExifImage } from 'exif';
 import * as exifr from 'exifr';
 import { File } from 'formidable';
-import { createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs';
+import { createReadStream, createWriteStream, exists, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs';
 import * as path from 'path';
 import { basename } from 'path';
 import * as sharp from 'sharp';
@@ -22,6 +22,7 @@ const parse = require('pdf-parse');
 const ffmpeg = require('fluent-ffmpeg');
 const fs = require('fs');
 const requestImageSize = require('../client/util/request-image-size');
+const md5File = require('md5-file');
 
 export enum SizeSuffix {
     Small = '_s',
@@ -99,34 +100,58 @@ export namespace DashUploadUtils {
         };
     }
 
+    function resolveExistingFile(name: string, path: string, directory: Directory, type?: string, duration?: number, rawText?: string) {
+        const data = { size: 0, path, name, type: type ?? '' };
+        const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration?.toString(), mime: '', toJson: () => undefined as any }) };
+        return {
+            source: file,
+            result: {
+                accessPaths: {
+                    agnostic: getAccessPaths(directory, path),
+                },
+                rawText,
+                duration,
+            },
+        };
+    }
+
     export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
             console.log('Uploading YouTube video: ' + videoId);
             const name = videoId;
             const path = name.replace(/^-/, '__') + '.mp4';
-            exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" -f "mp4[filesize<5M]/bestvideo[filesize<5M]+bestaudio/bestvideo+bestaudio"`, (error: any, stdout: any, stderr: any) => {
-                if (error) {
-                    console.log(`error: Error: ${error.message}`);
-                    res({
-                        source: {
-                            size: 0,
-                            path,
-                            name,
-                            type: '',
-                            toJSON: () => ({ name, path }),
-                        },
-                        result: { name: 'failed youtube query', message: `Could not upload YouTube video (${videoId}).  Error: ${error.message}` },
-                    });
-                } else {
-                    exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
-                        const time = Array.from(stdout.trim().split(':')).reverse();
-                        const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
-                        const data = { size: 0, path, name, type: 'video/mp4' };
-                        const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) };
-                        res(MoveParsedFile(file, Directory.videos));
-                    });
-                }
-            });
+            const finalPath = serverPathToFile(Directory.videos, path);
+            if (existsSync(finalPath)) {
+                exec(`youtube-dl -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
+                    const time = Array.from(stdout.trim().split(':')).reverse();
+                    const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
+                    res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined));
+                });
+            } else {
+                exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" -f "mp4[filesize<5M]/bestvideo[filesize<5M]+bestaudio/bestvideo+bestaudio"`, (error: any, stdout: any, stderr: any) => {
+                    if (error) {
+                        console.log(`error: Error: ${error.message}`);
+                        res({
+                            source: {
+                                size: 0,
+                                path,
+                                name,
+                                type: '',
+                                toJSON: () => ({ name, path }),
+                            },
+                            result: { name: 'failed youtube query', message: `Could not upload YouTube video (${videoId}).  Error: ${error.message}` },
+                        });
+                    } else {
+                        exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
+                            const time = Array.from(stdout.trim().split(':')).reverse();
+                            const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
+                            const data = { size: 0, path, name, type: 'video/mp4' };
+                            const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) };
+                            res(MoveParsedFile(file, Directory.videos));
+                        });
+                    }
+                });
+            }
         });
     }
 
@@ -144,6 +169,7 @@ export namespace DashUploadUtils {
                     const result = await UploadImage(path, basename(path));
                     return { source: file, result };
                 }
+                fs.unlink(path, () => {});
                 return { source: file, result: { name: 'Unsupported image format', message: `Could not upload unsupported file (${name}). Please convert to an .jpg` } };
             case 'video':
                 if (format.includes('x-matroska')) {
@@ -167,21 +193,25 @@ export namespace DashUploadUtils {
                             res();
                         })
                     );
-                    if (abort) return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } };
-                    // bcz: instead of aborting, we could convert the file using the code below to an mp4.  Problem is that this takes a long time and will clog up the server.
-                    // await new Promise(res =>
-                    //     ffmpeg(file.path)
-                    //         .videoCodec('libx264') // this will copy the data instead of reencode it
-                    //         .audioCodec('mp2')
-                    //         .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4'))
-                    //         .on('end', res)
-                    // );
-                    // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4');
-                    // format = '.mp4';
+                    if (abort) {
+                        // bcz: instead of aborting, we could convert the file using the code below to an mp4.  Problem is that this takes a long time and will clog up the server.
+                        // await new Promise(res =>
+                        //     ffmpeg(file.path)
+                        //         .videoCodec('libx264') // this will copy the data instead of reencode it
+                        //         .audioCodec('mp2')
+                        //         .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4'))
+                        //         .on('end', res)
+                        // );
+                        // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4');
+                        // format = '.mp4';
+                        fs.unlink(path, () => {});
+                        return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } };
+                    }
                 }
                 if (videoFormats.includes(format)) {
                     return MoveParsedFile(file, Directory.videos);
                 }
+                fs.unlink(path, () => {});
                 return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } };
             case 'application':
                 if (applicationFormats.includes(format)) {
@@ -195,6 +225,7 @@ export namespace DashUploadUtils {
                 if (audioFormats.includes(format)) {
                     return UploadAudio(file, format);
                 }
+                fs.unlink(path, () => {});
                 return { source: file, result: { name: 'Unsupported audio format', message: `Could not upload unsupported file (${name}). Please convert to an .mp3` } };
             case 'text':
                 if (types[1] == 'csv') {
@@ -203,20 +234,28 @@ export namespace DashUploadUtils {
         }
 
         console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`));
+        fs.unlink(path, () => {});
         return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) };
     }
 
     async function UploadPdf(file: File) {
-        const { path: sourcePath } = file;
-        const dataBuffer = readFileSync(sourcePath);
+        const fileKey = (await md5File(file.path)) + '.pdf';
+        const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
+        if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
+            return new Promise<Upload.FileResponse>(res => {
+                const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
+                const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));
+                var rawText = '';
+                readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText)));
+            });
+        }
+        const dataBuffer = readFileSync(file.path);
         const result: ParsedPDF = await parse(dataBuffer);
         await new Promise<void>((resolve, reject) => {
-            const name = path.basename(sourcePath);
-            const textFilename = `${name.substring(0, name.length - 4)}.txt`;
             const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
             writeStream.write(result.text, error => (error ? reject(error) : resolve()));
         });
-        return MoveParsedFile(file, Directory.pdfs, undefined, result.text);
+        return MoveParsedFile(file, Directory.pdfs, undefined, result.text, undefined, fileKey);
     }
 
     async function UploadCsv(file: File) {
@@ -368,9 +407,9 @@ export namespace DashUploadUtils {
      * @param suffix If the file doesn't have a suffix and you want to provide it one
      * to appear in the new location
      */
-    export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number): Promise<Upload.FileResponse> {
+    export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> {
         const { path: sourcePath } = file;
-        let name = path.basename(sourcePath);
+        let name = targetName ?? path.basename(sourcePath);
         suffix && (name += suffix);
         return new Promise(resolve => {
             const destinationPath = serverPathToFile(destination, name);
@@ -391,6 +430,11 @@ export namespace DashUploadUtils {
         });
     }
 
+    export function fExists(name: string, destination: Directory) {
+        const destinationPath = serverPathToFile(destination, name);
+        return existsSync(destinationPath);
+    }
+
     export function getAccessPaths(directory: Directory, fileName: string) {
         return {
             client: clientPathToFile(directory, fileName),