Merge branch 'master' into sophie-ai-images

6 files changed, 92 insertions, 57 deletions

diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts
index 5a8a6e4b6..eb52cff88 100644
--- a/src/fields/Doc.ts
+++ b/src/fields/Doc.ts
@@ -238,22 +238,6 @@ export class Doc extends RefField {
     public static get MyFilesystem() {
         return DocCast(Doc.UserDoc().myFilesystem);
     }
-    public static get MyFileOrphans() {
-        return DocCast(Doc.UserDoc().myFileOrphans);
-    }
-    public static AddFileOrphan(doc: Doc) {
-        if (
-            doc &&
-            Doc.MyFileOrphans instanceof Doc &&
-            Doc.IsDataProto(doc) &&
-            !Doc.IsSystem(doc) &&
-            ![DocumentType.CONFIG, DocumentType.KVP, DocumentType.LINK, DocumentType.LINKANCHOR].includes(doc.type as any) &&
-            !doc.isFolder &&
-            !doc.annotationOn
-        ) {
-            Doc.AddDocToList(Doc.MyFileOrphans, undefined, doc);
-        }
-    }
     public static get MyTools() {
         return DocCast(Doc.UserDoc().myTools);
     }
@@ -388,7 +372,7 @@ export class Doc extends RefField {
             } else {
                 return Cast(layoutField, Doc, null);
             }
-            return Cast(self[renderFieldKey + '-layout[' + templateLayoutDoc[Id] + ']'], Doc, null) || templateLayoutDoc;
+            return Cast(self[renderFieldKey + '_layout[' + templateLayoutDoc[Id] + ']'], Doc, null) || templateLayoutDoc;
         }
         return undefined;
     }
@@ -573,7 +557,7 @@ export namespace Doc {
 
     // compare whether documents or their protos match
     export function AreProtosEqual(doc?: Doc, other?: Doc) {
-        return doc && other && Doc.GetProto(doc) === Doc.GetProto(other);
+        return doc && other && (doc === other || Doc.GetProto(doc) === Doc.GetProto(other));
     }
 
     // Gets the data document for the document.  Note: this is mis-named -- it does not specifically
@@ -701,7 +685,8 @@ export namespace Doc {
     }
 
     export function BestEmbedding(doc: Doc) {
-        const bestEmbedding = Doc.GetProto(doc) ? DocListCast(doc.proto_embeddings).find(doc => !doc.embedContainer && doc.author === Doc.CurrentUserEmail) : doc;
+        const bestEmbedding = Doc.GetProto(doc) ? [doc, ...DocListCast(doc.proto_embeddings)].find(doc => !doc.embedContainer && doc.author === Doc.CurrentUserEmail) : doc;
+        bestEmbedding && Doc.AddDocToList(Doc.GetProto(doc), 'protoEmbeddings', doc);
         return bestEmbedding ?? Doc.MakeEmbedding(doc);
     }
 
@@ -782,7 +767,6 @@ export namespace Doc {
         copy.cloneOf = doc;
         cloneMap.set(doc[Id], copy);
 
-        Doc.AddFileOrphan(copy);
         return copy;
     }
     export function repairClone(clone: Doc, cloneMap: Map<string, Doc>, visited: Set<Doc>) {
@@ -917,7 +901,7 @@ export namespace Doc {
         // If it doesn't find the expanded layout, then it makes a delegate of the template layout and
         // saves it on the data doc indexed by the template layout's id.
         //
-        const expandedLayoutFieldKey = templateField + '-layout[' + templateLayoutDoc[Id] + ']';
+        const expandedLayoutFieldKey = templateField + '_layout[' + templateLayoutDoc[Id] + ']';
         let expandedTemplateLayout = targetDoc?.[expandedLayoutFieldKey];
 
         if (templateLayoutDoc.resolvedDataDoc instanceof Promise) {
@@ -985,6 +969,59 @@ export namespace Doc {
         return overwrite;
     }
 
+    export function FindReferences(infield: Doc | List<any>, references: Set<Doc>, system: boolean | undefined) {
+        if (infield instanceof List<any>) {
+            infield.forEach(val => (val instanceof Doc || val instanceof List) && FindReferences(val, references, system));
+            return;
+        }
+        const doc = infield as Doc;
+        if (references.has(doc)) {
+            references.add(doc);
+            return;
+        }
+        const excludeLists = doc.title === 'My Recently Closed' || doc.title === 'My Header Bar' || doc.title === 'My Dashboards';
+        if (system !== undefined && ((system && !Doc.IsSystem(doc)) || (!system && Doc.IsSystem(doc)))) return;
+        references.add(doc);
+        Object.keys(doc).forEach(key => {
+            if (key === 'proto') {
+                if (doc.proto instanceof Doc) {
+                    Doc.FindReferences(doc.proto, references, system);
+                }
+            } else {
+                const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
+                const field = key === 'author' ? Doc.CurrentUserEmail : ProxyField.WithoutProxy(() => doc[key]);
+                if (field instanceof RefField) {
+                    if (field instanceof Doc) {
+                        if (key === 'myLinkDatabase') {
+                            field instanceof Doc && references.add(field);
+                            // skip docs that have been closed and are scheduled for garbage collection
+                        } else {
+                            Doc.FindReferences(field, references, system);
+                        }
+                    }
+                } else if (cfield instanceof ComputedField) {
+                } else if (field instanceof ObjectField) {
+                    if (field instanceof Doc) {
+                        Doc.FindReferences(field, references, system);
+                    } else if (field instanceof List) {
+                        !excludeLists && Doc.FindReferences(field, references, system);
+                    } else if (field instanceof ProxyField) {
+                        if (key === 'myLinkDatabase') {
+                            field instanceof Doc && references.add(field);
+                            // skip docs that have been closed and are scheduled for garbage collection
+                        } else {
+                            Doc.FindReferences(field.value, references, system);
+                        }
+                    } else if (field instanceof PrefetchProxy) {
+                        Doc.FindReferences(field.value, references, system);
+                    }
+                } else if (field instanceof Promise) {
+                    debugger; //This shouldn't happend...
+                }
+            }
+        });
+    }
+
     export function MakeCopy(doc: Doc, copyProto: boolean = false, copyProtoId?: string, retitle = false): Doc {
         const copy = new Doc(copyProtoId, true);
         updateCachedAcls(copy);
@@ -1025,7 +1062,6 @@ export namespace Doc {
         if (retitle) {
             copy.title = incrementTitleCopy(StrCast(copy.title));
         }
-        Doc.AddFileOrphan(copy);
         return copy;
     }
 
@@ -1044,7 +1080,6 @@ export namespace Doc {
             if (!Doc.IsSystem(doc)) Doc.AddDocToList(doc[DocData], 'proto_embeddings', delegate);
             title && (delegate.title = title);
             delegate[Initializing] = false;
-            Doc.AddFileOrphan(delegate);
             return delegate;
         }
         return undefined;
@@ -1177,7 +1212,7 @@ export namespace Doc {
     // the document containing the view layout information - will be the Document itself unless the Document has
     // a layout field or 'layout' is given.
     export function Layout(doc: Doc, layout?: Doc): Doc {
-        const overrideLayout = layout && Cast(doc[`${StrCast(layout.isTemplateForField, 'data')}-layout[` + layout[Id] + ']'], Doc, null);
+        const overrideLayout = layout && Cast(doc[`${StrCast(layout.isTemplateForField, 'data')}_layout[` + layout[Id] + ']'], Doc, null);
         return overrideLayout || doc[DocLayout] || doc;
     }
     export function SetLayout(doc: Doc, layout: Doc | string) {
@@ -1309,7 +1344,7 @@ export namespace Doc {
     }
 
     export function LinkEndpoint(linkDoc: Doc, anchorDoc: Doc) {
-        if (linkDoc.link_anchor_2 === anchorDoc || (linkDoc.link_anchor_2 as Doc).annotationOn) return '2';
+        if (Doc.AreProtosEqual(linkDoc.link_anchor_2 as Doc, anchorDoc) || Doc.AreProtosEqual((linkDoc.link_anchor_2 as Doc).annotationOn as Doc, anchorDoc)) return '2';
         return Doc.AreProtosEqual(anchorDoc, (linkDoc.link_anchor_1 as Doc).annotationOn as Doc) || Doc.AreProtosEqual(anchorDoc, linkDoc.link_anchor_1 as Doc) ? '1' : '2';
     }
 
diff --git a/src/fields/DocSymbols.ts b/src/fields/DocSymbols.ts
index 66d1ab094..dc9d1084b 100644
--- a/src/fields/DocSymbols.ts
+++ b/src/fields/DocSymbols.ts
@@ -23,3 +23,5 @@ export const UpdatingFromServer = Symbol('DocUpdatingFromServer');
 export const Initializing = Symbol('DocInitializing');
 export const ForceServerWrite = Symbol('DocForceServerWrite');
 export const CachedUpdates = Symbol('DocCachedUpdates');
+
+export const DashVersion = 'v0.5.4';
diff --git a/src/fields/FieldLoader.tsx b/src/fields/FieldLoader.tsx
index 2a7b936f7..a5a71833c 100644
--- a/src/fields/FieldLoader.tsx
+++ b/src/fields/FieldLoader.tsx
@@ -6,10 +6,9 @@ import './FieldLoader.scss';
 
 @observer
 export class FieldLoader extends React.Component {
-    @observable public static ServerLoadStatus = { requested: 0, retrieved: 0 };
-    public static active = false;
+    @observable public static ServerLoadStatus = { requested: 0, retrieved: 0, message: '' };
 
     render() {
-        return <div className="fieldLoader">{`Requested: ${FieldLoader.ServerLoadStatus.requested} ... ${FieldLoader.ServerLoadStatus.retrieved} `}</div>;
+        return <div className="fieldLoader">{`${FieldLoader.ServerLoadStatus.message} request: ${FieldLoader.ServerLoadStatus.requested} ... ${FieldLoader.ServerLoadStatus.retrieved} `}</div>;
     }
 }
diff --git a/src/fields/List.ts b/src/fields/List.ts
index 183d644d3..f3fcc87f7 100644
--- a/src/fields/List.ts
+++ b/src/fields/List.ts
@@ -1,4 +1,4 @@
-import { action, observable } from 'mobx';
+import { action, computed, observable } from 'mobx';
 import { alias, list, serializable } from 'serializr';
 import { DocServer } from '../client/DocServer';
 import { ScriptingGlobals } from '../client/util/ScriptingGlobals';
@@ -72,13 +72,13 @@ class ListImpl<T extends Field> extends ObjectField {
             return res;
         },
         sort(cmpFunc: any) {
-            this[Self].__realFields(); // coerce retrieving entire array
+            this[Self].__realFields; // coerce retrieving entire array
             const res = this[Self].__fieldTuples.sort(cmpFunc ? (first: any, second: any) => cmpFunc(toRealField(first), toRealField(second)) : undefined);
             this[SelfProxy][FieldChanged]?.();
             return res;
         },
         splice: action(function (this: any, start: number, deleteCount: number, ...items: any[]) {
-            this[Self].__realFields(); // coerce retrieving entire array
+            this[Self].__realFields; // coerce retrieving entire array
             items = items.map(toObjectField);
             const list = this[Self];
             const removed = list.__fieldTuples.filter((item: any, i: number) => i >= start && i < start + deleteCount);
@@ -125,104 +125,104 @@ class ListImpl<T extends Field> extends ObjectField {
         },
         /// Accessor methods
         concat: action(function (this: any, ...items: any[]) {
-            this[Self].__realFields();
+            this[Self].__realFields;
             return this[Self].__fieldTuples.map(toRealField).concat(...items);
         }),
         includes(valueToFind: any, fromIndex: number) {
             if (valueToFind instanceof RefField) {
-                return this[Self].__realFields().includes(valueToFind, fromIndex);
+                return this[Self].__realFields.includes(valueToFind, fromIndex);
             } else {
                 return this[Self].__fieldTuples.includes(valueToFind, fromIndex);
             }
         },
         indexOf(valueToFind: any, fromIndex: number) {
             if (valueToFind instanceof RefField) {
-                return this[Self].__realFields().indexOf(valueToFind, fromIndex);
+                return this[Self].__realFields.indexOf(valueToFind, fromIndex);
             }
             return this[Self].__fieldTuples.indexOf(valueToFind, fromIndex);
         },
         join(separator: any) {
-            this[Self].__realFields();
+            this[Self].__realFields;
             return this[Self].__fieldTuples.map(toRealField).join(separator);
         },
         lastElement() {
-            return this[Self].__realFields().lastElement();
+            return this[Self].__realFields.lastElement();
         },
         lastIndexOf(valueToFind: any, fromIndex: number) {
             if (valueToFind instanceof RefField) {
-                return this[Self].__realFields().lastIndexOf(valueToFind, fromIndex);
+                return this[Self].__realFields.lastIndexOf(valueToFind, fromIndex);
             } else {
                 return this[Self].__fieldTuples.lastIndexOf(valueToFind, fromIndex);
             }
         },
         slice(begin: number, end: number) {
-            this[Self].__realFields();
+            this[Self].__realFields;
             return this[Self].__fieldTuples.slice(begin, end).map(toRealField);
         },
 
         /// Iteration methods
         entries() {
-            return this[Self].__realFields().entries();
+            return this[Self].__realFields.entries();
         },
         every(callback: any, thisArg: any) {
-            return this[Self].__realFields().every(callback, thisArg);
+            return this[Self].__realFields.every(callback, thisArg);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.every((element:any, index:number, array:any) => callback(toRealField(element), index, array), thisArg);
         },
         filter(callback: any, thisArg: any) {
-            return this[Self].__realFields().filter(callback, thisArg);
+            return this[Self].__realFields.filter(callback, thisArg);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.filter((element:any, index:number, array:any) => callback(toRealField(element), index, array), thisArg);
         },
         find(callback: any, thisArg: any) {
-            return this[Self].__realFields().find(callback, thisArg);
+            return this[Self].__realFields.find(callback, thisArg);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.find((element:any, index:number, array:any) => callback(toRealField(element), index, array), thisArg);
         },
         findIndex(callback: any, thisArg: any) {
-            return this[Self].__realFields().findIndex(callback, thisArg);
+            return this[Self].__realFields.findIndex(callback, thisArg);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.findIndex((element:any, index:number, array:any) => callback(toRealField(element), index, array), thisArg);
         },
         forEach(callback: any, thisArg: any) {
-            return this[Self].__realFields().forEach(callback, thisArg);
+            return this[Self].__realFields.forEach(callback, thisArg);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.forEach((element:any, index:number, array:any) => callback(toRealField(element), index, array), thisArg);
         },
         map(callback: any, thisArg: any) {
-            return this[Self].__realFields().map(callback, thisArg);
+            return this[Self].__realFields.map(callback, thisArg);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.map((element:any, index:number, array:any) => callback(toRealField(element), index, array), thisArg);
         },
         reduce(callback: any, initialValue: any) {
-            return this[Self].__realFields().reduce(callback, initialValue);
+            return this[Self].__realFields.reduce(callback, initialValue);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.reduce((acc:any, element:any, index:number, array:any) => callback(acc, toRealField(element), index, array), initialValue);
         },
         reduceRight(callback: any, initialValue: any) {
-            return this[Self].__realFields().reduceRight(callback, initialValue);
+            return this[Self].__realFields.reduceRight(callback, initialValue);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.reduceRight((acc:any, element:any, index:number, array:any) => callback(acc, toRealField(element), index, array), initialValue);
         },
         some(callback: any, thisArg: any) {
-            return this[Self].__realFields().some(callback, thisArg);
+            return this[Self].__realFields.some(callback, thisArg);
             // TODO This is probably more efficient, but technically the callback can take the array, which would mean we would have to map the actual array anyway.
             // If we don't want to support the array parameter, we should use this version instead
             // return this[Self].__fieldTuples.some((element:any, index:number, array:any) => callback(toRealField(element), index, array), thisArg);
         },
         values() {
-            return this[Self].__realFields().values();
+            return this[Self].__realFields.values();
         },
         [Symbol.iterator]() {
-            return this[Self].__realFields().values();
+            return this[Self].__realFields.values();
         },
     };
     static listGetter(target: any, prop: string | symbol, receiver: any): any {
@@ -262,7 +262,7 @@ class ListImpl<T extends Field> extends ObjectField {
 
     // this requests all ProxyFields at the same time to avoid the overhead
     // of separate network requests and separate updates to the React dom.
-    private __realFields() {
+    @computed private get __realFields() {
         const unrequested = this[FieldTuples].filter(f => f instanceof ProxyField && f.needsRequesting).map(f => f as ProxyField<RefField>);
         // if we find any ProxyFields that don't have a current value, then
         // start the server request for all of them
diff --git a/src/fields/Types.ts b/src/fields/Types.ts
index 251b1149d..69dbe9756 100644
--- a/src/fields/Types.ts
+++ b/src/fields/Types.ts
@@ -1,11 +1,10 @@
-import { Field, Opt, FieldResult, Doc } from './Doc';
+import { DateField } from './DateField';
+import { Doc, Field, FieldResult, Opt } from './Doc';
 import { List } from './List';
 import { RefField } from './RefField';
-import { DateField } from './DateField';
-import { ScriptField } from './ScriptField';
-import { URLField, WebField, ImageField, CsvField } from './URLField';
-import { TextField } from '@material-ui/core';
 import { RichTextField } from './RichTextField';
+import { ScriptField } from './ScriptField';
+import { CsvField, ImageField, WebField } from './URLField';
 
 export type ToType<T extends InterfaceValue> = T extends 'string'
     ? string
diff --git a/src/fields/URLField.ts b/src/fields/URLField.ts
index 8db4d6003..8ac20b1e5 100644
--- a/src/fields/URLField.ts
+++ b/src/fields/URLField.ts
@@ -25,7 +25,7 @@ export abstract class URLField extends ObjectField {
     constructor(url: URL | string) {
         super();
         if (typeof url === 'string') {
-            url = (url.startsWith('http') || url.startsWith('https')) ? new URL(url) : new URL(url, window.location.origin);
+            url = url.startsWith('http') ? new URL(url) : new URL(url, window.location.origin);
         }
         this.url = url;
     }