Merge branch 'moreUpgrading' into dataViz-annotations

8 files changed, 90 insertions, 54 deletions

diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts
index e91fcd858..8903a9f97 100644
--- a/src/fields/Doc.ts
+++ b/src/fields/Doc.ts
@@ -1,5 +1,5 @@
 import { saveAs } from 'file-saver';
-import { action, computed, observable, ObservableMap, ObservableSet, runInAction } from 'mobx';
+import { action, computed, makeObservable, observable, ObservableMap, ObservableSet, runInAction } from 'mobx';
 import { computedFn } from 'mobx-utils';
 import { alias, map, serializable } from 'serializr';
 import { DocServer } from '../client/DocServer';
@@ -30,7 +30,7 @@ import { ComputedField, ScriptField } from './ScriptField';
 import { BoolCast, Cast, DocCast, FieldValue, NumCast, StrCast, ToConstructor } from './Types';
 import { AudioField, CsvField, ImageField, PdfField, VideoField, WebField } from './URLField';
 import { containedFieldChangedHandler, deleteProperty, GetEffectiveAcl, getField, getter, makeEditable, makeReadOnly, setter, SharingPermissions } from './util';
-import JSZip = require('jszip');
+import * as JSZip from 'jszip';
 export const LinkedTo = '-linkedTo';
 export namespace Field {
     export function toKeyValueString(doc: Doc, key: string): string {
@@ -131,9 +131,9 @@ export function updateCachedAcls(doc: Doc) {
 @Deserializable('Doc', updateCachedAcls, ['id'])
 export class Doc extends RefField {
     @observable public static RecordingEvent = 0;
-    @observable public static GuestDashboard: Doc | undefined;
-    @observable public static GuestTarget: Doc | undefined;
-    @observable public static GuestMobile: Doc | undefined;
+    @observable public static GuestDashboard: Doc | undefined = undefined;
+    @observable public static GuestTarget: Doc | undefined = undefined;
+    @observable public static GuestMobile: Doc | undefined = undefined;
     public static CurrentUserEmail: string = '';
 
     public static get MySharedDocs()          { return DocCast(Doc.UserDoc().mySharedDocs); } // prettier-ignore
@@ -147,8 +147,9 @@ export class Doc extends RefField {
     public static get MyTopBarBtns()          { return DocCast(Doc.UserDoc().myTopBarBtns); } // prettier-ignore
     public static get MyRecentlyClosed()      { return DocCast(Doc.UserDoc().myRecentlyClosed); } // prettier-ignore
     public static get MyTrails()              { return DocCast(Doc.ActiveDashboard?.myTrails); } // prettier-ignore
-    public static get MyOverlayDocs()         { return DocListCast(Doc.ActiveDashboard?.myOverlayDocs ?? DocCast(Doc.UserDoc().myOverlayDocs).data); } // prettier-ignore
-    public static get MyPublishedDocs()       { return DocListCast(Doc.ActiveDashboard?.myPublishedDocs ?? DocCast(Doc.UserDoc().myPublishedDocs).data); } // prettier-ignore
+    public static get MyCalendars()           { return DocCast(Doc.ActiveDashboard?.myCalendars); } // prettier-ignore
+    public static get MyOverlayDocs()         { return DocListCast(Doc.ActiveDashboard?.myOverlayDocs ?? DocCast(Doc.UserDoc().myOverlayDocs)?.data); } // prettier-ignore
+    public static get MyPublishedDocs()       { return DocListCast(Doc.ActiveDashboard?.myPublishedDocs ?? DocCast(Doc.UserDoc().myPublishedDocs)?.data); } // prettier-ignore
     public static get MyDashboards()          { return DocCast(Doc.UserDoc().myDashboards); } // prettier-ignore
     public static get MyTemplates()           { return DocCast(Doc.UserDoc().myTemplates); } // prettier-ignore
     public static get MyImports()             { return DocCast(Doc.UserDoc().myImports); } // prettier-ignore
@@ -178,6 +179,7 @@ export class Doc extends RefField {
 
     constructor(id?: FieldId, forceSave?: boolean) {
         super(id);
+        makeObservable(this);
         const docProxy = new Proxy<this>(this, {
             set: setter,
             get: getter,
@@ -185,7 +187,36 @@ export class Doc extends RefField {
             has: (target, key) => GetEffectiveAcl(target) !== AclPrivate && key in target.__fieldTuples,
             ownKeys: target => {
                 const keys = GetEffectiveAcl(target) !== AclPrivate ? Object.keys(target[FieldKeys]) : [];
-                return [...keys, '__LAYOUT__'];
+                return [
+                    ...keys,
+                    AclAdmin,
+                    AclAugment,
+                    AclEdit,
+                    AclPrivate,
+                    AclReadonly,
+                    Animation,
+                    AudioPlay,
+                    Brushed,
+                    CachedUpdates,
+                    DirectLinks,
+                    DocAcl,
+                    DocCss,
+                    DocData,
+                    DocFields,
+                    DocLayout,
+                    DocViews,
+                    FieldKeys,
+                    FieldTuples,
+                    ForceServerWrite,
+                    Height,
+                    Highlight,
+                    Initializing,
+                    Self,
+                    SelfProxy,
+                    UpdatingFromServer,
+                    Width,
+                    '__LAYOUT__',
+                ];
             },
             getOwnPropertyDescriptor: (target, prop) => {
                 if (prop.toString() === '__LAYOUT__' || !(prop in target[FieldKeys])) {
@@ -527,17 +558,17 @@ export namespace Doc {
     }
 
     export function MakeEmbedding(doc: Doc, id?: string) {
-        const embedding = !GetT(doc, 'isDataDoc', 'boolean', true) && doc.proto ? Doc.MakeCopy(doc, undefined, id) : Doc.MakeDelegate(doc, id);
+        const embedding = (!GetT(doc, 'isDataDoc', 'boolean', true) && doc.proto) || doc.type === DocumentType.CONFIG ? Doc.MakeCopy(doc, undefined, id) : Doc.MakeDelegate(doc, id);
         const layout = Doc.LayoutField(embedding);
         if (layout instanceof Doc && layout !== embedding && layout === Doc.Layout(embedding)) {
             Doc.SetLayout(embedding, Doc.MakeEmbedding(layout));
         }
         embedding.createdFrom = doc;
         embedding.proto_embeddingId = Doc.GetProto(doc).proto_embeddingId = DocListCast(Doc.GetProto(doc).proto_embeddings).length - 1;
-        embedding.title = ComputedField.MakeFunction(`renameEmbedding(this)`);
+        !Doc.GetT(embedding, 'title', 'string', true) && (embedding.title = ComputedField.MakeFunction(`renameEmbedding(this)`));
         embedding.author = Doc.CurrentUserEmail;
 
-        Doc.AddDocToList(Doc.GetProto(doc)[DocData], 'proto_embeddings', embedding);
+        Doc.AddDocToList(doc[DocData], 'proto_embeddings', embedding);
 
         return embedding;
     }
@@ -751,7 +782,7 @@ export namespace Doc {
             });
     }
 
-    const _pendingMap: Map<string, boolean> = new Map();
+    const _pendingMap = new Set<string>();
     //
     // Returns an expanded template layout for a target data document if there is a template relationship
     // between the two. If so, the layoutDoc is expanded into a new document that inherits the properties
@@ -773,23 +804,25 @@ export namespace Doc {
 
         if (templateLayoutDoc.resolvedDataDoc instanceof Promise) {
             expandedTemplateLayout = undefined;
-            _pendingMap.set(targetDoc[Id] + expandedLayoutFieldKey, true);
-        } else if (expandedTemplateLayout === undefined && !_pendingMap.get(targetDoc[Id] + expandedLayoutFieldKey)) {
-            if (templateLayoutDoc.resolvedDataDoc === (targetDoc.rootDocument || Doc.GetProto(targetDoc))) {
+            _pendingMap.add(targetDoc[Id] + expandedLayoutFieldKey);
+        } else if (expandedTemplateLayout === undefined && !_pendingMap.has(targetDoc[Id] + expandedLayoutFieldKey)) {
+            if (templateLayoutDoc.resolvedDataDoc === (targetDoc.rootDocument ?? Doc.GetProto(targetDoc))) {
                 expandedTemplateLayout = templateLayoutDoc; // reuse an existing template layout if its for the same document with the same params
             } else {
                 templateLayoutDoc.resolvedDataDoc && (templateLayoutDoc = DocCast(templateLayoutDoc.proto, templateLayoutDoc)); // if the template has already been applied (ie, a nested template), then use the template's prototype
                 if (!targetDoc[expandedLayoutFieldKey]) {
-                    _pendingMap.set(targetDoc[Id] + expandedLayoutFieldKey, true);
+                    _pendingMap.add(targetDoc[Id] + expandedLayoutFieldKey);
                     setTimeout(
                         action(() => {
                             const newLayoutDoc = Doc.MakeDelegate(templateLayoutDoc, undefined, '[' + templateLayoutDoc.title + ']');
-                            newLayoutDoc.rootDocument = targetDoc;
                             const dataDoc = Doc.GetProto(targetDoc);
+                            newLayoutDoc.rootDocument = targetDoc;
+                            newLayoutDoc.embedContainer = targetDoc;
                             newLayoutDoc.resolvedDataDoc = dataDoc;
                             newLayoutDoc['acl-Guest'] = SharingPermissions.Edit;
-                            if (dataDoc[templateField] === undefined && templateLayoutDoc[templateField] instanceof List && (templateLayoutDoc[templateField] as any).length) {
-                                dataDoc[templateField] = ComputedField.MakeFunction(`ObjectField.MakeCopy(templateLayoutDoc["${templateField}"] as List)`, { templateLayoutDoc: Doc.name }, { templateLayoutDoc });
+                            if (dataDoc[templateField] === undefined && (templateLayoutDoc[templateField] as any)?.length) {
+                                dataDoc[templateField] = ObjectField.MakeCopy(templateLayoutDoc[templateField] as List<Doc>);
+                                // ComputedField.MakeFunction(`ObjectField.MakeCopy(templateLayoutDoc["${templateField}"])`, { templateLayoutDoc: Doc.name }, { templateLayoutDoc });
                             }
                             targetDoc[expandedLayoutFieldKey] = newLayoutDoc;
 
@@ -814,7 +847,8 @@ export namespace Doc {
     }
 
     export function FindReferences(infield: Doc | List<any>, references: Set<Doc>, system: boolean | undefined) {
-        if (infield instanceof List<any>) {
+        if (infield instanceof Promise) return;
+        if (!(infield instanceof Doc)) {
             infield.forEach(val => (val instanceof Doc || val instanceof List) && FindReferences(val, references, system));
             return;
         }
@@ -867,7 +901,7 @@ export namespace Doc {
     }
 
     export function MakeCopy(doc: Doc, copyProto: boolean = false, copyProtoId?: string, retitle = false): Doc {
-        const copy = new Doc(copyProtoId, true);
+        const copy = runInAction(() => new Doc(copyProtoId, true));
         updateCachedAcls(copy);
         const exclude = [...StrListCast(doc.cloneFieldFilter), 'dragFactory_count', 'cloneFieldFilter'];
         Object.keys(doc)
@@ -900,7 +934,7 @@ export namespace Doc {
             Doc.GetProto(copy).embedContainer = undefined;
             Doc.GetProto(copy).proto_embeddings = new List<Doc>([copy]);
         } else {
-            Doc.AddDocToList(Doc.GetProto(copy)[DocData], 'proto_embeddings', copy);
+            Doc.AddDocToList(copy[DocData], 'proto_embeddings', copy);
         }
         copy.embedContainer = undefined;
         if (retitle) {
@@ -1059,7 +1093,7 @@ export namespace Doc {
     }
     export function NativeHeight(doc?: Doc, dataDoc?: Doc, useHeight?: boolean) {
         if (!doc) return 0;
-        const nheight = (Doc.NativeWidth(doc, dataDoc, useHeight) * NumCast(doc._height)) / NumCast(doc._width);
+        const nheight = (Doc.NativeWidth(doc, dataDoc, useHeight) / NumCast(doc._width)) * NumCast(doc._height); // divide before multiply to avoid floating point errrorin case nativewidth = width
         const dheight = NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_nativeHeight'], useHeight ? NumCast(doc._height) : 0);
         return NumCast(doc._nativeHeight, nheight || dheight);
     }
@@ -1199,14 +1233,14 @@ export namespace Doc {
         return !doc
             ? undefined
             : doc.isTemplateDoc
-            ? doc
-            : Cast(doc.dragFactory, Doc, null)?.isTemplateDoc
-            ? doc.dragFactory
-            : Cast(Doc.Layout(doc), Doc, null)?.isTemplateDoc
-            ? Cast(Doc.Layout(doc), Doc, null).resolvedDataDoc
-                ? Doc.Layout(doc).proto
-                : Doc.Layout(doc)
-            : undefined;
+              ? doc
+              : Cast(doc.dragFactory, Doc, null)?.isTemplateDoc
+                ? doc.dragFactory
+                : Cast(Doc.Layout(doc), Doc, null)?.isTemplateDoc
+                  ? Cast(Doc.Layout(doc), Doc, null).resolvedDataDoc
+                      ? Doc.Layout(doc).proto
+                      : Doc.Layout(doc)
+                  : undefined;
     }
 
     export function deiconifyView(doc: Doc) {
@@ -1305,12 +1339,12 @@ export namespace Doc {
         });
     }
 
-    export function styleFromLayoutString(rootDoc: Doc, layoutDoc: Doc, props: any, scale: number) {
+    export function styleFromLayoutString(doc: Doc, props: any, scale: number) {
         const style: { [key: string]: any } = {};
         const divKeys = ['width', 'height', 'fontSize', 'transform', 'left', 'backgroundColor', 'left', 'right', 'top', 'bottom', 'pointerEvents', 'position'];
         const replacer = (match: any, expr: string, offset: any, string: any) => {
             // bcz: this executes a script to convert a property expression string:  { script }  into a value
-            return ScriptField.MakeFunction(expr, { self: Doc.name, this: Doc.name, scale: 'number' })?.script.run({ self: rootDoc, this: layoutDoc, scale }).result?.toString() ?? '';
+            return ScriptField.MakeFunction(expr, { self: Doc.name, this: Doc.name, scale: 'number' })?.script.run({ this: doc, self: doc, scale }).result?.toString() ?? '';
         };
         divKeys.map((prop: string) => {
             const p = props[prop];
diff --git a/src/fields/DocSymbols.ts b/src/fields/DocSymbols.ts
index 1fa99249a..9c563abbf 100644
--- a/src/fields/DocSymbols.ts
+++ b/src/fields/DocSymbols.ts
@@ -27,4 +27,4 @@ export const Initializing = Symbol('DocInitializing');
 export const ForceServerWrite = Symbol('DocForceServerWrite');
 export const CachedUpdates = Symbol('DocCachedUpdates');
 
-export const DashVersion = 'v0.6.50';
+export const DashVersion = 'v0.7.0';
diff --git a/src/fields/List.ts b/src/fields/List.ts
index da007e972..8c8ff1ea3 100644
--- a/src/fields/List.ts
+++ b/src/fields/List.ts
@@ -1,4 +1,4 @@
-import { action, computed, observable } from 'mobx';
+import { action, computed, makeObservable, observable } from 'mobx';
 import { alias, list, serializable } from 'serializr';
 import { DocServer } from '../client/DocServer';
 import { ScriptingGlobals } from '../client/util/ScriptingGlobals';
@@ -233,12 +233,13 @@ class ListImpl<T extends Field> extends ObjectField {
     }
     constructor(fields?: T[]) {
         super();
+        makeObservable(this);
         const list = new Proxy<this>(this, {
             set: setter,
             get: ListImpl.listGetter,
             ownKeys: target => {
                 const keys = Object.keys(target.__fieldTuples);
-                return [...keys, '__realFields'];
+                return [...keys, FieldTuples, Self, SelfProxy, '__realFields'];
             },
             getOwnPropertyDescriptor: (target, prop) => {
                 if (prop in target[FieldTuples]) {
diff --git a/src/fields/RichTextUtils.ts b/src/fields/RichTextUtils.ts
index dfd02dbc0..b84a91709 100644
--- a/src/fields/RichTextUtils.ts
+++ b/src/fields/RichTextUtils.ts
@@ -15,7 +15,7 @@ import { Doc, Opt } from './Doc';
 import { Id } from './FieldSymbols';
 import { RichTextField } from './RichTextField';
 import { Cast, StrCast } from './Types';
-import Color = require('color');
+import * as Color from 'color';
 
 export namespace RichTextUtils {
     const delimiter = '\n';
diff --git a/src/fields/Schema.ts b/src/fields/Schema.ts
index f035eeb0d..f5e64ae1f 100644
--- a/src/fields/Schema.ts
+++ b/src/fields/Schema.ts
@@ -94,6 +94,7 @@ export function makeStrictInterface<T extends Interface>(schema: T): (doc: Doc)
 }
 
 export function createSchema<T extends Interface>(schema: T): T & { proto: ToConstructor<Doc> } {
+    return undefined as any;
     (schema as any).proto = Doc;
     return schema as any;
 }
diff --git a/src/fields/ScriptField.ts b/src/fields/ScriptField.ts
index 2b8750714..62690a9fb 100644
--- a/src/fields/ScriptField.ts
+++ b/src/fields/ScriptField.ts
@@ -15,7 +15,7 @@ function optional(propSchema: PropSchema) {
     return custom(
         value => {
             if (value !== undefined) {
-                return propSchema.serializer(value);
+                return propSchema.serializer(value, '', undefined); // this function only takes one parameter, but I think its typescript typings are messed up to take 3
             }
             return SKIP;
         },
@@ -97,7 +97,7 @@ export class ScriptField extends ObjectField {
     constructor(script: CompiledScript | undefined, setterscript?: CompiledScript, rawscript?: string) {
         super();
 
-        const captured = script?.options.capturedVariables;
+        const captured = script?.options?.capturedVariables;
         if (captured) {
             this.captures = new List<string>(Object.keys(captured).map(key => key + ':' + (captured[key] instanceof Doc ? 'ID->' + (captured[key] as Doc)[Id] : captured[key].toString())));
         }
@@ -151,7 +151,7 @@ export class ComputedField extends ScriptField {
     _lastComputedResult: any;
     //TODO maybe add an observable cache based on what is passed in for doc, considering there shouldn't really be that many possible values for doc
     value = computedFn((doc: Doc) => this._valueOutsideReaction(doc));
-    _valueOutsideReaction = (doc: Doc) => (this._lastComputedResult = this.script.run({ this: doc, self: Cast(doc.rootDocument, Doc, null) ?? doc, value: '', _last_: this._lastComputedResult, _readOnly_: true }, console.log).result);
+    _valueOutsideReaction = (doc: Doc) => (this._lastComputedResult = this.script.compiled && this.script.run({ this: doc, self: doc, value: '', _last_: this._lastComputedResult, _readOnly_: true }, console.log).result);
 
     [ToValue](doc: Doc) {
         return ComputedField.toValue(doc, this);
@@ -167,35 +167,35 @@ export class ComputedField extends ScriptField {
         return compiled.compiled ? new ComputedField(compiled, compiledsetscript) : undefined;
     }
     public static MakeInterpolatedNumber(fieldKey: string, interpolatorKey: string, doc: Doc, curTimecode: number, defaultVal: Opt<number>) {
-        if (!doc[`${fieldKey}-indexed`]) {
+        if (!doc[`${fieldKey}_indexed`]) {
             const flist = new List<number>(numberRange(curTimecode + 1).map(i => undefined) as any as number[]);
             flist[curTimecode] = Cast(doc[fieldKey], 'number', null);
-            doc[`${fieldKey}-indexed`] = flist;
+            doc[`${fieldKey}_indexed`] = flist;
         }
-        const getField = ScriptField.CompileScript(`getIndexVal(self['${fieldKey}-indexed'], self.${interpolatorKey}, ${defaultVal})`, {}, true, {});
-        const setField = ScriptField.CompileScript(`setIndexVal(self['${fieldKey}-indexed'], self.${interpolatorKey}, value)`, { value: 'any' }, true, {});
+        const getField = ScriptField.CompileScript(`getIndexVal(this['${fieldKey}_indexed'], this.${interpolatorKey}, ${defaultVal})`, {}, true, {});
+        const setField = ScriptField.CompileScript(`setIndexVal(this['${fieldKey}_indexed'], this.${interpolatorKey}, value)`, { value: 'any' }, true, {});
         return getField.compiled ? new ComputedField(getField, setField?.compiled ? setField : undefined) : undefined;
     }
     public static MakeInterpolatedString(fieldKey: string, interpolatorKey: string, doc: Doc, curTimecode: number) {
-        if (!doc[`${fieldKey}-indexed`]) {
+        if (!doc[`${fieldKey}_`]) {
             const flist = new List<string>(numberRange(curTimecode + 1).map(i => undefined) as any as string[]);
             flist[curTimecode] = StrCast(doc[fieldKey]);
-            doc[`${fieldKey}-indexed`] = flist;
+            doc[`${fieldKey}_indexed`] = flist;
         }
-        const getField = ScriptField.CompileScript(`getIndexVal(self['${fieldKey}-indexed'], self.${interpolatorKey})`, {}, true, {});
-        const setField = ScriptField.CompileScript(`setIndexVal(self['${fieldKey}-indexed'], self.${interpolatorKey}, value)`, { value: 'any' }, true, {});
+        const getField = ScriptField.CompileScript(`getIndexVal(this['${fieldKey}_indexed'], this.${interpolatorKey})`, {}, true, {});
+        const setField = ScriptField.CompileScript(`setIndexVal(this['${fieldKey}_indexed'], this.${interpolatorKey}, value)`, { value: 'any' }, true, {});
         return getField.compiled ? new ComputedField(getField, setField?.compiled ? setField : undefined) : undefined;
     }
     public static MakeInterpolatedDataField(fieldKey: string, interpolatorKey: string, doc: Doc, curTimecode: number) {
         if (doc[`${fieldKey}`] instanceof List) return;
-        if (!doc[`${fieldKey}-indexed`]) {
+        if (!doc[`${fieldKey}_indexed`]) {
             const flist = new List<Field>(numberRange(curTimecode + 1).map(i => undefined) as any as Field[]);
             flist[curTimecode] = Field.Copy(doc[fieldKey]);
-            doc[`${fieldKey}-indexed`] = flist;
+            doc[`${fieldKey}_indexed`] = flist;
         }
-        const getField = ScriptField.CompileScript(`getIndexVal(self['${fieldKey}-indexed'], self.${interpolatorKey})`, {}, true, {});
+        const getField = ScriptField.CompileScript(`getIndexVal(this['${fieldKey}_indexed'], this.${interpolatorKey})`, {}, true, {});
         const setField = ScriptField.CompileScript(
-            `{setIndexVal (self['${fieldKey}-indexed'], self.${interpolatorKey}, value); console.log(self["${fieldKey}-indexed"][self.${interpolatorKey}],self.data,self["${fieldKey}-indexed"]))}`,
+            `{setIndexVal (this['${fieldKey}_indexed'], this.${interpolatorKey}, value); console.log(this["${fieldKey}_indexed"][this.${interpolatorKey}],this.data,this["${fieldKey}_indexed"]))}`,
             { value: 'any' },
             false,
             {}
diff --git a/src/fields/URLField.ts b/src/fields/URLField.ts
index 8ac20b1e5..817b62373 100644
--- a/src/fields/URLField.ts
+++ b/src/fields/URLField.ts
@@ -34,7 +34,7 @@ export abstract class URLField extends ObjectField {
         if (Utils.prepend(this.url?.pathname) === this.url?.href) {
             return `new ${this.constructor.name}("${this.url.pathname}")`;
         }
-        return `new ${this.constructor.name}("${this.url.href}")`;
+        return `new ${this.constructor.name}("${this.url?.href}")`;
     }
     [ToString]() {
         if (Utils.prepend(this.url?.pathname) === this.url?.href) {
diff --git a/src/fields/util.ts b/src/fields/util.ts
index ca02284da..545fe4478 100644
--- a/src/fields/util.ts
+++ b/src/fields/util.ts
@@ -21,7 +21,7 @@ function _readOnlySetter(): never {
     throw new Error("Documents can't be modified in read-only mode");
 }
 
-const tracing = false;
+var tracing = false;
 export function TraceMobx() {
     tracing && trace();
 }