made dataViz nodes linked to schema nodes update automatically as cahnges are made. fixed user created templates from disappearing from menu, and made them work. added toJavascriptString and made DashField views convert to text. added support for turning text into javascript rendering (paint) code.

15 files changed, 94 insertions, 40 deletions

diff --git a/src/fields/CursorField.ts b/src/fields/CursorField.ts
index 84917ae53..870dfbeac 100644
--- a/src/fields/CursorField.ts
+++ b/src/fields/CursorField.ts
@@ -1,6 +1,6 @@
 import { createSimpleSchema, object, serializable } from 'serializr';
 import { Deserializable } from '../client/util/SerializationHelper';
-import { Copy, FieldChanged, ToScriptString, ToString } from './FieldSymbols';
+import { Copy, FieldChanged, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 import { ObjectField } from './ObjectField';
 
 export type CursorPosition = {
@@ -55,6 +55,9 @@ export default class CursorField extends ObjectField {
         return new CursorField(this.data);
     }
 
+    [ToJavascriptString]() {
+        return 'invalid';
+    }
     [ToScriptString]() {
         return 'invalid';
     }
diff --git a/src/fields/DateField.ts b/src/fields/DateField.ts
index 2ea619bd9..1e5222fb6 100644
--- a/src/fields/DateField.ts
+++ b/src/fields/DateField.ts
@@ -1,7 +1,7 @@
 import { Deserializable } from '../client/util/SerializationHelper';
 import { serializable, date } from 'serializr';
 import { ObjectField } from './ObjectField';
-import { Copy, ToScriptString, ToString } from './FieldSymbols';
+import { Copy, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 import { scriptingGlobal, ScriptingGlobals } from '../client/util/ScriptingGlobals';
 
 @scriptingGlobal
@@ -23,6 +23,9 @@ export class DateField extends ObjectField {
         return `${this.date.toLocaleString()}`;
     }
 
+    [ToJavascriptString]() {
+        return `"${this.date.toISOString()}"`;
+    }
     [ToScriptString]() {
         return `new DateField(new Date("${this.date.toISOString()}"))`;
     }
diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts
index ff416bbe7..f4141cf46 100644
--- a/src/fields/Doc.ts
+++ b/src/fields/Doc.ts
@@ -18,7 +18,7 @@ import {
     DocAcl, DocCss, DocData, DocFields, DocLayout, DocViews, FieldKeys, FieldTuples, ForceServerWrite, Height, Highlight,
     Initializing, Self, SelfProxy, UpdatingFromServer, Width
 } from './DocSymbols'; // prettier-ignore
-import { Copy, FieldChanged, HandleUpdate, Id, Parent, ToScriptString, ToString } from './FieldSymbols';
+import { Copy, FieldChanged, HandleUpdate, Id, Parent, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 import { InkField, InkTool } from './InkField';
 import { List, ListFieldName } from './List';
 import { ObjectField } from './ObjectField';
@@ -52,6 +52,23 @@ export namespace Field {
             default:       return field?.[ToScriptString]?.() ?? 'null';
         } // prettier-ignore
     }
+    export function toJavascriptString(field: Field) {
+        var rawjava = '';
+
+        switch (typeof field) {
+            case 'string':
+            case 'number':
+            case 'boolean':rawjava = String(field);
+                           break;
+            default:       rawjava = field?.[ToJavascriptString]?.() ?? 'null';
+        } // prettier-ignore
+        var script = rawjava;
+        Doc.MyPublishedDocs.forEach(doc => {
+            const regex = new RegExp(`^\\^${doc.title}\\s`, 'm');
+            script = script.replace(regex, Cast(doc.text, RichTextField, null)?.Text ?? '');
+        });
+        return script;
+    }
     export function toString(field: Field) {
         if (typeof field === 'string' || typeof field === 'number' || typeof field === 'boolean') return String(field);
         return field?.[ToString]?.() || '';
@@ -287,6 +304,7 @@ export class Doc extends RefField {
     public [DocFields] = () => this[Self][FieldTuples]; // Object.keys(this).reduce((fields, key) => { fields[key] = this[key]; return fields; }, {} as any);
     public [Width] = () => NumCast(this[SelfProxy]._width);
     public [Height] = () => NumCast(this[SelfProxy]._height);
+    public [ToJavascriptString] = () => `idToDoc("${this[Self][Id]}")`; // what should go here?
     public [ToScriptString] = () => `idToDoc("${this[Self][Id]}")`;
     public [ToString] = () => `Doc(${GetEffectiveAcl(this[SelfProxy]) === AclPrivate ? '-inaccessible-' : this[SelfProxy].title})`;
     public get [DocLayout]() { return this[SelfProxy].__LAYOUT__; } // prettier-ignore
@@ -508,12 +526,9 @@ export namespace Doc {
      * Removes doc from the list of Docs at listDoc[fieldKey]
      * @returns true if successful, false otherwise.
      */
-    export function RemoveDocFromList(listDoc: Doc, fieldKey: string | undefined, doc: Doc) {
+    export function RemoveDocFromList(listDoc: Doc, fieldKey: string | undefined, doc: Doc, ignoreProto = false) {
         const key = fieldKey ? fieldKey : Doc.LayoutFieldKey(listDoc);
-        if (listDoc[key] === undefined) {
-            listDoc[DocData][key] = new List<Doc>();
-        }
-        const list = Cast(listDoc[key], listSpec(Doc));
+        const list = Doc.Get(listDoc, key, ignoreProto) === undefined ? (listDoc[DocData][key] = new List<Doc>()) : Cast(listDoc[key], listSpec(Doc));
         if (list) {
             const ind = list.indexOf(doc);
             if (ind !== -1) {
@@ -528,12 +543,9 @@ export namespace Doc {
      * Adds doc to the list of Docs stored at listDoc[fieldKey].
      * @returns true if successful, false otherwise.
      */
-    export function AddDocToList(listDoc: Doc, fieldKey: string | undefined, doc: Doc, relativeTo?: Doc, before?: boolean, first?: boolean, allowDuplicates?: boolean, reversed?: boolean) {
+    export function AddDocToList(listDoc: Doc, fieldKey: string | undefined, doc: Doc, relativeTo?: Doc, before?: boolean, first?: boolean, allowDuplicates?: boolean, reversed?: boolean, ignoreProto?: boolean) {
         const key = fieldKey ? fieldKey : Doc.LayoutFieldKey(listDoc);
-        if (listDoc[key] === undefined) {
-            listDoc[DocData][key] = new List<Doc>();
-        }
-        const list = Cast(listDoc[key], listSpec(Doc));
+        const list = Doc.Get(listDoc, key, ignoreProto) === undefined ? (listDoc[DocData][key] = new List<Doc>()) : Cast(listDoc[key], listSpec(Doc));
         if (list) {
             if (!allowDuplicates) {
                 const pind = list.findIndex(d => d instanceof Doc && d[Id] === doc[Id]);
@@ -1020,7 +1032,7 @@ export namespace Doc {
     //
     export function MakeMetadataFieldTemplate(templateField: Doc, templateDoc: Opt<Doc>): boolean {
         // find the metadata field key that this template field doc will display (indicated by its title)
-        const metadataFieldKey = StrCast(templateField.isTemplateForField) || StrCast(templateField.title).replace(/^-/, '');
+        const metadataFieldKey = StrCast(templateField.isTemplateForField) || StrCast(templateField.title).replace(/^-/, '') || Doc.LayoutFieldKey(templateField);
 
         // update the original template to mark it as a template
         templateField.isTemplateForField = metadataFieldKey;
@@ -1640,3 +1652,6 @@ ScriptingGlobals.add(function setDocFilter(container: Doc, key: string, value: a
 ScriptingGlobals.add(function setDocRangeFilter(container: Doc, key: string, range: number[]) {
     Doc.setDocRangeFilter(container, key, range);
 });
+ScriptingGlobals.add(function toJavascriptString(str: string) {
+    return Field.toJavascriptString(str as Field);
+});
diff --git a/src/fields/FieldSymbols.ts b/src/fields/FieldSymbols.ts
index 0dbeb064b..0c44d2417 100644
--- a/src/fields/FieldSymbols.ts
+++ b/src/fields/FieldSymbols.ts
@@ -5,5 +5,6 @@ export const Parent = Symbol('FieldParent');
 export const Copy = Symbol('FieldCopy');
 export const ToValue = Symbol('FieldToValue');
 export const ToScriptString = Symbol('FieldToScriptString');
+export const ToJavascriptString = Symbol('FieldToJavascriptString');
 export const ToPlainText = Symbol('FieldToPlainText');
 export const ToString = Symbol('FieldToString');
diff --git a/src/fields/HtmlField.ts b/src/fields/HtmlField.ts
index 6e8bba977..b67f0f7e9 100644
--- a/src/fields/HtmlField.ts
+++ b/src/fields/HtmlField.ts
@@ -1,9 +1,9 @@
-import { Deserializable } from "../client/util/SerializationHelper";
-import { serializable, primitive } from "serializr";
-import { ObjectField } from "./ObjectField";
-import { Copy, ToScriptString, ToString} from "./FieldSymbols";
+import { Deserializable } from '../client/util/SerializationHelper';
+import { serializable, primitive } from 'serializr';
+import { ObjectField } from './ObjectField';
+import { Copy, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 
-@Deserializable("html")
+@Deserializable('html')
 export class HtmlField extends ObjectField {
     @serializable(primitive())
     readonly html: string;
@@ -17,8 +17,11 @@ export class HtmlField extends ObjectField {
         return new HtmlField(this.html);
     }
 
+    [ToJavascriptString]() {
+        return 'invalid';
+    }
     [ToScriptString]() {
-        return "invalid";
+        return 'invalid';
     }
     [ToString]() {
         return this.html;
diff --git a/src/fields/IconField.ts b/src/fields/IconField.ts
index 76c4ddf1b..4d2badb68 100644
--- a/src/fields/IconField.ts
+++ b/src/fields/IconField.ts
@@ -1,9 +1,9 @@
-import { Deserializable } from "../client/util/SerializationHelper";
-import { serializable, primitive } from "serializr";
-import { ObjectField } from "./ObjectField";
-import { Copy, ToScriptString, ToString } from "./FieldSymbols";
+import { Deserializable } from '../client/util/SerializationHelper';
+import { serializable, primitive } from 'serializr';
+import { ObjectField } from './ObjectField';
+import { Copy, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 
-@Deserializable("icon")
+@Deserializable('icon')
 export class IconField extends ObjectField {
     @serializable(primitive())
     readonly icon: string;
@@ -17,10 +17,13 @@ export class IconField extends ObjectField {
         return new IconField(this.icon);
     }
 
+    [ToJavascriptString]() {
+        return 'invalid';
+    }
     [ToScriptString]() {
-        return "invalid";
+        return 'invalid';
     }
     [ToString]() {
-        return "ICONfield";
+        return 'ICONfield';
     }
 }
diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts
index 22bea3927..b3e01229a 100644
--- a/src/fields/InkField.ts
+++ b/src/fields/InkField.ts
@@ -2,7 +2,7 @@ import { Bezier } from 'bezier-js';
 import { alias, createSimpleSchema, list, object, serializable } from 'serializr';
 import { ScriptingGlobals } from '../client/util/ScriptingGlobals';
 import { Deserializable } from '../client/util/SerializationHelper';
-import { Copy, ToScriptString, ToString } from './FieldSymbols';
+import { Copy, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 import { ObjectField } from './ObjectField';
 
 // Helps keep track of the current ink tool in use.
@@ -85,6 +85,9 @@ export class InkField extends ObjectField {
         return new InkField(this.inkData);
     }
 
+    [ToJavascriptString]() {
+        return '[' + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}}`) + ']';
+    }
     [ToScriptString]() {
         return 'new InkField([' + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}}`) + '])';
     }
diff --git a/src/fields/List.ts b/src/fields/List.ts
index b8ad552d2..9458a9611 100644
--- a/src/fields/List.ts
+++ b/src/fields/List.ts
@@ -5,7 +5,7 @@ import { ScriptingGlobals } from '../client/util/ScriptingGlobals';
 import { Deserializable, afterDocDeserialize, autoObject } from '../client/util/SerializationHelper';
 import { Field } from './Doc';
 import { FieldTuples, Self, SelfProxy } from './DocSymbols';
-import { Copy, FieldChanged, Parent, ToScriptString, ToString } from './FieldSymbols';
+import { Copy, FieldChanged, Parent, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 import { ObjectField } from './ObjectField';
 import { ProxyField } from './Proxy';
 import { RefField } from './RefField';
@@ -310,6 +310,9 @@ class ListImpl<T extends Field> extends ObjectField {
     private [Self] = this;
     private [SelfProxy]: List<Field>; // also used in utils.ts even though it won't be found using find all references
 
+    [ToJavascriptString]() {
+        return `[${(this as any).map((field: any) => Field.toScriptString(field))}]`;
+    }
     [ToScriptString]() {
         return `new List([${(this as any).map((field: any) => Field.toScriptString(field))}])`;
     }
diff --git a/src/fields/ObjectField.ts b/src/fields/ObjectField.ts
index b5bc2952a..e1b5b036c 100644
--- a/src/fields/ObjectField.ts
+++ b/src/fields/ObjectField.ts
@@ -1,5 +1,5 @@
 import { RefField } from './RefField';
-import { FieldChanged, Parent, Copy, ToScriptString, ToString } from './FieldSymbols';
+import { FieldChanged, Parent, Copy, ToScriptString, ToString, ToJavascriptString } from './FieldSymbols';
 import { ScriptingGlobals } from '../client/util/ScriptingGlobals';
 import { Field } from './Doc';
 
@@ -12,6 +12,7 @@ export abstract class ObjectField {
     public [Parent]?: RefField | ObjectField;
     abstract [Copy](): ObjectField;
 
+    abstract [ToJavascriptString](): string;
     abstract [ToScriptString](): string;
     abstract [ToString](): string;
 }
diff --git a/src/fields/Proxy.ts b/src/fields/Proxy.ts
index 3a46e3581..820d9b6ff 100644
--- a/src/fields/Proxy.ts
+++ b/src/fields/Proxy.ts
@@ -4,7 +4,7 @@ import { DocServer } from '../client/DocServer';
 import { scriptingGlobal } from '../client/util/ScriptingGlobals';
 import { Deserializable } from '../client/util/SerializationHelper';
 import { Field, FieldWaiting, Opt } from './Doc';
-import { Copy, Id, ToScriptString, ToString, ToValue } from './FieldSymbols';
+import { Copy, Id, ToJavascriptString, ToScriptString, ToString, ToValue } from './FieldSymbols';
 import { ObjectField } from './ObjectField';
 import { RefField } from './RefField';
 
@@ -38,6 +38,9 @@ export class ProxyField<T extends RefField> extends ObjectField {
         return new ProxyField<T>(this.fieldId);
     }
 
+    [ToJavascriptString]() {
+        return Field.toScriptString(this[ToValue](undefined)?.value);
+    }
     [ToScriptString]() {
         return Field.toScriptString(this[ToValue](undefined)?.value); // not sure this is quite right since it doesn't recreate a proxy field, but better than 'invalid' ?
     }
diff --git a/src/fields/RefField.ts b/src/fields/RefField.ts
index b6ef69750..01828dd14 100644
--- a/src/fields/RefField.ts
+++ b/src/fields/RefField.ts
@@ -1,11 +1,11 @@
-import { serializable, primitive, alias } from "serializr";
-import { Utils } from "../Utils";
-import { Id, HandleUpdate, ToScriptString, ToString } from "./FieldSymbols";
-import { afterDocDeserialize } from "../client/util/SerializationHelper";
+import { serializable, primitive, alias } from 'serializr';
+import { Utils } from '../Utils';
+import { Id, HandleUpdate, ToScriptString, ToString, ToJavascriptString } from './FieldSymbols';
+import { afterDocDeserialize } from '../client/util/SerializationHelper';
 
 export type FieldId = string;
 export abstract class RefField {
-    @serializable(alias("id", primitive({ afterDeserialize: afterDocDeserialize })))
+    @serializable(alias('id', primitive({ afterDeserialize: afterDocDeserialize })))
     private __id: FieldId;
     readonly [Id]: FieldId;
 
@@ -16,6 +16,7 @@ export abstract class RefField {
 
     protected [HandleUpdate]?(diff: any): void | Promise<void>;
 
+    abstract [ToJavascriptString](): string;
     abstract [ToScriptString](): string;
     abstract [ToString](): string;
 }
diff --git a/src/fields/RichTextField.ts b/src/fields/RichTextField.ts
index 3e75a071f..50cfab988 100644
--- a/src/fields/RichTextField.ts
+++ b/src/fields/RichTextField.ts
@@ -1,7 +1,7 @@
 import { serializable } from 'serializr';
 import { scriptingGlobal } from '../client/util/ScriptingGlobals';
 import { Deserializable } from '../client/util/SerializationHelper';
-import { Copy, ToScriptString, ToString } from './FieldSymbols';
+import { Copy, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
 import { ObjectField } from './ObjectField';
 
 @scriptingGlobal
@@ -27,6 +27,9 @@ export class RichTextField extends ObjectField {
         return new RichTextField(this.Data, this.Text);
     }
 
+    [ToJavascriptString]() {
+        return '`' + this.Text + '`';
+    }
     [ToScriptString]() {
         return `new RichTextField("${this.Data.replace(/"/g, '\\"')}", "${this.Text}")`;
     }
diff --git a/src/fields/SchemaHeaderField.ts b/src/fields/SchemaHeaderField.ts
index 6dde2e5aa..fb4dc4e5b 100644
--- a/src/fields/SchemaHeaderField.ts
+++ b/src/fields/SchemaHeaderField.ts
@@ -1,7 +1,7 @@
 import { Deserializable } from '../client/util/SerializationHelper';
 import { serializable, primitive } from 'serializr';
 import { ObjectField } from './ObjectField';
-import { Copy, ToScriptString, ToString, FieldChanged } from './FieldSymbols';
+import { Copy, ToScriptString, ToString, FieldChanged, ToJavascriptString } from './FieldSymbols';
 import { scriptingGlobal, ScriptingGlobals } from '../client/util/ScriptingGlobals';
 import { ColumnType } from '../client/views/collections/collectionSchema/CollectionSchemaView';
 
@@ -114,6 +114,9 @@ export class SchemaHeaderField extends ObjectField {
         return new SchemaHeaderField(this.heading, this.color, this.type, this.width, this.desc, this.collapsed);
     }
 
+    [ToJavascriptString]() {
+        return `["${this.heading}","${this.color}",${this.type},${this.width},${this.desc},${this.collapsed}]`;
+    }
     [ToScriptString]() {
         return `schemaHeaderField("${this.heading}","${this.color}",${this.type},${this.width},${this.desc},${this.collapsed})`;
     }
diff --git a/src/fields/ScriptField.ts b/src/fields/ScriptField.ts
index 62690a9fb..c7fe72ca6 100644
--- a/src/fields/ScriptField.ts
+++ b/src/fields/ScriptField.ts
@@ -6,7 +6,7 @@ import { scriptingGlobal, ScriptingGlobals } from '../client/util/ScriptingGloba
 import { autoObject, Deserializable } from '../client/util/SerializationHelper';
 import { numberRange } from '../Utils';
 import { Doc, Field, Opt } from './Doc';
-import { Copy, Id, ToScriptString, ToString, ToValue } from './FieldSymbols';
+import { Copy, Id, ToJavascriptString, ToScriptString, ToString, ToValue } from './FieldSymbols';
 import { List } from './List';
 import { ObjectField } from './ObjectField';
 import { Cast, StrCast } from './Types';
@@ -113,6 +113,9 @@ export class ScriptField extends ObjectField {
         return `${this.script.originalScript} + ${this.setterscript?.originalScript}`;
     }
 
+    [ToJavascriptString]() {
+        return this.script.originalScript;
+    }
     [ToScriptString]() {
         return this.script.originalScript;
     }
diff --git a/src/fields/URLField.ts b/src/fields/URLField.ts
index 817b62373..87334ad16 100644
--- a/src/fields/URLField.ts
+++ b/src/fields/URLField.ts
@@ -1,7 +1,7 @@
 import { Deserializable } from '../client/util/SerializationHelper';
 import { serializable, custom } from 'serializr';
 import { ObjectField } from './ObjectField';
-import { ToScriptString, ToString, Copy } from './FieldSymbols';
+import { ToScriptString, ToString, Copy, ToJavascriptString } from './FieldSymbols';
 import { scriptingGlobal } from '../client/util/ScriptingGlobals';
 import { Utils } from '../Utils';
 
@@ -36,6 +36,12 @@ export abstract class URLField extends ObjectField {
         }
         return `new ${this.constructor.name}("${this.url?.href}")`;
     }
+    [ToJavascriptString]() {
+        if (Utils.prepend(this.url?.pathname) === this.url?.href) {
+            return `new ${this.constructor.name}("${this.url.pathname}")`;
+        }
+        return `new ${this.constructor.name}("${this.url?.href}")`;
+    }
     [ToString]() {
         if (Utils.prepend(this.url?.pathname) === this.url?.href) {
             return this.url.pathname;