path: root/src/fields/Doc.ts

import { saveAs } from 'file-saver';
import { action, computed, makeObservable, observable, ObservableMap, ObservableSet, runInAction } from 'mobx';
import { computedFn } from 'mobx-utils';
import { alias, map, serializable } from 'serializr';
import { DocServer } from '../client/DocServer';
import { CollectionViewType, DocumentType } from '../client/documents/DocumentTypes';
import { LinkManager } from '../client/util/LinkManager';
import { scriptingGlobal, ScriptingGlobals } from '../client/util/ScriptingGlobals';
import { afterDocDeserialize, autoObject, Deserializable, SerializationHelper } from '../client/util/SerializationHelper';
import { undoable } from '../client/util/UndoManager';
import { DocumentView } from '../client/views/nodes/DocumentView';
import { decycle } from '../decycler/decycler';
import * as JSZipUtils from '../JSZipUtils';
import { incrementTitleCopy, Utils } from '../Utils';
import { DateField } from './DateField';
import {
    AclAdmin, AclAugment, AclEdit, AclPrivate, AclReadonly, Animation, AudioPlay, Brushed, CachedUpdates, DirectLinks,
    DocAcl, DocCss, DocData, DocFields, DocLayout, DocViews, FieldKeys, FieldTuples, ForceServerWrite, Height, Highlight,
    Initializing, Self, SelfProxy, UpdatingFromServer, Width
} from './DocSymbols'; // prettier-ignore
import { Copy, FieldChanged, HandleUpdate, Id, Parent, ToScriptString, ToString } from './FieldSymbols';
import { InkField, InkTool } from './InkField';
import { List, ListFieldName } from './List';
import { ObjectField } from './ObjectField';
import { PrefetchProxy, ProxyField } from './Proxy';
import { FieldId, RefField } from './RefField';
import { RichTextField } from './RichTextField';
import { listSpec } from './Schema';
import { ComputedField, ScriptField } from './ScriptField';
import { BoolCast, Cast, DocCast, FieldValue, NumCast, StrCast, ToConstructor } from './Types';
import { AudioField, CsvField, ImageField, PdfField, VideoField, WebField } from './URLField';
import { containedFieldChangedHandler, deleteProperty, GetEffectiveAcl, getField, getter, makeEditable, makeReadOnly, setter, SharingPermissions } from './util';
import * as JSZip from 'jszip';
export const LinkedTo = '-linkedTo';
export namespace Field {
    export function toKeyValueString(doc: Doc, key: string): string {
        const onDelegate = Object.keys(doc).includes(key.replace(/^_/, ''));
        const field = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
        return !Field.IsField(field)
            ? key.startsWith('_')
                ? '='
                : ''
            : (onDelegate ? '=' : '') + (field instanceof ComputedField ? `:=${field.script.originalScript}` : field instanceof ScriptField ? `$=${field.script.originalScript}` : Field.toScriptString(field));
    }
    export function toScriptString(field: Field) {
        switch (typeof field) {
            case 'string': if (field.startsWith('{"')) return `'${field}'`; // bcz: hack ... want to quote the string the right way. if there are nested "'s, then use ' instead of ".  In this case, test for the start of a JSON string of the format {"property": ... } and use outer 's instead of "s
                           return !field.includes('`') ? `\`${field}\`` : `"${field}"`;
            case 'number':
            case 'boolean':return String(field);
            default:       return field?.[ToScriptString]?.() ?? 'null';
        } // prettier-ignore
    }
    export function toString(field: Field) {
        if (typeof field === 'string' || typeof field === 'number' || typeof field === 'boolean') return String(field);
        return field?.[ToString]?.() || '';
    }
    export function IsField(field: any): field is Field;
    export function IsField(field: any, includeUndefined: true): field is Field | undefined;
    export function IsField(field: any, includeUndefined: boolean = false): field is Field | undefined {
        return ['string', 'number', 'boolean'].includes(typeof field) || field instanceof ObjectField || field instanceof RefField || (includeUndefined && field === undefined);
    }
    export function Copy(field: any) {
        return field instanceof ObjectField ? ObjectField.MakeCopy(field) : field;
    }
}
export type Field = number | string | boolean | ObjectField | RefField;
export type Opt<T> = T | undefined;
export type FieldWaiting<T extends RefField = RefField> = T extends undefined ? never : Promise<T | undefined>;
export type FieldResult<T extends Field = Field> = Opt<T> | FieldWaiting<Extract<T, RefField>>;

/**
 * Cast any field to either a List of Docs or undefined if the given field isn't a List of Docs.
 * If a default value is given, that will be returned instead of undefined.
 * If a default value is given, the returned value should not be modified as it might be a temporary value.
 * If no default value is given, and the returned value is not undefined, it can be safely modified.
 */
export function DocListCastAsync(field: FieldResult): Promise<Doc[] | undefined>;
export function DocListCastAsync(field: FieldResult, defaultValue: Doc[]): Promise<Doc[]>;
export function DocListCastAsync(field: FieldResult, defaultValue?: Doc[]) {
    const list = Cast(field, listSpec(Doc));
    return list ? Promise.all(list).then(() => list) : Promise.resolve(defaultValue);
}
export function NumListCast(field: FieldResult, defaultVal: number[] = []) {
    return Cast(field, listSpec('number'), defaultVal);
}
export function StrListCast(field: FieldResult, defaultVal: string[] = []) {
    return Cast(field, listSpec('string'), defaultVal);
}
export function DocListCast(field: FieldResult, defaultVal: Doc[] = []) {
    return Cast(field, listSpec(Doc), defaultVal).filter(d => d instanceof Doc) as Doc[];
}

export enum aclLevel {
    unset = -1,
    unshared = 0,
    viewable = 1,
    augmentable = 2,
    editable = 3,
    admin = 4,
}
// prettier-ignore
export const HierarchyMapping: Map<symbol, { level:aclLevel; name: SharingPermissions; image: string }> = new Map([
    [AclPrivate,  { level: aclLevel.unshared,     name: SharingPermissions.None,    image: '▲' }],
    [AclReadonly, { level: aclLevel.viewable,     name: SharingPermissions.View,    image: '♦' }],
    [AclAugment,  { level: aclLevel.augmentable,  name: SharingPermissions.Augment, image: '⬟' }],
    [AclEdit,     { level: aclLevel.editable,     name: SharingPermissions.Edit,    image: '⬢' }],
    [AclAdmin,    { level: aclLevel.admin,        name: SharingPermissions.Admin,   image: '⬢' }],
]);
export const ReverseHierarchyMap: Map<string, { level: aclLevel; acl: symbol; image: string }> = new Map(Array.from(HierarchyMapping.entries()).map(value => [value[1].name, { level: value[1].level, acl: value[0], image: value[1].image }]));

// caches the document access permissions for the current user.
// this recursively updates all protos as well.
export function updateCachedAcls(doc: Doc) {
    if (!doc) return;

    const target = (doc as any)?.__fieldTuples ?? doc;
    const permissions: { [key: string]: symbol } = !target.author || target.author === Doc.CurrentUserEmail ? { 'acl-Me': AclAdmin } : {};
    Object.keys(target).filter(key => key.startsWith('acl') && (permissions[key] = ReverseHierarchyMap.get(StrCast(target[key]))!.acl));
    if (Object.keys(permissions).length || doc[DocAcl]?.length) {
        runInAction(() => (doc[DocAcl] = permissions));
    }

    if (doc.proto instanceof Promise) {
        doc.proto.then(proto => updateCachedAcls(DocCast(proto)));
        return doc.proto;
    }
}

@scriptingGlobal
@Deserializable('Doc', updateCachedAcls, ['id'])
export class Doc extends RefField {
    @observable public static RecordingEvent = 0;
    @observable public static GuestDashboard: Doc | undefined = undefined;
    @observable public static GuestTarget: Doc | undefined = undefined;
    @observable public static GuestMobile: Doc | undefined = undefined;
    public static CurrentUserEmail: string = '';

    public static get MySharedDocs()          { return DocCast(Doc.UserDoc().mySharedDocs); } // prettier-ignore
    public static get MyUserDocView()         { return DocCast(Doc.UserDoc().myUserDocView); } // prettier-ignore
    public static get MyDockedBtns()          { return DocCast(Doc.UserDoc().myDockedBtns); } // prettier-ignore
    public static get MySearcher()            { return DocCast(Doc.UserDoc().mySearcher); } // prettier-ignore
    public static get MyHeaderBar()           { return DocCast(Doc.UserDoc().myHeaderBar); } // prettier-ignore
    public static get MyLeftSidebarMenu()     { return DocCast(Doc.UserDoc().myLeftSidebarMenu); } // prettier-ignore
    public static get MyLeftSidebarPanel()    { return DocCast(Doc.UserDoc().myLeftSidebarPanel); } // prettier-ignore
    public static get MyContextMenuBtns()     { return DocCast(Doc.UserDoc().myContextMenuBtns); } // prettier-ignore
    public static get MyTopBarBtns()          { return DocCast(Doc.UserDoc().myTopBarBtns); } // prettier-ignore
    public static get MyRecentlyClosed()      { return DocCast(Doc.UserDoc().myRecentlyClosed); } // prettier-ignore
    public static get MyTrails()              { return DocCast(Doc.ActiveDashboard?.myTrails); } // prettier-ignore
    public static get MyOverlayDocs()         { return DocListCast(Doc.ActiveDashboard?.myOverlayDocs ?? DocCast(Doc.UserDoc().myOverlayDocs)?.data); } // prettier-ignore
    public static get MyPublishedDocs()       { return DocListCast(Doc.ActiveDashboard?.myPublishedDocs ?? DocCast(Doc.UserDoc().myPublishedDocs)?.data); } // prettier-ignore
    public static get MyDashboards()          { return DocCast(Doc.UserDoc().myDashboards); } // prettier-ignore
    public static get MyTemplates()           { return DocCast(Doc.UserDoc().myTemplates); } // prettier-ignore
    public static get MyImports()             { return DocCast(Doc.UserDoc().myImports); } // prettier-ignore
    public static get MyFilesystem()          { return DocCast(Doc.UserDoc().myFilesystem); } // prettier-ignore
    public static get MyTools()               { return DocCast(Doc.UserDoc().myTools); } // prettier-ignore
    public static get noviceMode()            { return BoolCast(Doc.UserDoc().noviceMode);  } // prettier-ignore
    public static set noviceMode(val)         { Doc.UserDoc().noviceMode = val; } // prettier-ignore
    public static get IsSharingEnabled()      { return BoolCast(Doc.UserDoc().isSharingEnabled); } // prettier-ignore
    public static set IsSharingEnabled(val)   { Doc.UserDoc().isSharingEnabled = val; } // prettier-ignore
    public static get defaultAclPrivate()     { return Doc.UserDoc().defaultAclPrivate; } // prettier-ignore
    public static set defaultAclPrivate(val)  { Doc.UserDoc().defaultAclPrivate = val; } // prettier-ignore
    public static get ActivePage()            { return StrCast(Doc.UserDoc().activePage); } // prettier-ignore
    public static set ActivePage(val)         { Doc.UserDoc().activePage = val; } // prettier-ignore
    public static get ActiveTool(): InkTool   { return StrCast(Doc.UserDoc().activeTool, InkTool.None) as InkTool;  } // prettier-ignore
    public static set ActiveTool(tool:InkTool){ Doc.UserDoc().activeTool = tool; } // prettier-ignore
    public static get ActivePresentation()    { return DocCast(Doc.ActiveDashboard?.activePresentation) as Opt<Doc>;  } // prettier-ignore
    public static set ActivePresentation(val) { Doc.ActiveDashboard && (Doc.ActiveDashboard.activePresentation = val) } // prettier-ignore
    public static get ActiveDashboard()       { return DocCast(Doc.UserDoc().activeDashboard); } // prettier-ignore
    public static set ActiveDashboard(val: Opt<Doc>) {  Doc.UserDoc().activeDashboard = val; } // prettier-ignore

    public static IsInMyOverlay(doc: Doc)     { return Doc.MyOverlayDocs.includes(doc); } // prettier-ignore
    public static AddToMyOverlay(doc: Doc)    { Doc.ActiveDashboard?.myOverlayDocs ? Doc.AddDocToList(Doc.ActiveDashboard, 'myOverlayDocs', doc) : Doc.AddDocToList(DocCast(Doc.UserDoc().myOverlayDocs), undefined, doc); } // prettier-ignore
    public static RemFromMyOverlay(doc: Doc)  { Doc.ActiveDashboard?.myOverlayDocs ? Doc.RemoveDocFromList(Doc.ActiveDashboard,'myOverlayDocs', doc) : Doc.RemoveDocFromList(DocCast(Doc.UserDoc().myOverlayDocs), undefined, doc); } // prettier-ignore
    public static AddToMyPublished(doc: Doc)  { Doc.ActiveDashboard?.myPublishedDocs ? Doc.AddDocToList(Doc.ActiveDashboard, 'myPublishedDocs', doc) : Doc.AddDocToList(DocCast(Doc.UserDoc().myPublishedDocs), undefined, doc); } // prettier-ignore
    public static RemFromMyPublished(doc: Doc){ Doc.ActiveDashboard?.myPublishedDocs ? Doc.RemoveDocFromList(Doc.ActiveDashboard,'myPublishedDocs', doc) : Doc.RemoveDocFromList(DocCast(Doc.UserDoc().myPublishedDocs), undefined, doc); } // prettier-ignore
    public static IsComicStyle(doc?: Doc)     { return doc && Doc.ActiveDashboard && !Doc.IsSystem(doc) && Doc.UserDoc().renderStyle === 'comic' ; } // prettier-ignore

    constructor(id?: FieldId, forceSave?: boolean) {
        super(id);
        makeObservable(this);
        const docProxy = new Proxy<this>(this, {
            set: setter,
            get: getter,
            // getPrototypeOf: (target) => Cast(target[SelfProxy].proto, Doc) || null, // TODO this might be able to replace the proto logic in getter
            has: (target, key) => GetEffectiveAcl(target) !== AclPrivate && key in target.__fieldTuples,
            ownKeys: target => {
                const keys = GetEffectiveAcl(target) !== AclPrivate ? Object.keys(target[FieldKeys]) : [];
                return [
                    ...keys,
                    AclAdmin,
                    AclAugment,
                    AclEdit,
                    AclPrivate,
                    AclReadonly,
                    Animation,
                    AudioPlay,
                    Brushed,
                    CachedUpdates,
                    DirectLinks,
                    DocAcl,
                    DocCss,
                    DocData,
                    DocFields,
                    DocLayout,
                    DocViews,
                    FieldKeys,
                    FieldTuples,
                    ForceServerWrite,
                    Height,
                    Highlight,
                    Initializing,
                    Self,
                    SelfProxy,
                    UpdatingFromServer,
                    Width,
                    '__LAYOUT__',
                ];
            },
            getOwnPropertyDescriptor: (target, prop) => {
                if (prop.toString() === '__LAYOUT__' || !(prop in target[FieldKeys])) {
                    return Reflect.getOwnPropertyDescriptor(target, prop);
                }
                return {
                    configurable: true, //TODO Should configurable be true?
                    enumerable: true,
                    value: 0, //() => target.__fieldTuples[prop])
                };
            },
            deleteProperty: deleteProperty,
            defineProperty: () => {
                throw new Error("Currently properties can't be defined on documents using Object.defineProperty");
            },
        });
        this[SelfProxy] = docProxy;
        if (!id || forceSave) {
            DocServer.CreateField(docProxy);
        }
        return docProxy;
    }

    [key: string]: FieldResult;

    @serializable(alias('fields', map(autoObject(), { afterDeserialize: afterDocDeserialize })))
    private get __fieldTuples() {
        return this[FieldTuples];
    }
    private set __fieldTuples(value) {
        // called by deserializer to set all fields in one shot
        this[FieldTuples] = value;
        for (const key in value) {
            const field = value[key];
            field !== undefined && (this[FieldKeys][key] = true);
            if (field instanceof ObjectField) {
                field[Parent] = this[Self];
                field[FieldChanged] = containedFieldChangedHandler(this[SelfProxy], key, field);
            }
        }
    }

    @observable private [FieldTuples]: any = {};
    @observable private [FieldKeys]: any = {};
    /// all of the raw acl's that have been set on this document.  Use GetEffectiveAcl to determine the actual ACL of the doc for editing
    @observable public [DocAcl]: { [key: string]: symbol } = {};
    @observable public [DocCss]: number = 0; // incrementer denoting a change to CSS layout
    @observable public [DirectLinks] = new ObservableSet<Doc>();
    @observable public [AudioPlay]: any; // meant to store sound object from Howl
    @observable public [Animation]: Opt<Doc>;
    @observable public [Highlight]: boolean = false;
    @observable public [Brushed]: boolean = false;
    @observable public [DocViews] = new ObservableSet<DocumentView>();

    private [Self] = this;
    private [SelfProxy]: any;
    private [UpdatingFromServer]: boolean = false;
    private [ForceServerWrite]: boolean = false;
    private [CachedUpdates]: { [key: string]: () => void | Promise<any> } = {};

    public [Initializing]: boolean = false;
    public [FieldChanged] = (diff: undefined | { op: '$addToSet' | '$remFromSet' | '$set'; items: Field[] | undefined; length: number | undefined; hint?: any }, serverOp: any) => {
        if (!this[UpdatingFromServer] || this[ForceServerWrite]) {
            DocServer.UpdateField(this[Id], serverOp);
        }
    };
    public [DocFields] = () => this[Self][FieldTuples]; // Object.keys(this).reduce((fields, key) => { fields[key] = this[key]; return fields; }, {} as any);
    public [Width] = () => NumCast(this[SelfProxy]._width);
    public [Height] = () => NumCast(this[SelfProxy]._height);
    public [ToScriptString] = () => `idToDoc("${this[Self][Id]}")`;
    public [ToString] = () => `Doc(${GetEffectiveAcl(this[SelfProxy]) === AclPrivate ? '-inaccessible-' : this[SelfProxy].title})`;
    public get [DocLayout]() { return this[SelfProxy].__LAYOUT__; } // prettier-ignore
    public get [DocData](): Doc {
        const self = this[SelfProxy];
        return self.resolvedDataDoc && !self.isTemplateForField ? self : Doc.GetProto(Cast(Doc.Layout(self).resolvedDataDoc, Doc, null) || self);
    }
    @computed get __LAYOUT__(): Doc | undefined {
        const self = this[SelfProxy];
        const templateLayoutDoc = Cast(Doc.LayoutField(self), Doc, null);
        if (templateLayoutDoc) {
            let renderFieldKey: any;
            const layoutField = templateLayoutDoc[StrCast(templateLayoutDoc.layout_fieldKey, 'layout')];
            if (typeof layoutField === 'string') {
                renderFieldKey = layoutField.split("fieldKey={'")[1].split("'")[0]; //layoutField.split("'")[1];
            } else {
                return Cast(layoutField, Doc, null);
            }
            return Cast(self[renderFieldKey + '_layout[' + templateLayoutDoc[Id] + ']'], Doc, null) || templateLayoutDoc;
        }
        return undefined;
    }

    public async [HandleUpdate](diff: any) {
        const set = diff.$set;
        const sameAuthor = this.author === Doc.CurrentUserEmail;
        if (set) {
            for (const key in set) {
                const fprefix = 'fields.';
                if (!key.startsWith(fprefix)) {
                    continue;
                }
                const fKey = key.substring(fprefix.length);
                const fn = async () => {
                    const value = await SerializationHelper.Deserialize(set[key]);
                    const prev = GetEffectiveAcl(this);
                    this[UpdatingFromServer] = true;
                    this[fKey] = value;
                    this[UpdatingFromServer] = false;
                    if (fKey.startsWith('acl')) {
                        updateCachedAcls(this);
                    }
                    if (prev === AclPrivate && GetEffectiveAcl(this) !== AclPrivate) {
                        DocServer.GetRefField(this[Id], true);
                    }
                };
                const writeMode = DocServer.getFieldWriteMode(fKey);
                if (fKey.startsWith('acl') || writeMode !== DocServer.WriteMode.Playground) {
                    delete this[CachedUpdates][fKey];
                    await fn();
                } else {
                    this[CachedUpdates][fKey] = fn;
                }
            }
        }
        const unset = diff.$unset;
        if (unset) {
            for (const key in unset) {
                if (!key.startsWith('fields.')) {
                    continue;
                }
                const fKey = key.substring(7);
                const fn = () => {
                    this[UpdatingFromServer] = true;
                    delete this[fKey];
                    this[UpdatingFromServer] = false;
                };
                if (sameAuthor || DocServer.getFieldWriteMode(fKey) !== DocServer.WriteMode.Playground) {
                    delete this[CachedUpdates][fKey];
                    await fn();
                } else {
                    this[CachedUpdates][fKey] = fn;
                }
            }
        }
    }
}

export namespace Doc {
    export function SetContainer(doc: Doc, container: Doc) {
        doc.embedContainer = container;
    }
    export function RunCachedUpdate(doc: Doc, field: string) {
        const update = doc[CachedUpdates][field];
        if (update) {
            update();
            delete doc[CachedUpdates][field];
        }
    }
    export function AddCachedUpdate(doc: Doc, field: string, oldValue: any) {
        const val = oldValue;
        doc[CachedUpdates][field] = () => {
            doc[UpdatingFromServer] = true;
            doc[field] = val;
            doc[UpdatingFromServer] = false;
        };
    }
    export function MakeReadOnly(): { end(): void } {
        makeReadOnly();
        return {
            end() {
                makeEditable();
            },
        };
    }

    export function Get(doc: Doc, key: string, ignoreProto: boolean = false): FieldResult {
        try {
            return getField(doc[Self], key, ignoreProto);
        } catch {
            return doc;
        }
    }
    export function GetT<T extends Field>(doc: Doc, key: string, ctor: ToConstructor<T>, ignoreProto: boolean = false): FieldResult<T> {
        return Cast(Get(doc, key, ignoreProto), ctor) as FieldResult<T>;
    }
    export function IsDataProto(doc: Doc) {
        return GetT(doc, 'isDataDoc', 'boolean', true);
    }
    export function IsBaseProto(doc: Doc) {
        return GetT(doc, 'isBaseProto', 'boolean', true);
    }
    export function IsSystem(doc: Doc) {
        return GetT(doc, 'isSystem', 'boolean', true);
    }
    export function IsDelegateField(doc: Doc, fieldKey: string) {
        return doc && Get(doc, fieldKey, true) !== undefined;
    }
    //
    // this will write the value to the key on either the data doc or the embedding doc.  The choice
    // of where to write it is based on:
    // 1) if the embedding Doc already has this field defined on it, then it will be written to the embedding
    // 2) if the data doc has the field, then it's written there.
    // 3) if neither already has the field, then 'defaultProto' determines whether to write it to the data doc (or the embedding)
    //
    export async function SetInPlace(doc: Doc, key: string, value: Field | undefined, defaultProto: boolean) {
        if (key.startsWith('_')) key = key.substring(1);
        const hasProto = Doc.GetProto(doc) !== doc ? Doc.GetProto(doc) : undefined;
        const onDeleg = Object.getOwnPropertyNames(doc).indexOf(key) !== -1;
        const onProto = hasProto && Object.getOwnPropertyNames(hasProto).indexOf(key) !== -1;
        if (onDeleg || !hasProto || (!onProto && !defaultProto)) {
            doc[key] = value;
        } else hasProto[key] = value;
    }
    export function GetAllPrototypes(doc: Doc): Doc[] {
        const protos: Doc[] = [];
        let d: Opt<Doc> = doc;
        while (d) {
            protos.push(d);
            d = DocCast(FieldValue(d.proto));
        }
        return protos;
    }

    /**
     * This function is intended to model Object.assign({}, {}) [https://mzl.la/1Mo3l21], which copies
     * the values of the properties of a source object into the target.
     *
     * This is just a specific, Dash-authored version that serves the same role for our
     * Doc class.
     *
     * @param doc the target document into which you'd like to insert the new fields
     * @param fields the fields to project onto the target. Its type signature defines a mapping from some string key
     * to a potentially undefined field, where each entry in this mapping is optional.
     */
    export function assign<K extends string>(doc: Doc, fields: Partial<Record<K, Opt<Field>>>, skipUndefineds: boolean = false, isInitializing = false) {
        isInitializing && (doc[Initializing] = true);
        for (const key in fields) {
            if (fields.hasOwnProperty(key)) {
                const value = fields[key];
                if (!skipUndefineds || value !== undefined) {
                    // Do we want to filter out undefineds?
                    doc[key] = value;
                }
            }
        }
        isInitializing && (doc[Initializing] = false);
        return doc;
    }

    // compare whether documents or their protos match
    export function AreProtosEqual(doc?: Doc, other?: Doc) {
        return doc && other && (doc === other || Doc.GetProto(doc) === Doc.GetProto(other));
    }

    // Gets the data document for the document.  Note: this is mis-named -- it does not specifically
    // return the doc's proto, but rather recursively searches through the proto inheritance chain
    // and returns the document who's proto is undefined or whose proto is marked as a data doc ('isDataDoc').
    export function GetProto(doc: Doc): Doc {
        const proto = doc && (Doc.GetT(doc, 'isDataDoc', 'boolean', true) ? doc : DocCast(doc.proto, doc));
        return proto === doc ? proto : Doc.GetProto(proto);
    }
    export function GetDataDoc(doc: Doc): Doc {
        const proto = Doc.GetProto(doc);
        return proto === doc ? proto : Doc.GetDataDoc(proto);
    }

    export function allKeys(doc: Doc): string[] {
        const results: Set<string> = new Set();

        let proto: Doc | undefined = doc;
        while (proto) {
            Object.keys(proto).forEach(key => results.add(key));
            proto = DocCast(FieldValue(proto.proto));
        }

        return Array.from(results);
    }

    /**
     * @returns the index of doc toFind in list of docs, -1 otherwise
     */
    export function IndexOf(toFind: Doc, list: Doc[]) {
        const index = list.indexOf(toFind);
        return index !== -1 ? index : list.findIndex(doc => Doc.AreProtosEqual(doc, toFind));
    }

    /**
     * Removes doc from the list of Docs at listDoc[fieldKey]
     * @returns true if successful, false otherwise.
     */
    export function RemoveDocFromList(listDoc: Doc, fieldKey: string | undefined, doc: Doc) {
        const key = fieldKey ? fieldKey : Doc.LayoutFieldKey(listDoc);
        if (listDoc[key] === undefined) {
            Doc.GetProto(listDoc)[key] = new List<Doc>();
        }
        const list = Cast(listDoc[key], listSpec(Doc));
        if (list) {
            const ind = list.indexOf(doc);
            if (ind !== -1) {
                list.splice(ind, 1);
                return true;
            }
        }
        return false;
    }

    /**
     * Adds doc to the list of Docs stored at listDoc[fieldKey].
     * @returns true if successful, false otherwise.
     */
    export function AddDocToList(listDoc: Doc, fieldKey: string | undefined, doc: Doc, relativeTo?: Doc, before?: boolean, first?: boolean, allowDuplicates?: boolean, reversed?: boolean) {
        const key = fieldKey ? fieldKey : Doc.LayoutFieldKey(listDoc);
        if (listDoc[key] === undefined) {
            Doc.GetProto(listDoc)[key] = new List<Doc>();
        }
        const list = Cast(listDoc[key], listSpec(Doc));
        if (list) {
            if (!allowDuplicates) {
                const pind = list.findIndex(d => d instanceof Doc && d[Id] === doc[Id]);
                if (pind !== -1) {
                    return true;
                }
            }
            if (first) {
                list.splice(0, 0, doc);
            } else {
                const ind = relativeTo ? list.indexOf(relativeTo) : -1;
                if (ind === -1) {
                    if (reversed) list.splice(0, 0, doc);
                    else list.push(doc);
                } else {
                    if (reversed) list.splice(before ? list.length - ind + 1 : list.length - ind, 0, doc);
                    else list.splice(before ? ind : ind + 1, 0, doc);
                }
            }
            return true;
        }
        return false;
    }

    export function MakeEmbedding(doc: Doc, id?: string) {
        const embedding = (!GetT(doc, 'isDataDoc', 'boolean', true) && doc.proto) || doc.type === DocumentType.CONFIG ? Doc.MakeCopy(doc, undefined, id) : Doc.MakeDelegate(doc, id);
        const layout = Doc.LayoutField(embedding);
        if (layout instanceof Doc && layout !== embedding && layout === Doc.Layout(embedding)) {
            Doc.SetLayout(embedding, Doc.MakeEmbedding(layout));
        }
        embedding.createdFrom = doc;
        embedding.proto_embeddingId = Doc.GetProto(doc).proto_embeddingId = DocListCast(Doc.GetProto(doc).proto_embeddings).length - 1;
        !Doc.GetT(embedding, 'title', 'string', true) && (embedding.title = ComputedField.MakeFunction(`renameEmbedding(this)`));
        embedding.author = Doc.CurrentUserEmail;

        Doc.AddDocToList(doc[DocData], 'proto_embeddings', embedding);

        return embedding;
    }

    export function BestEmbedding(doc: Doc) {
        const bestEmbedding = Doc.GetProto(doc) ? [doc, ...DocListCast(doc.proto_embeddings)].find(doc => !doc.embedContainer && doc.author === Doc.CurrentUserEmail) : doc;
        bestEmbedding && Doc.AddDocToList(Doc.GetProto(doc), 'protoEmbeddings', doc);
        return bestEmbedding ?? Doc.MakeEmbedding(doc);
    }

    // this lists out all the tag ids that can be  in a RichTextField that might contain document ids.
    // if a document is cloned, we need to make sure to clone all of these referenced documents as well;
    export const DocsInTextFieldIds = ['audioId', 'textId', 'anchorId', 'docId'];
    export async function makeClone(doc: Doc, cloneMap: Map<string, Doc>, linkMap: Map<string, Doc>, rtfs: { copy: Doc; key: string; field: RichTextField }[], exclusions: string[], pruneDocs: Doc[], cloneLinks: boolean): Promise<Doc> {
        if (Doc.IsBaseProto(doc)) return doc;
        if (cloneMap.get(doc[Id])) return cloneMap.get(doc[Id])!;
        const copy = new Doc(undefined, true);
        cloneMap.set(doc[Id], copy);
        const filter = [...exclusions, ...StrListCast(doc.cloneFieldFilter)];
        await Promise.all(
            Object.keys(doc).map(async key => {
                if (filter.includes(key)) return;
                const assignKey = (val: any) => (copy[key] = val);
                const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
                const field = ProxyField.WithoutProxy(() => doc[key]);
                const copyObjectField = async (field: ObjectField) => {
                    const list = await Cast(doc[key], listSpec(Doc));
                    const docs = list && (await DocListCastAsync(list))?.filter(d => d instanceof Doc);
                    if (docs !== undefined && docs.length) {
                        const clones = await Promise.all(docs.map(async d => Doc.makeClone(d, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks)));
                        assignKey(new List<Doc>(clones));
                    } else {
                        assignKey(ObjectField.MakeCopy(field));
                        if (field instanceof RichTextField) {
                            if (DocsInTextFieldIds.some(id => field.Data.includes(`"${id}":`))) {
                                const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => '(' + exp + ')').join('|') + ')":"([a-z-A-Z0-9_]*)"', 'g');
                                const rawdocids = field.Data.match(docidsearch);
                                const docids = rawdocids?.map((str: string) =>
                                    DocsInTextFieldIds.reduce((output, exp) => output.replace(new RegExp(`${exp}":`, 'g'), ''), str)
                                        .replace(/"/g, '')
                                        .trim()
                                );
                                const results = docids && (await DocServer.GetRefFields(docids));
                                const docs = results && Array.from(Object.keys(results)).map(key => DocCast(results[key]));
                                docs?.map(doc => doc && Doc.makeClone(doc, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks));
                                rtfs.push({ copy, key, field });
                            }
                        }
                    }
                };
                const docAtKey = doc[key];
                if (key === 'author') {
                    assignKey(Doc.CurrentUserEmail);
                } else if (docAtKey instanceof Doc) {
                    if (pruneDocs.includes(docAtKey)) {
                        // prune doc and do nothing
                    } else if (!Doc.IsSystem(docAtKey) && (key.startsWith('layout') || ['embedContainer', 'annotationOn', 'proto'].includes(key) || ((key === 'link_anchor_1' || key === 'link_anchor_2') && doc.author === Doc.CurrentUserEmail))) {
                        assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks));
                    } else {
                        assignKey(docAtKey);
                    }
                } else if (field instanceof RefField) {
                    assignKey(field);
                } else if (cfield instanceof ComputedField) {
                    assignKey(cfield[Copy]());
                } else if (field instanceof ObjectField) {
                    await copyObjectField(field);
                } else if (field instanceof Promise) {
                    debugger; //This shouldn't happen...
                } else {
                    assignKey(field);
                }
            })
        );
        Array.from(doc[DirectLinks]).forEach(async link => {
            if (
                cloneLinks ||
                ((cloneMap.has(DocCast(link.link_anchor_1)?.[Id]) || cloneMap.has(DocCast(DocCast(link.link_anchor_1)?.annotationOn)?.[Id])) &&
                    (cloneMap.has(DocCast(link.link_anchor_2)?.[Id]) || cloneMap.has(DocCast(DocCast(link.link_anchor_2)?.annotationOn)?.[Id])))
            ) {
                linkMap.set(link[Id], await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks));
            }
        });
        Doc.SetInPlace(copy, 'title', '>:' + doc.title, true);
        copy.cloneOf = doc;
        cloneMap.set(doc[Id], copy);

        return copy;
    }
    export function repairClone(clone: Doc, cloneMap: Map<string, Doc>, visited: Set<Doc>) {
        if (visited.has(clone)) return;
        visited.add(clone);
        Object.keys(clone)
            .filter(key => key !== 'cloneOf')
            .map(key => {
                const docAtKey = DocCast(clone[key]);
                if (docAtKey && !Doc.IsSystem(docAtKey)) {
                    if (!Array.from(cloneMap.values()).includes(docAtKey)) {
                        clone[key] = cloneMap.get(docAtKey[Id]);
                    } else {
                        repairClone(docAtKey, cloneMap, visited);
                    }
                }
            });
    }
    export function MakeClones(docs: Doc[], cloneLinks: boolean) {
        const cloneMap = new Map<string, Doc>();
        return docs.map(doc => Doc.MakeClone(doc, cloneLinks, cloneMap));
    }

    export async function MakeClone(doc: Doc, cloneLinks = true, cloneMap: Map<string, Doc> = new Map()) {
        const linkMap = new Map<string, Doc>();
        const rtfMap: { copy: Doc; key: string; field: RichTextField }[] = [];
        const copy = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf'], doc.embedContainer ? [DocCast(doc.embedContainer)] : [], cloneLinks);
        const repaired = new Set<Doc>();
        const linkedDocs = Array.from(linkMap.values());
        linkedDocs.map((link: Doc) => LinkManager.Instance.addLink(link, true));
        rtfMap.map(({ copy, key, field }) => {
            const replacer = (match: any, attr: string, id: string, offset: any, string: any) => {
                const mapped = cloneMap.get(id);
                return attr + '"' + (mapped ? mapped[Id] : id) + '"';
            };
            const replacer2 = (match: any, href: string, id: string, offset: any, string: any) => {
                const mapped = cloneMap.get(id);
                return href + (mapped ? mapped[Id] : id);
            };
            const re = new RegExp(`(${Doc.localServerPath()})([^"]*)`, 'g');
            const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => `"${exp}":`).join('|') + ')"([^"]+)"', 'g');
            copy[key] = new RichTextField(field.Data.replace(docidsearch, replacer).replace(re, replacer2), field.Text);
        });
        const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs];
        clonedDocs.map(clone => Doc.repairClone(clone, cloneMap, repaired));
        return { clone: copy, map: cloneMap, linkMap };
    }

    export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') {
        const { clone, map, linkMap } = await Doc.MakeClone(doc);
        const proms = new Set<string>();
        function replacer(key: any, value: any) {
            if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined;
            if (value?.__type === 'image') {
                const extension = value.url.replace(/.*\./, '');
                proms.add(value.url.replace('.' + extension, '_o.' + extension));
                return SerializationHelper.Serialize(new ImageField(value.url));
            }
            if (value?.__type === 'pdf') {
                proms.add(value.url);
                return SerializationHelper.Serialize(new PdfField(value.url));
            }
            if (value?.__type === 'audio') {
                proms.add(value.url);
                return SerializationHelper.Serialize(new AudioField(value.url));
            }
            if (value?.__type === 'video') {
                proms.add(value.url);
                return SerializationHelper.Serialize(new VideoField(value.url));
            }
            if (
                value instanceof Doc ||
                value instanceof ScriptField ||
                value instanceof RichTextField ||
                value instanceof InkField ||
                value instanceof CsvField ||
                value instanceof WebField ||
                value instanceof DateField ||
                value instanceof ProxyField ||
                value instanceof ComputedField
            ) {
                return SerializationHelper.Serialize(value);
            }
            if (value instanceof Array && key !== ListFieldName && key !== InkField.InkDataFieldName) return { fields: value, __type: 'list' };
            return value;
        }

        const docs: { [id: string]: any } = {};
        const links: { [id: string]: any } = {};
        Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1]));
        Array.from(linkMap.entries()).forEach(l => (links[l[0]] = l[1]));
        const jsonDocs = JSON.stringify({ id: clone[Id], docs, links }, decycle(replacer));

        const zip = new JSZip();
        var count = 0;
        const promArr = Array.from(proms)
            .filter(url => url?.startsWith('/files'))
            .map(url => url.replace('/', '')); // window.location.origin));
        console.log(promArr.length);
        if (!promArr.length) {
            zip.file('docs.json', jsonDocs);
            zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename));
        } else
            promArr.forEach((url, i) => {
                // loading a file and add it in a zip file
                JSZipUtils.getBinaryContent(window.location.origin + '/' + url, (err: any, data: any) => {
                    if (err) throw err; // or handle the error
                    // // Generate a directory within the Zip file structure
                    // const assets = zip.folder("assets");
                    // assets.file(filename, data, {binary: true});
                    const assetPathOnServer = promArr[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%');
                    zip.file(assetPathOnServer, data, { binary: true });
                    console.log(' => ' + url);
                    if (++count === promArr.length) {
                        zip.file('docs.json', jsonDocs);
                        zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename));
                        // const a = document.createElement("a");
                        // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`);
                        // a.href = url;
                        // a.download = `DocExport-${this.props.Document[Id]}.zip`;
                        // a.click();
                    }
                });
            });
    }

    const _pendingMap = new Set<string>();
    //
    // Returns an expanded template layout for a target data document if there is a template relationship
    // between the two. If so, the layoutDoc is expanded into a new document that inherits the properties
    // of the original layout while allowing for individual layout properties to be overridden in the expanded layout.
    export function expandTemplateLayout(templateLayoutDoc: Doc, targetDoc?: Doc) {
        // nothing to do if the layout isn't a template or we don't have a target that's different than the template
        if (!targetDoc || templateLayoutDoc === targetDoc || (!templateLayoutDoc.isTemplateForField && !templateLayoutDoc.isTemplateDoc)) {
            return templateLayoutDoc;
        }

        const templateField = StrCast(templateLayoutDoc.isTemplateForField, Doc.LayoutFieldKey(templateLayoutDoc)); // the field that the template renders
        // First it checks if an expanded layout already exists -- if so it will be stored on the dataDoc
        // using the template layout doc's id as the field key.
        // If it doesn't find the expanded layout, then it makes a delegate of the template layout and
        // saves it on the data doc indexed by the template layout's id.
        //
        const expandedLayoutFieldKey = templateField + '_layout[' + templateLayoutDoc[Id] + ']';
        let expandedTemplateLayout = targetDoc?.[expandedLayoutFieldKey];

        if (templateLayoutDoc.resolvedDataDoc instanceof Promise) {
            expandedTemplateLayout = undefined;
            _pendingMap.add(targetDoc[Id] + expandedLayoutFieldKey);
        } else if (expandedTemplateLayout === undefined && !_pendingMap.has(targetDoc[Id] + expandedLayoutFieldKey)) {
            if (templateLayoutDoc.resolvedDataDoc === (targetDoc.rootDocument ?? Doc.GetProto(targetDoc))) {
                expandedTemplateLayout = templateLayoutDoc; // reuse an existing template layout if its for the same document with the same params
            } else {
                templateLayoutDoc.resolvedDataDoc && (templateLayoutDoc = DocCast(templateLayoutDoc.proto, templateLayoutDoc)); // if the template has already been applied (ie, a nested template), then use the template's prototype
                if (!targetDoc[expandedLayoutFieldKey]) {
                    _pendingMap.add(targetDoc[Id] + expandedLayoutFieldKey);
                    setTimeout(
                        action(() => {
                            const newLayoutDoc = Doc.MakeDelegate(templateLayoutDoc, undefined, '[' + templateLayoutDoc.title + ']');
                            const dataDoc = Doc.GetProto(targetDoc);
                            newLayoutDoc.rootDocument = targetDoc;
                            newLayoutDoc.embedContainer = targetDoc;
                            newLayoutDoc.resolvedDataDoc = dataDoc;
                            newLayoutDoc['acl-Guest'] = SharingPermissions.Edit;
                            if (dataDoc[templateField] === undefined && (templateLayoutDoc[templateField] as any)?.length) {
                                dataDoc[templateField] = ObjectField.MakeCopy(templateLayoutDoc[templateField] as List<Doc>);
                                // ComputedField.MakeFunction(`ObjectField.MakeCopy(templateLayoutDoc["${templateField}"])`, { templateLayoutDoc: Doc.name }, { templateLayoutDoc });
                            }
                            targetDoc[expandedLayoutFieldKey] = newLayoutDoc;

                            _pendingMap.delete(targetDoc[Id] + expandedLayoutFieldKey);
                        })
                    );
                }
            }
        }
        return expandedTemplateLayout instanceof Doc ? expandedTemplateLayout : undefined; // layout is undefined if the expandedTemplateLayout is pending.
    }

    // if the childDoc is a template for a field, then this will return the expanded layout with its data doc.
    // otherwise, it just returns the childDoc
    export function GetLayoutDataDocPair(containerDoc: Doc, containerDataDoc: Opt<Doc>, childDoc: Doc) {
        if (!childDoc || childDoc instanceof Promise || !Doc.GetProto(childDoc)) {
            console.log('Warning: GetLayoutDataDocPair childDoc not defined');
            return { layout: childDoc, data: childDoc };
        }
        const resolvedDataDoc = Doc.AreProtosEqual(containerDataDoc, containerDoc) || (!childDoc.isTemplateDoc && !childDoc.isTemplateForField) ? undefined : containerDataDoc;
        return { layout: Doc.expandTemplateLayout(childDoc, resolvedDataDoc), data: resolvedDataDoc };
    }

    export function FindReferences(infield: Doc | List<any>, references: Set<Doc>, system: boolean | undefined) {
        if (infield instanceof Promise) return;
        if (!(infield instanceof Doc)) {
            infield.forEach(val => (val instanceof Doc || val instanceof List) && FindReferences(val, references, system));
            return;
        }
        const doc = infield as Doc;
        if (references.has(doc)) {
            references.add(doc);
            return;
        }
        const excludeLists = [Doc.MyRecentlyClosed, Doc.MyHeaderBar, Doc.MyDashboards].includes(doc);
        if (system !== undefined && ((system && !Doc.IsSystem(doc)) || (!system && Doc.IsSystem(doc)))) return;
        references.add(doc);
        Object.keys(doc).forEach(key => {
            if (key === 'proto') {
                if (doc.proto instanceof Doc) {
                    Doc.FindReferences(doc.proto, references, system);
                }
            } else {
                const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
                const field = key === 'author' ? Doc.CurrentUserEmail : ProxyField.WithoutProxy(() => doc[key]);
                if (field instanceof RefField) {
                    if (field instanceof Doc) {
                        if (key === 'myLinkDatabase') {
                            field instanceof Doc && references.add(field);
                            // skip docs that have been closed and are scheduled for garbage collection
                        } else {
                            Doc.FindReferences(field, references, system);
                        }
                    }
                } else if (cfield instanceof ComputedField) {
                } else if (field instanceof ObjectField) {
                    if (field instanceof Doc) {
                        Doc.FindReferences(field, references, system);
                    } else if (field instanceof List) {
                        !excludeLists && Doc.FindReferences(field, references, system);
                    } else if (field instanceof ProxyField) {
                        if (key === 'myLinkDatabase') {
                            field instanceof Doc && references.add(field);
                            // skip docs that have been closed and are scheduled for garbage collection
                        } else {
                            Doc.FindReferences(field.value, references, system);
                        }
                    } else if (field instanceof PrefetchProxy) {
                        Doc.FindReferences(field.value, references, system);
                    }
                } else if (field instanceof Promise) {
                    debugger; //This shouldn't happend...
                }
            }
        });
    }

    export function MakeCopy(doc: Doc, copyProto: boolean = false, copyProtoId?: string, retitle = false): Doc {
        const copy = runInAction(() => new Doc(copyProtoId, true));
        updateCachedAcls(copy);
        const exclude = [...StrListCast(doc.cloneFieldFilter), 'dragFactory_count', 'cloneFieldFilter'];
        Object.keys(doc)
            .filter(key => !exclude.includes(key))
            .forEach(key => {
                if (key === 'proto' && copyProto) {
                    if (doc.proto instanceof Doc) {
                        copy[key] = Doc.MakeCopy(doc.proto, false);
                    }
                } else {
                    const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
                    const field = key === 'author' ? Doc.CurrentUserEmail : ProxyField.WithoutProxy(() => doc[key]);
                    if (field instanceof RefField) {
                        copy[key] = field;
                    } else if (cfield instanceof ComputedField) {
                        copy[key] = cfield[Copy](); // ComputedField.MakeFunction(cfield.script.originalScript);
                    } else if (field instanceof ObjectField) {
                        copy[key] =
                            doc[key] instanceof Doc && key.includes('layout[')
                                ? undefined // remove expanded template field documents
                                : ObjectField.MakeCopy(field);
                    } else if (field instanceof Promise) {
                        debugger; //This shouldn't happend...
                    } else {
                        copy[key] = field;
                    }
                }
            });
        if (copyProto) {
            Doc.GetProto(copy).embedContainer = undefined;
            Doc.GetProto(copy).proto_embeddings = new List<Doc>([copy]);
        } else {
            Doc.AddDocToList(copy[DocData], 'proto_embeddings', copy);
        }
        copy.embedContainer = undefined;
        if (retitle) {
            copy.title = incrementTitleCopy(StrCast(copy.title));
        }
        return copy;
    }

    export function MakeDelegate(doc: Doc, id?: string, title?: string): Doc;
    export function MakeDelegate(doc: Opt<Doc>, id?: string, title?: string): Opt<Doc>;
    export function MakeDelegate(doc: Opt<Doc>, id?: string, title?: string): Opt<Doc> {
        if (doc) {
            const delegate = new Doc(id, true);
            delegate[Initializing] = true;
            updateCachedAcls(delegate);
            delegate.proto = doc;
            delegate.author = Doc.CurrentUserEmail;
            Object.keys(doc)
                .filter(key => key.startsWith('acl'))
                .forEach(key => (delegate[key] = doc[key]));
            if (!Doc.IsSystem(doc)) Doc.AddDocToList(doc[DocData], 'proto_embeddings', delegate);
            title && (delegate.title = title);
            delegate[Initializing] = false;
            return delegate;
        }
        return undefined;
    }

    // Makes a delegate of a document by first creating a delegate where data should be stored
    // (ie, the 'data' doc), and then creates another delegate of that (ie, the 'layout' doc).
    // This is appropriate if you're trying to create a document that behaves like all
    // regularly created documents (e.g, text docs, pdfs, etc which all have data/layout docs)
    export function MakeDelegateWithProto(doc: Doc, id?: string, title?: string): Doc {
        const delegateProto = new Doc();
        delegateProto[Initializing] = true;
        delegateProto.proto = doc;
        delegateProto.author = Doc.CurrentUserEmail;
        delegateProto.isDataDoc = true;
        title && (delegateProto.title = title);
        const delegate = new Doc(id, true);
        delegate[Initializing] = true;
        delegate.proto = delegateProto;
        delegate.author = Doc.CurrentUserEmail;
        Doc.AddDocToList(delegateProto[DocData], 'proto_embeddings', delegate);
        delegate[Initializing] = false;
        delegateProto[Initializing] = false;
        return delegate;
    }

    let _applyCount: number = 0;
    export function ApplyTemplate(templateDoc: Doc) {
        if (templateDoc) {
            const proto = new Doc();
            proto.author = Doc.CurrentUserEmail;
            const target = Doc.MakeDelegate(proto);
            const targetKey = StrCast(templateDoc.layout_fieldKey, 'layout');
            const applied = ApplyTemplateTo(templateDoc, target, targetKey, templateDoc.title + '(...' + _applyCount++ + ')');
            target.layout_fieldKey = targetKey;
            applied && (Doc.GetProto(applied).type = templateDoc.type);
            return applied;
        }
        return undefined;
    }
    export function ApplyTemplateTo(templateDoc: Doc, target: Doc, targetKey: string, titleTarget: string | undefined) {
        if (!Doc.AreProtosEqual(target[targetKey] as Doc, templateDoc)) {
            if (target.resolvedDataDoc) {
                target[targetKey] = new PrefetchProxy(templateDoc);
            } else {
                titleTarget && (Doc.GetProto(target).title = titleTarget);
                const setDoc = [AclAdmin, AclEdit, AclAugment].includes(GetEffectiveAcl(Doc.GetProto(target))) ? Doc.GetProto(target) : target;
                setDoc[targetKey] = new PrefetchProxy(templateDoc);
            }
        }
        return target;
    }

    //
    //  This function converts a generic field layout display into a field layout that displays a specific
    // metadata field indicated by the title of the template field (not the default field that it was rendering)
    //
    export function MakeMetadataFieldTemplate(templateField: Doc, templateDoc: Opt<Doc>): boolean {
        // find the metadata field key that this template field doc will display (indicated by its title)
        const metadataFieldKey = StrCast(templateField.isTemplateForField) || StrCast(templateField.title).replace(/^-/, '');

        // update the original template to mark it as a template
        templateField.isTemplateForField = metadataFieldKey;
        templateField.title = metadataFieldKey;

        const templateFieldValue = templateField[metadataFieldKey] || templateField[Doc.LayoutFieldKey(templateField)];
        const templateCaptionValue = templateField.caption;
        // move any data that the template field had been rendering over to the template doc so that things will still be rendered
        // when the template field is adjusted to point to the new metadatafield key.
        // note 1: if the template field contained a list of documents, each of those documents will be converted to templates as well.
        // note 2: this will not overwrite any field that already exists on the template doc at the field key
        if (!templateDoc?.[metadataFieldKey] && templateFieldValue instanceof ObjectField) {
            Cast(templateFieldValue, listSpec(Doc), [])?.map(d => d instanceof Doc && MakeMetadataFieldTemplate(d, templateDoc));
            Doc.GetProto(templateField)[metadataFieldKey] = ObjectField.MakeCopy(templateFieldValue);
        }
        // get the layout string that the template uses to specify its layout
        const templateFieldLayoutString = StrCast(Doc.LayoutField(Doc.Layout(templateField)));

        // change it to render the target metadata field instead of what it was rendering before and assign it to the template field layout document.
        Doc.Layout(templateField).layout = templateFieldLayoutString.replace(/fieldKey={'[^']*'}/, `fieldKey={'${metadataFieldKey}'}`);

        return true;
    }

    // converts a document id to a url path on the server
    export function globalServerPath(doc: Doc | string = ''): string {
        return Utils.prepend('/doc/' + (doc instanceof Doc ? doc[Id] : doc));
    }
    // converts a document id to a url path on the server
    export function localServerPath(doc?: Doc): string {
        return '/doc/' + (doc ? doc[Id] : '');
    }

    export function GetBrushHighlightStatus(doc: Doc) {
        return Doc.IsHighlighted(doc) ? DocBrushStatus.highlighted : Doc.GetBrushStatus(doc);
    }
    export class DocBrush {
        BrushedDoc = new Set<Doc>();
        SearchMatchDoc: ObservableMap<Doc, { searchMatch: number }> = new ObservableMap();
        brushDoc = action((doc: Doc, unbrush: boolean) => {
            unbrush ? this.BrushedDoc.delete(doc) : this.BrushedDoc.add(doc);
            doc[Brushed] = !unbrush;
        });
    }
    export const brushManager = new DocBrush();

    export class UserDocData {
        @observable _user_doc: Doc = undefined!;
        @observable _sharing_doc: Doc = undefined!;
        @observable _searchQuery: string = '';
    }

    // the document containing the view layout information - will be the Document itself unless the Document has
    // a layout field or 'layout' is given.
    export function Layout(doc: Doc, layout?: Doc): Doc {
        const overrideLayout = layout && Cast(doc[`${StrCast(layout.isTemplateForField, 'data')}_layout[` + layout[Id] + ']'], Doc, null);
        return overrideLayout || doc[DocLayout] || doc;
    }
    export function SetLayout(doc: Doc, layout: Doc | string) {
        doc[StrCast(doc.layout_fieldKey, 'layout')] = layout;
    }
    export function LayoutField(doc: Doc) {
        return doc[StrCast(doc.layout_fieldKey, 'layout')];
    }
    export function LayoutFieldKey(doc: Doc): string {
        return StrCast(Doc.Layout(doc).layout).split("'")[1]; // bcz: TODO check on this .  used to always reference 'layout', now it uses the layout speicfied by the current layout_fieldKey
    }
    export function NativeAspect(doc: Doc, dataDoc?: Doc, useDim?: boolean) {
        return Doc.NativeWidth(doc, dataDoc, useDim) / (Doc.NativeHeight(doc, dataDoc, useDim) || 1);
    }
    export function NativeWidth(doc?: Doc, dataDoc?: Doc, useWidth?: boolean) {
        return !doc ? 0 : NumCast(doc._nativeWidth, NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_nativeWidth'], useWidth ? NumCast(doc._width) : 0));
    }
    export function NativeHeight(doc?: Doc, dataDoc?: Doc, useHeight?: boolean) {
        if (!doc) return 0;
        const nheight = (Doc.NativeWidth(doc, dataDoc, useHeight) / NumCast(doc._width)) * NumCast(doc._height); // divide before multiply to avoid floating point errrorin case nativewidth = width
        const dheight = NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_nativeHeight'], useHeight ? NumCast(doc._height) : 0);
        return NumCast(doc._nativeHeight, nheight || dheight);
    }
    export function SetNativeWidth(doc: Doc, width: number | undefined, fieldKey?: string) {
        doc[(fieldKey ?? Doc.LayoutFieldKey(doc)) + '_nativeWidth'] = width;
    }
    export function SetNativeHeight(doc: Doc, height: number | undefined, fieldKey?: string) {
        doc[(fieldKey ?? Doc.LayoutFieldKey(doc)) + '_nativeHeight'] = height;
    }

    const manager = new UserDocData();
    export function SearchQuery(): string {
        return manager._searchQuery;
    }
    export function SetSearchQuery(query: string) {
        runInAction(() => (manager._searchQuery = query));
    }
    export function UserDoc(): Doc {
        return manager._user_doc;
    }
    export function SharingDoc(): Doc {
        return Doc.MySharedDocs;
    }
    export function LinkDBDoc(): Doc {
        return Cast(Doc.UserDoc().myLinkDatabase, Doc, null);
    }
    export function SetUserDoc(doc: Doc) {
        return (manager._user_doc = doc);
    }

    const isSearchMatchCache = computedFn(function IsSearchMatch(doc: Doc) {
        return brushManager.SearchMatchDoc.has(doc) ? brushManager.SearchMatchDoc.get(doc) : brushManager.SearchMatchDoc.has(Doc.GetProto(doc)) ? brushManager.SearchMatchDoc.get(Doc.GetProto(doc)) : undefined;
    });
    export function IsSearchMatch(doc: Doc) {
        return isSearchMatchCache(doc);
    }
    export function IsSearchMatchUnmemoized(doc: Doc) {
        return brushManager.SearchMatchDoc.has(doc) ? brushManager.SearchMatchDoc.get(doc) : brushManager.SearchMatchDoc.has(Doc.GetProto(doc)) ? brushManager.SearchMatchDoc.get(Doc.GetProto(doc)) : undefined;
    }
    export function SetSearchMatch(doc: Doc, results: { searchMatch: number }) {
        if (doc && GetEffectiveAcl(doc) !== AclPrivate && GetEffectiveAcl(Doc.GetProto(doc)) !== AclPrivate) {
            brushManager.SearchMatchDoc.set(doc, results);
        }
        return doc;
    }
    export function SearchMatchNext(doc: Doc, backward: boolean) {
        if (!doc || GetEffectiveAcl(doc) === AclPrivate || GetEffectiveAcl(Doc.GetProto(doc)) === AclPrivate) return doc;
        const result = brushManager.SearchMatchDoc.get(doc);
        const num = Math.abs(result?.searchMatch || 0) + 1;
        runInAction(() => result && brushManager.SearchMatchDoc.set(doc, { searchMatch: backward ? -num : num }));
        return doc;
    }
    export function ClearSearchMatches() {
        brushManager.SearchMatchDoc.clear();
    }

    export enum DocBrushStatus {
        unbrushed = 0,
        protoBrushed = 1,
        selfBrushed = 2,
        highlighted = 3,
    }
    // returns 'how' a Doc has been brushed over - whether the document itself was brushed, it's prototype, or neither
    export function GetBrushStatus(doc: Doc) {
        if (!doc || GetEffectiveAcl(doc) === AclPrivate || GetEffectiveAcl(Doc.GetProto(doc)) === AclPrivate || doc.opacity === 0) return DocBrushStatus.unbrushed;
        return doc[Brushed] ? DocBrushStatus.selfBrushed : Doc.GetProto(doc)[Brushed] ? DocBrushStatus.protoBrushed : DocBrushStatus.unbrushed;
    }
    export function BrushDoc(doc: Doc, unbrush = false) {
        if (doc && GetEffectiveAcl(doc) !== AclPrivate && GetEffectiveAcl(Doc.GetProto(doc)) !== AclPrivate) {
            brushManager.brushDoc(doc, unbrush);
            brushManager.brushDoc(Doc.GetProto(doc), unbrush);
        }
        return doc;
    }
    export function UnBrushDoc(doc: Doc) {
        return BrushDoc(doc, true);
    }
    export function UnBrushAllDocs() {
        Array.from(brushManager.BrushedDoc).forEach(action(doc => (doc[Brushed] = false)));
    }

    let UnhighlightWatchers: (() => void)[] = [];
    export let UnhighlightTimer: any;
    export function AddUnHighlightWatcher(watcher: () => void) {
        if (UnhighlightTimer) {
            UnhighlightWatchers.push(watcher);
        } else watcher();
    }
    export function linkFollowUnhighlight() {
        clearTimeout(UnhighlightTimer);
        UnhighlightWatchers.forEach(watcher => watcher());
        UnhighlightWatchers.length = 0;
        highlightedDocs.forEach(doc => Doc.UnHighlightDoc(doc));
        document.removeEventListener('pointerdown', linkFollowUnhighlight);
    }
    export function linkFollowHighlight(destDoc: Doc | Doc[], dataAndDisplayDocs = true, presentation_effect?: Doc) {
        linkFollowUnhighlight();
        (destDoc instanceof Doc ? [destDoc] : destDoc).forEach(doc => Doc.HighlightDoc(doc, dataAndDisplayDocs, presentation_effect));
        document.removeEventListener('pointerdown', linkFollowUnhighlight);
        document.addEventListener('pointerdown', linkFollowUnhighlight);
        if (UnhighlightTimer) clearTimeout(UnhighlightTimer);
        UnhighlightTimer = window.setTimeout(() => {
            linkFollowUnhighlight();
            UnhighlightTimer = 0;
        }, 5000);
    }

    export var highlightedDocs = new ObservableSet<Doc>();
    export function IsHighlighted(doc: Doc) {
        if (!doc || GetEffectiveAcl(doc) === AclPrivate || GetEffectiveAcl(Doc.GetProto(doc)) === AclPrivate || doc.opacity === 0) return false;
        return doc[Highlight] || Doc.GetProto(doc)[Highlight];
    }
    export function HighlightDoc(doc: Doc, dataAndDisplayDocs = true, presentation_effect?: Doc) {
        runInAction(() => {
            highlightedDocs.add(doc);
            doc[Highlight] = true;
            doc[Animation] = presentation_effect;
            if (dataAndDisplayDocs) {
                highlightedDocs.add(Doc.GetProto(doc));
                Doc.GetProto(doc)[Highlight] = true;
            }
        });
    }
    /// if doc is defined, then it is unhighlighted, otherwise all highlighted docs are unhighlighted
    export function UnHighlightDoc(doc?: Doc) {
        runInAction(() => {
            (doc ? [doc] : Array.from(highlightedDocs)).forEach(doc => {
                highlightedDocs.delete(doc);
                highlightedDocs.delete(Doc.GetProto(doc));
                doc[Highlight] = Doc.GetProto(doc)[Highlight] = false;
                doc[Animation] = undefined;
            });
        });
    }

    export function getDocTemplate(doc?: Doc) {
        return !doc
            ? undefined
            : doc.isTemplateDoc
              ? doc
              : Cast(doc.dragFactory, Doc, null)?.isTemplateDoc
                ? doc.dragFactory
                : Cast(Doc.Layout(doc), Doc, null)?.isTemplateDoc
                  ? Cast(Doc.Layout(doc), Doc, null).resolvedDataDoc
                      ? Doc.Layout(doc).proto
                      : Doc.Layout(doc)
                  : undefined;
    }

    export function deiconifyView(doc: Doc) {
        StrCast(doc.layout_fieldKey).split('_')[1] === 'icon' && setNativeView(doc);
    }

    export function setNativeView(doc: any) {
        const prevLayout = StrCast(doc.layout_fieldKey).split('_')[1];
        const deiconify = prevLayout === 'icon' && StrCast(doc.deiconifyLayout) ? 'layout_' + StrCast(doc.deiconifyLayout) : '';
        prevLayout === 'icon' && (doc.deiconifyLayout = undefined);
        doc.layout_fieldKey = deiconify || 'layout';
    }
    export function setDocRangeFilter(container: Opt<Doc>, key: string, range?: readonly number[], modifiers?: 'remove') {
        if (!container) return;

        const childFiltersByRanges = Cast(container._childFiltersByRanges, listSpec('string'), []);

        for (let i = 0; i < childFiltersByRanges.length; i += 3) {
            if (childFiltersByRanges[i] === key) {
                childFiltersByRanges.splice(i, 3);
                break;
            }
        }
        if (range !== undefined) {
            childFiltersByRanges.push(key);
            childFiltersByRanges.push(range[0].toString());
            childFiltersByRanges.push(range[1].toString());
            container._childFiltersByRanges = new List<string>(childFiltersByRanges);
        }

        if (modifiers) {
            childFiltersByRanges.splice(0, 3);
            container._childFiltersByRanges = new List<string>(childFiltersByRanges);
        }
    }

    export const FilterSep = '::';
    export const FilterAny = '--any--';
    export const FilterNone = '--undefined--';

    // filters document in a container collection:
    // all documents with the specified value for the specified key are included/excluded
    // based on the modifiers :"check", "x", undefined
    export function setDocFilter(container: Opt<Doc>, key: string, value: any, modifiers: 'remove' | 'match' | 'check' | 'x' | 'exists' | 'unset', toggle?: boolean, fieldPrefix?: string, append: boolean = true) {
        if (!container) return;
        const filterField = '_' + (fieldPrefix ? fieldPrefix + '_' : '') + 'childFilters';
        const childFilters = StrListCast(container[filterField]);
        runInAction(() => {
            for (let i = 0; i < childFilters.length; i++) {
                const fields = childFilters[i].split(FilterSep); // split key:value:modifier
                if (fields[0] === key && (fields[1] === value.toString() || modifiers === 'match' || (fields[2] === 'match' && modifiers === 'remove'))) {
                    if (fields[2] === modifiers && modifiers && fields[1] === value.toString()) {
                        if (toggle) modifiers = 'remove';
                        else return;
                    }
                    childFilters.splice(i, 1);
                    container[filterField] = new List<string>(childFilters);
                    break;
                }
            }
            if (!childFilters.length && modifiers === 'match' && value === undefined) {
                container[filterField] = undefined;
            } else if (modifiers !== 'remove') {
                !append && (childFilters.length = 0);
                childFilters.push(key + FilterSep + value + FilterSep + modifiers);
                container[filterField] = new List<string>(childFilters);
            }
        });
    }
    export function readDocRangeFilter(doc: Doc, key: string) {
        const childFiltersByRanges = Cast(doc._childFiltersByRanges, listSpec('string'), []);
        for (let i = 0; i < childFiltersByRanges.length; i += 3) {
            if (childFiltersByRanges[i] === key) {
                return [Number(childFiltersByRanges[i + 1]), Number(childFiltersByRanges[i + 2])];
            }
        }
    }
    export function assignDocToField(doc: Doc, field: string, id: string) {
        DocServer.GetRefField(id).then(layout => layout instanceof Doc && (doc[field] = layout));
        return id;
    }

    export function toggleNativeDimensions(layoutDoc: Doc, contentScale: number, panelWidth: number, panelHeight: number) {
        runInAction(() => {
            if (Doc.NativeWidth(layoutDoc) || Doc.NativeHeight(layoutDoc)) {
                layoutDoc._freeform_scale = NumCast(layoutDoc._freeform_scale, 1) * contentScale;
                layoutDoc._nativeWidth = undefined;
                layoutDoc._nativeHeight = undefined;
            } else {
                layoutDoc._layout_autoHeight = false;
                if (!Doc.NativeWidth(layoutDoc)) {
                    layoutDoc._nativeWidth = NumCast(layoutDoc._width, panelWidth);
                    layoutDoc._nativeHeight = NumCast(layoutDoc._height, panelHeight);
                }
            }
        });
    }

    export function styleFromLayoutString(doc: Doc, props: any, scale: number) {
        const style: { [key: string]: any } = {};
        const divKeys = ['width', 'height', 'fontSize', 'transform', 'left', 'backgroundColor', 'left', 'right', 'top', 'bottom', 'pointerEvents', 'position'];
        const replacer = (match: any, expr: string, offset: any, string: any) => {
            // bcz: this executes a script to convert a property expression string:  { script }  into a value
            return ScriptField.MakeFunction(expr, { self: Doc.name, this: Doc.name, scale: 'number' })?.script.run({ this: doc, self: doc, scale }).result?.toString() ?? '';
        };
        divKeys.map((prop: string) => {
            const p = props[prop];
            typeof p === 'string' && (style[prop] = p?.replace(/{([^.'][^}']+)}/g, replacer));
        });
        return style;
    }

    export function Paste(docids: string[], clone: boolean, addDocument: (doc: Doc | Doc[]) => boolean, ptx?: number, pty?: number, newPoint?: number[]) {
        DocServer.GetRefFields(docids).then(async fieldlist => {
            const list = Array.from(Object.values(fieldlist))
                .map(d => DocCast(d))
                .filter(d => d);
            const docs = clone ? (await Promise.all(Doc.MakeClones(list, false))).map(res => res.clone) : list;
            if (ptx !== undefined && pty !== undefined && newPoint !== undefined) {
                const firstx = list.length ? NumCast(list[0].x) + ptx - newPoint[0] : 0;
                const firsty = list.length ? NumCast(list[0].y) + pty - newPoint[1] : 0;
                docs.forEach(doc => {
                    doc.x = NumCast(doc.x) - firstx;
                    doc.y = NumCast(doc.y) - firsty;
                });
            }
            undoable(addDocument, 'Paste Doc')(docs); // embedContainer gets set in addDocument
        });
    }

    // prettier-ignore
    export function toIcon(doc?: Doc, isOpen?: Opt<boolean>) {
        if (isOpen) return doc?.isFolder ? 'chevron-down' : 'folder-open';
        switch (StrCast(doc?.type)) {
            case DocumentType.IMG:  return 'image';
            case DocumentType.COMPARISON: return 'columns';
            case DocumentType.RTF: return 'sticky-note';
            case DocumentType.COL:
                if (doc?.isFolder) {
                    switch (doc.type_collection) {
                        default: return isOpen === false ? 'chevron-right' : 'question';
                    } // prettier-ignore
                }
                switch (doc?.type_collection) {
                    case CollectionViewType.Freeform :   return 'object-group';
                    case CollectionViewType.NoteTaking : return 'chalkboard';
                    case CollectionViewType.Schema :     return 'table-cells';
                    case CollectionViewType.Docking:     return 'solar-panel';
                    default:                             return 'folder';
                } // prettier-ignore
            case DocumentType.WEB: return 'globe-asia';
            case DocumentType.SCREENSHOT: return 'photo-video';
            case DocumentType.WEBCAM: return 'video';
            case DocumentType.AUDIO: return 'microphone';
            case DocumentType.BUTTON: return 'bolt';
            case DocumentType.PRES: return 'tv';
            case DocumentType.SCRIPTING: return 'terminal';
            case DocumentType.IMPORT: return 'cloud-upload-alt';
            case DocumentType.VID: return 'video';
            case DocumentType.INK: return 'pen-nib';
            case DocumentType.PDF: return 'file-pdf';
            case DocumentType.LINK: return 'link';
            case DocumentType.MAP: return 'map-marker-alt';
            case DocumentType.DATAVIZ: return 'chart-bar';
            case DocumentType.EQUATION: return 'calculator';
            case DocumentType.SIMULATION: return 'rocket';
            case DocumentType.CONFIG: return 'folder-closed';
        }
        return 'question';
    }

    ///
    // imports a previously exported zip file which contains a set of documents and their assets (eg, images, videos)
    // the 'remap' parameter determines whether the ids of the documents loaded should be kept as they were, or remapped to new ids
    // If they are not remapped, loading the file will overwrite any existing documents with those ids
    //
    export async function importDocument(file: File, remap = false) {
        const upload = Utils.prepend('/uploadDoc');
        const formData = new FormData();
        if (file) {
            formData.append('file', file);
            formData.append('remap', remap.toString());
            const response = await fetch(upload, { method: 'POST', body: formData });
            const json = await response.json();
            if (json !== 'error') {
                const docs = await DocServer.GetRefFields(json.docids as string[]);
                const doc = DocCast(await DocServer.GetRefField(json.id));
                const links = await DocServer.GetRefFields(json.linkids as string[]);
                Array.from(Object.keys(links))
                    .map(key => links[key])
                    .forEach(link => link instanceof Doc && LinkManager.Instance.addLink(link));
                return doc;
            }
        }
        return undefined;
    }

    export namespace Get {
        const primitives = ['string', 'number', 'boolean'];

        export interface JsonConversionOpts {
            data: any;
            title?: string;
            appendToExisting?: { targetDoc: Doc; fieldKey?: string };
            excludeEmptyObjects?: boolean;
        }

        const defaultKey = 'json';

        /**
         * This function takes any valid JSON(-like) data, i.e. parsed or unparsed, and at arbitrarily
         * deep levels of nesting, converts the data and structure into nested documents with the appropriate fields.
         *
         * After building a hierarchy within / below a top-level document, it then returns that top-level parent.
         *
         * If we've received a string, treat it like valid JSON and try to parse it into an object. If this fails, the
         * string is invalid JSON, so we should assume that the input is the result of a JSON.parse()
         * call that returned a regular string value to be stored as a Field.
         *
         * If we've received something other than a string, since the caller might also pass in the results of a
         * JSON.parse() call, valid input might be an object, an array (still typeof object), a boolean or a number.
         * Anything else (like a function, etc. passed in naively as any) is meaningless for this operation.
         *
         * All TS/JS objects get converted directly to documents, directly preserving the key value structure. Everything else,
         * lacking the key value structure, gets stored as a field in a wrapper document.
         *
         * @param data for convenience and flexibility, either a valid JSON string to be parsed,
         * or the result of any JSON.parse() call.
         * @param title an optional title to give to the highest parent document in the hierarchy.
         * If whether this function creates a new document or appendToExisting is specified and that document already has a title,
         * because this title field can be left undefined for the opposite behavior, including a title will overwrite the existing title.
         * @param appendToExisting **if specified**, there are two cases, both of which return the target document:
         *
         * 1) the json to be converted can be represented as a document, in which case the target document will act as the root
         * of the tree and receive all the conversion results as new fields on itself
         * 2) the json can't be represented as a document, in which case the function will assign the field-level conversion
         * results to either the specified key on the target document, or to its "json" key by default.
         *
         * If not specified, the function creates and returns a new entirely generic document (different from the Doc.Create calls)
         * to act as the root of the tree.
         *
         * One might choose to specify this field if you want to write to a document returned from a Document.Create function call,
         * say a TreeView document that will be rendered, not just an untyped, identityless doc that would otherwise be created
         * from a default call to new Doc.
         *
         * @param excludeEmptyObjects whether non-primitive objects (TypeScript objects and arrays) should be converted even
         * if they contain no data. By default, empty objects and arrays are ignored.
         */
        export function FromJson({ data, title, appendToExisting, excludeEmptyObjects }: JsonConversionOpts): Opt<Doc> {
            if (excludeEmptyObjects === undefined) {
                excludeEmptyObjects = true;
            }
            if (data === undefined || data === null || ![...primitives, 'object'].includes(typeof data)) {
                return undefined;
            }
            let resolved: any;
            try {
                resolved = JSON.parse(typeof data === 'string' ? data : JSON.stringify(data));
            } catch (e) {
                return undefined;
            }
            let output: Opt<Doc>;
            if (typeof resolved === 'object' && !(resolved instanceof Array)) {
                output = convertObject(resolved, excludeEmptyObjects, title, appendToExisting?.targetDoc);
            } else {
                // give the proper types to the data extracted from the JSON
                const result = toField(resolved, excludeEmptyObjects);
                if (appendToExisting) {
                    (output = appendToExisting.targetDoc)[appendToExisting.fieldKey || defaultKey] = result;
                } else {
                    (output = new Doc()).json = result;
                }
            }
            title && output && (output.title = title);
            return output;
        }

        /**
         * For each value of the object, recursively convert it to its appropriate field value
         * and store the field at the appropriate key in the document if it is not undefined
         * @param object the object to convert
         * @returns the object mapped from JSON to field values, where each mapping
         * might involve arbitrary recursion (since toField might itself call convertObject)
         */
        const convertObject = (object: any, excludeEmptyObjects: boolean, title?: string, target?: Doc): Opt<Doc> => {
            const hasEntries = Object.keys(object).length;
            if (hasEntries || !excludeEmptyObjects) {
                const resolved = target ?? new Doc();
                if (hasEntries) {
                    let result: Opt<Field>;
                    Object.keys(object).map(key => {
                        // if excludeEmptyObjects is true, any qualifying conversions from toField will
                        // be undefined, and thus the results that would have
                        // otherwise been empty (List or Doc)s will just not be written
                        if ((result = toField(object[key], excludeEmptyObjects, key))) {
                            resolved[key] = result;
                        }
                    });
                }
                title && (resolved.title = title);
                return resolved;
            }
        };

        /**
         * For each element in the list, recursively convert it to a document or other field
         * and push the field to the list if it is not undefined
         * @param list the list to convert
         * @returns the list mapped from JSON to field values, where each mapping
         * might involve arbitrary recursion (since toField might itself call convertList)
         */
        const convertList = (list: Array<any>, excludeEmptyObjects: boolean): Opt<List<Field>> => {
            const target = new List();
            let result: Opt<Field>;
            // if excludeEmptyObjects is true, any qualifying conversions from toField will
            // be undefined, and thus the results that would have
            // otherwise been empty (List or Doc)s will just not be written
            list.map(item => (result = toField(item, excludeEmptyObjects)) && target.push(result));
            if (target.length || !excludeEmptyObjects) {
                return target;
            }
        };

        const toField = (data: any, excludeEmptyObjects: boolean, title?: string): Opt<Field> => {
            if (data === null || data === undefined) {
                return undefined;
            }
            if (primitives.includes(typeof data)) {
                return data;
            }
            if (typeof data === 'object') {
                return data instanceof Array ? convertList(data, excludeEmptyObjects) : convertObject(data, excludeEmptyObjects, title, undefined);
            }
            throw new Error(`How did ${data} of type ${typeof data} end up in JSON?`);
        };
    }
}

export function IdToDoc(id: string) {
    return DocCast(DocServer.GetCachedRefField(id));
}
ScriptingGlobals.add(function idToDoc(id: string): any {
    return IdToDoc(id);
});
ScriptingGlobals.add(function renameEmbedding(doc: any) {
    return StrCast(Doc.GetProto(doc).title).replace(/\([0-9]*\)/, '') + `(${doc.proto_embeddingId})`;
});
ScriptingGlobals.add(function getProto(doc: any) {
    return Doc.GetProto(doc);
});
ScriptingGlobals.add(function getDocTemplate(doc?: any) {
    return Doc.getDocTemplate(doc);
});
ScriptingGlobals.add(function getEmbedding(doc: any) {
    return Doc.MakeEmbedding(doc);
});
ScriptingGlobals.add(function getCopy(doc: any, copyProto: any) {
    return doc.isTemplateDoc ? Doc.ApplyTemplate(doc) : Doc.MakeCopy(doc, copyProto);
});
ScriptingGlobals.add(function copyField(field: any) {
    return Field.Copy(field);
});
ScriptingGlobals.add(function docList(field: any) {
    return DocListCast(field);
});
ScriptingGlobals.add(function addDocToList(doc: Doc, field: string, added: Doc) {
    return Doc.AddDocToList(doc, field, added);
});
ScriptingGlobals.add(function setInPlace(doc: any, field: any, value: any) {
    return Doc.SetInPlace(doc, field, value, false);
});
ScriptingGlobals.add(function sameDocs(doc1: any, doc2: any) {
    return Doc.AreProtosEqual(doc1, doc2);
});
ScriptingGlobals.add(function assignDoc(doc: Doc, field: string, id: string) {
    return Doc.assignDocToField(doc, field, id);
});
ScriptingGlobals.add(function docCastAsync(doc: FieldResult): any {
    return Cast(doc, Doc);
});
ScriptingGlobals.add(function activePresentationItem() {
    const curPres = Doc.ActivePresentation;
    return curPres && DocListCast(curPres[Doc.LayoutFieldKey(curPres)])[NumCast(curPres._itemIndex)];
});
ScriptingGlobals.add(function setDocFilter(container: Doc, key: string, value: any, modifiers: 'match' | 'check' | 'x' | 'remove') {
    Doc.setDocFilter(container, key, value, modifiers);
});
ScriptingGlobals.add(function setDocRangeFilter(container: Doc, key: string, range: number[]) {
    Doc.setDocRangeFilter(container, key, range);
});