path: root/src/fields/Doc.ts

/* eslint-disable no-use-before-define */
import { action, computed, makeObservable, observable, ObservableMap, ObservableSet, runInAction } from 'mobx';
import { computedFn } from 'mobx-utils';
import { alias, map, serializable } from 'serializr';
import { DocServer } from '../client/DocServer';
import { CollectionViewType, DocumentType } from '../client/documents/DocumentTypes';
import { scriptingGlobal, ScriptingGlobals } from '../client/util/ScriptingGlobals';
import { afterDocDeserialize, autoObject, Deserializable, SerializationHelper } from '../client/util/SerializationHelper';
import { undoable, UndoManager } from '../client/util/UndoManager';
import { ClientUtils, incrementTitleCopy } from '../ClientUtils';
import {
    AclAdmin, AclAugment, AclEdit, AclPrivate, AclReadonly, Animation, AudioPlay, Brushed, CachedUpdates, DirectLinks,
    DocAcl, DocCss, DocData, DocLayout, DocViews, FieldKeys, FieldTuples, ForceServerWrite, Height, Highlight,
    Initializing, Self, SelfProxy, TransitionTimer, UpdatingFromServer, Width
} from './DocSymbols'; // prettier-ignore
import { Copy, FieldChanged, HandleUpdate, Id, Parent, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols';
import { InkTool } from './InkField';
import { List } from './List';
import { ObjectField, serverOpType } from './ObjectField';
import { PrefetchProxy, ProxyField } from './Proxy';
import { FieldId, RefField } from './RefField';
import { RichTextField } from './RichTextField';
import { listSpec } from './Schema';
import { ComputedField, ScriptField } from './ScriptField';
import { BoolCast, Cast, DocCast, FieldValue, NumCast, StrCast, ToConstructor, toList } from './Types';
import { containedFieldChangedHandler, deleteProperty, GetEffectiveAcl, getField, getter, makeEditable, makeReadOnly, setter, SharingPermissions } from './util';

export let ObjGetRefField: (id: string, force?: boolean) => Promise<Doc | undefined>;
export let ObjGetRefFields: (ids: string[]) => Promise<Map<string, Doc | undefined>>;

export function SetObjGetRefField(func: (id: string, force?: boolean) => Promise<Doc | undefined>) {
    ObjGetRefField = func;
}
export function SetObjGetRefFields(func: (ids: string[]) => Promise<Map<string, Doc | undefined>>) {
    ObjGetRefFields = func;
}
export const LinkedTo = '-linkedTo';
export namespace Field {
    /**
     * Converts a field to its equivalent input string in the key value box such that if the string
     * is entered into a keyValueBox it will create an equivalent field (except if showComputedValue is set).
     * @param doc doc containing key
     * @param key field key to display
     * @param showComputedValue whether copmuted function should display its value instead of its function
     * @returns string representation of the field
     */
    export function toKeyValueString(doc: Doc, key: string, showComputedValue?: boolean): string {
        const isOnDelegate = !Doc.IsDataProto(doc) && Object.keys(doc).includes(key.replace(/^_/, ''));
        const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
        const valFunc = (field: FieldType): string => {
            const res =
                field instanceof ComputedField && showComputedValue
                    ? field.value(doc)
                    : field instanceof ComputedField
                      ? `:=${field.script.originalScript.replace(/dashCallChat\(_setCacheResult_, this, `(.*)`\)/, '(($1))')}`
                      : field instanceof ScriptField
                        ? `$=${field.script.originalScript}`
                        : Field.toScriptString(field);
            const resStr = (res + '').replace(/^`(.*)`$/, '$1');
            return typeof field === 'string' && (+resStr).toString() !== resStr && !Array.from('+-*/.').some(k => Array.from(resStr).includes(k))
                ? resStr
                : (res + '') // adjust the key value string to be easier to enter: represent any initial list as an array with []
                      .trim()
                      .replace(/^new List\((.*)\)$/, '$1');
        };
        return !Field.IsField(cfield) ? (key.startsWith('_') ? '=' : '') : (isOnDelegate ? '=' : '') + valFunc(cfield);
    }
    export function toScriptString(field: FieldType) {
        switch (typeof field) {
            case 'string': if (field.startsWith('{"')) return `'${field}'`; // bcz: hack ... want to quote the string the right way. if there are nested "'s, then use ' instead of ".  In this case, test for the start of a JSON string of the format {"property": ... } and use outer 's instead of "s
                           return !field.includes('`') ? `\`${field}\`` : `"${field}"`;
            case 'number':
            case 'boolean':return String(field);
            default:       return field?.[ToScriptString]?.() ?? 'null';
        } // prettier-ignore
    }
    export function toJavascriptString(field: FieldType) {
        let rawjava = '';

        switch (typeof field) {
            case 'string':
            case 'number':
            case 'boolean':rawjava = String(field);
                           break;
            default:       rawjava = field?.[ToJavascriptString]?.() ?? '';
        } // prettier-ignore
        let script = rawjava;
        // this is a bit hacky, but we treat '^@' references to a published document
        // as a kind of macro to include the content of those documents
        Doc.MyPublishedDocs.forEach(doc => {
            const regexMultilineFlag = 'm';
            const regex = new RegExp(`^\\^${StrCast(doc.title).replace(/[()]*/g, '')}\\s`, regexMultilineFlag); // need to remove characters that can cause the regular expression to be invalid
            const sections = (Cast(doc.text, RichTextField, null)?.Text ?? '').split('--DOCDATA--');
            if (script.match(regex)) {
                script = script.replace(regex, sections[0]) + (sections.length > 1 ? sections[1] : '');
            }
        });
        return script;
    }
    export function toString(fieldIn: unknown) {
        const field = fieldIn as FieldType;
        if (typeof field === 'string' || typeof field === 'number' || typeof field === 'boolean') return String(field);
        return field?.[ToString]?.() || '';
    }
    export function IsField(field: unknown): field is FieldType;
    export function IsField(field: unknown, includeUndefined: true): field is FieldType | undefined;
    export function IsField(field: unknown, includeUndefined: boolean = false): field is FieldType | undefined {
        return ['string', 'number', 'boolean'].includes(typeof field) || field instanceof ObjectField || field instanceof RefField || (includeUndefined && field === undefined);
    }
    // eslint-disable-next-line @typescript-eslint/no-shadow
    export function Copy(field: unknown) {
        return field instanceof ObjectField ? ObjectField.MakeCopy(field) : (field as FieldType);
    }
    UndoManager.SetFieldPrinter(toString);
}
export type FieldType = number | string | boolean | ObjectField | RefField;
export type Opt<T> = T | undefined;
export type FieldWaiting<T extends RefField = RefField> = T extends undefined ? never : Promise<T | undefined>;
export type FieldResult<T extends FieldType = FieldType> = Opt<T> | FieldWaiting<Extract<T, RefField>>;

/**
 * Cast any field to either a List of Docs or undefined if the given field isn't a List of Docs.
 * If a default value is given, that will be returned instead of undefined.
 * If a default value is given, the returned value should not be modified as it might be a temporary value.
 * If no default value is given, and the returned value is not undefined, it can be safely modified.
 */
export function DocListCastAsync(field: FieldResult): Promise<Doc[] | undefined>;
export function DocListCastAsync(field: FieldResult, defaultValue: Doc[]): Promise<Doc[]>;
export function DocListCastAsync(field: FieldResult, defaultValue?: Doc[]) {
    const list = Cast(field, listSpec(Doc));
    return list ? Promise.all(list).then(() => list) : Promise.resolve(defaultValue);
}
export function NumListCast(field: FieldResult, defaultVal: number[] = []) {
    return Cast(field, listSpec('number'), defaultVal);
}
export function StrListCast(field: FieldResult, defaultVal: string[] = []) {
    return Cast(field, listSpec('string'), defaultVal);
}
export function DocListCast(field: FieldResult, defaultVal: Doc[] = []) {
    return Cast(field, listSpec(Doc), defaultVal).filter(d => d instanceof Doc) as Doc[];
}

export enum aclLevel {
    unset = -1,
    unshared = 0,
    viewable = 1,
    augmentable = 2,
    editable = 3,
    admin = 4,
}
// prettier-ignore
export const HierarchyMapping: Map<symbol, { level:aclLevel; name: SharingPermissions; image: string }> = new Map([
    [AclPrivate,  { level: aclLevel.unshared,     name: SharingPermissions.None,    image: '▲' }],
    [AclReadonly, { level: aclLevel.viewable,     name: SharingPermissions.View,    image: '♦' }],
    [AclAugment,  { level: aclLevel.augmentable,  name: SharingPermissions.Augment, image: '⬟' }],
    [AclEdit,     { level: aclLevel.editable,     name: SharingPermissions.Edit,    image: '⬢' }],
    [AclAdmin,    { level: aclLevel.admin,        name: SharingPermissions.Admin,   image: '⬢' }],
]);
export const ReverseHierarchyMap: Map<string, { level: aclLevel; acl: symbol; image: string }> = new Map(Array.from(HierarchyMapping.entries()).map(value => [value[1].name, { level: value[1].level, acl: value[0], image: value[1].image }]));

// caches the document access permissions for the current user.
// this recursively updates all protos as well.
export function updateCachedAcls(doc: Doc) {
    if (doc) {
        const target = doc[FieldTuples] ?? doc;
        const permissions: { [key: string]: symbol } = !target.author || target.author === ClientUtils.CurrentUserEmail() ? { acl_Me: AclAdmin } : {};
        Object.keys(target).forEach(key => {
            key.startsWith('acl_') && (permissions[key] = ReverseHierarchyMap.get(StrCast(target[key]))!.acl);
        });
        if (Object.keys(permissions).length || doc[DocAcl]?.length) {
            runInAction(() => {
                doc[DocAcl] = permissions;
            });
        }

        if (doc.proto instanceof Promise) {
            doc.proto.then(proto => updateCachedAcls(DocCast(proto)));
            return doc.proto;
        }
    }
    return undefined;
}

@scriptingGlobal
@Deserializable('Doc', (obj: unknown) => updateCachedAcls(obj as Doc), ['id'])
export class Doc extends RefField {
    @observable public static RecordingEvent = 0;
    @observable public static GuestDashboard: Doc | undefined = undefined;
    @observable public static GuestTarget: Doc | undefined = undefined;
    @observable.shallow public static CurrentlyLoading: Doc[] = observable([]);
    // DocServer api
    public static FindDocByTitle(title: string) {
        const foundDocId =
            title &&
            Array.from(Object.keys(DocServer.Cache()))
                .filter(key => DocServer.Cache()[key] instanceof Doc)
                .find(key => (DocServer.Cache()[key] as Doc).title === title);

        return foundDocId ? (DocServer.Cache()[foundDocId] as Doc) : undefined;
    }
    // removes from currently loading doc set
    public static removeCurrentlyLoading(doc: Doc) {
        if (Doc.CurrentlyLoading) {
            const index = Doc.CurrentlyLoading.indexOf(doc);
            runInAction(() => index !== -1 && Doc.CurrentlyLoading.splice(index, 1));
        }
    }
    // adds doc to currently loading display
    public static addCurrentlyLoading(doc: Doc) {
        if (Doc.CurrentlyLoading.indexOf(doc) === -1) {
            runInAction(() => Doc.CurrentlyLoading.push(doc));
        }
    }
    // LinkManager api
    public static AddLink: (link: Doc, checkExists?: boolean) => void;
    public static DeleteLink: (link: Doc) => void;
    public static Links: (link: Doc | undefined) => Doc[];
    public static getOppositeAnchor: (linkDoc: Doc, anchor: Doc) => Doc | undefined;
    // KeyValue SetField
    public static SetField: (doc: Doc, key: string, value: string, forceOnDelegate?: boolean, setResult?: (value: FieldResult) => void) => boolean;
    // UserDoc "API"
    public static get MySharedDocs()          { return DocCast(Doc.UserDoc().mySharedDocs); } // prettier-ignore
    public static get MyUserDocView()         { return DocCast(Doc.UserDoc().myUserDocView); } // prettier-ignore
    public static get MyDockedBtns()          { return DocCast(Doc.UserDoc().myDockedBtns); } // prettier-ignore
    public static get MySearcher()            { return DocCast(Doc.UserDoc().mySearcher); } // prettier-ignore
    public static get MyImageGrouper()        { return DocCast(Doc.UserDoc().myImageGrouper); } //prettier-ignore
    public static get MyFaceCollection()      { return DocCast(Doc.UserDoc().myFaceCollection); } //prettier-ignore
    public static get MyHeaderBar()           { return DocCast(Doc.UserDoc().myHeaderBar); } // prettier-ignore
    public static get MyLeftSidebarMenu()     { return DocCast(Doc.UserDoc().myLeftSidebarMenu); } // prettier-ignore
    public static get MyLeftSidebarPanel()    { return DocCast(Doc.UserDoc().myLeftSidebarPanel); } // prettier-ignore
    public static get MyContextMenuBtns()     { return DocCast(Doc.UserDoc().myContextMenuBtns); } // prettier-ignore
    public static get MyTopBarBtns()          { return DocCast(Doc.UserDoc().myTopBarBtns); } // prettier-ignore
    public static get MyRecentlyClosed()      { return DocCast(Doc.UserDoc().myRecentlyClosed); } // prettier-ignore
    public static get MyTrails()              { return DocCast(Doc.ActiveDashboard?.myTrails); } // prettier-ignore
    public static get MyCalendars()           { return DocCast(Doc.ActiveDashboard?.myCalendars); } // prettier-ignore
    public static get MyOverlayDocs()         { return DocListCast(Doc.ActiveDashboard?.myOverlayDocs ?? DocCast(Doc.UserDoc().myOverlayDocs)?.data); } // prettier-ignore
    public static get MyPublishedDocs()       { return DocListCast(Doc.ActiveDashboard?.myPublishedDocs).concat(DocListCast(DocCast(Doc.UserDoc().myPublishedDocs)?.data)); } // prettier-ignore
    public static get MyDashboards()          { return DocCast(Doc.UserDoc().myDashboards); } // prettier-ignore
    public static get MyTemplates()           { return DocCast(Doc.UserDoc().myTemplates); } // prettier-ignore
    public static get MyImports()             { return DocCast(Doc.UserDoc().myImports); } // prettier-ignore
    public static get MyFilesystem()          { return DocCast(Doc.UserDoc().myFilesystem); } // prettier-ignore
    public static get MyTools()               { return DocCast(Doc.UserDoc().myTools); } // prettier-ignore
    public static get noviceMode()            { return BoolCast(Doc.UserDoc().noviceMode);  } // prettier-ignore
    public static set noviceMode(val)         { Doc.UserDoc().noviceMode = val; } // prettier-ignore
    public static get IsSharingEnabled()      { return BoolCast(Doc.UserDoc().isSharingEnabled); } // prettier-ignore
    public static set IsSharingEnabled(val)   { Doc.UserDoc().isSharingEnabled = val; } // prettier-ignore
    public static get IsInfoUIDisabled()      { return BoolCast(Doc.UserDoc().isInfoUIDisabled); } // prettier-ignore
    public static set IsInfoUIDisabled(val)   { Doc.UserDoc().isInfoUIDisabled = val; } // prettier-ignore
    public static get defaultAclPrivate()     { return Doc.UserDoc().defaultAclPrivate; } // prettier-ignore
    public static set defaultAclPrivate(val)  { Doc.UserDoc().defaultAclPrivate = val; } // prettier-ignore
    public static get ActivePage()            { return StrCast(Doc.UserDoc().activePage); } // prettier-ignore
    public static set ActivePage(val)         { Doc.UserDoc().activePage = val; } // prettier-ignore
    public static get ActiveTool(): InkTool   { return StrCast(Doc.UserDoc().activeTool, InkTool.None) as InkTool;  } // prettier-ignore
    public static set ActiveTool(tool:InkTool){ Doc.UserDoc().activeTool = tool; } // prettier-ignore
    public static get ActivePresentation()    { return DocCast(Doc.ActiveDashboard?.activePresentation) as Opt<Doc>;  } // prettier-ignore
    public static set ActivePresentation(val) { Doc.ActiveDashboard && (Doc.ActiveDashboard.activePresentation = val) } // prettier-ignore
    public static get ActiveDashboard()       { return DocCast(Doc.UserDoc().activeDashboard); } // prettier-ignore
    public static set ActiveDashboard(val: Opt<Doc>) {  Doc.UserDoc().activeDashboard = val; } // prettier-ignore
    public static get MyFilterHotKeys()       { return DocListCast(DocCast(DocCast(Doc.UserDoc().myContextMenuBtns)?.Filter)?.data).filter(key => key.toolType !== "-opts-"); } // prettier-ignore
    public static RemFromFilterHotKeys(doc: Doc) {
        return Doc.RemoveDocFromList(DocCast(DocCast(Doc.UserDoc().myContextMenuBtns)?.Filter), 'data', doc);
    }
    public static AddToFilterHotKeys(doc: Doc) {
        return Doc.AddDocToList(DocCast(DocCast(Doc.UserDoc().myContextMenuBtns)?.Filter), 'data', doc);
    }
    public static IsInMyOverlay(doc: Doc)     { return Doc.MyOverlayDocs.includes(doc); } // prettier-ignore
    public static AddToMyOverlay(doc: Doc)    { return Doc.ActiveDashboard ? Doc.AddDocToList(Doc.ActiveDashboard, 'myOverlayDocs', doc) : Doc.AddDocToList(DocCast(Doc.UserDoc().myOverlayDocs), undefined, doc); } // prettier-ignore
    public static RemFromMyOverlay(doc: Doc)  { return Doc.ActiveDashboard ? Doc.RemoveDocFromList(Doc.ActiveDashboard,'myOverlayDocs', doc) : Doc.RemoveDocFromList(DocCast(Doc.UserDoc().myOverlayDocs), undefined, doc); } // prettier-ignore
    public static AddToMyPublished(doc: Doc)  {
        doc[DocData].title_custom = true;
        doc[DocData].layout_showTitle = 'title'; 
        Doc.ActiveDashboard ? Doc.AddDocToList(Doc.ActiveDashboard, 'myPublishedDocs', doc) : Doc.AddDocToList(DocCast(Doc.UserDoc().myPublishedDocs), undefined, doc); } // prettier-ignore
    public static RemFromMyPublished(doc: Doc){ 
        doc[DocData].title_custom = false;
        doc[DocData].layout_showTitle = undefined;
        Doc.ActiveDashboard ? Doc.RemoveDocFromList(Doc.ActiveDashboard,'myPublishedDocs', doc) : Doc.RemoveDocFromList(DocCast(Doc.UserDoc().myPublishedDocs), undefined, doc); } // prettier-ignore
    public static IsComicStyle(doc?: Doc)     { return doc && Doc.ActiveDashboard && !Doc.IsSystem(doc) && Doc.UserDoc().renderStyle === 'comic' ; } // prettier-ignore

    constructor(id?: FieldId, forceSave?: boolean) {
        super(id);
        makeObservable(this);
        const docProxy = new Proxy<this>(this, {
            set: setter,
            get: getter,
            // getPrototypeOf: (target) => Cast(target[SelfProxy].proto, Doc) || null, // TODO this might be able to replace the proto logic in getter
            has: (target, key) => GetEffectiveAcl(target) !== AclPrivate && key in target.__fieldTuples,
            ownKeys: target => {
                const keys = GetEffectiveAcl(target) !== AclPrivate ? Object.keys(target[FieldKeys]) : [];
                return [
                    ...keys,
                    AclAdmin,
                    AclAugment,
                    AclEdit,
                    AclPrivate,
                    AclReadonly,
                    Animation,
                    AudioPlay,
                    Brushed,
                    CachedUpdates,
                    DirectLinks,
                    DocAcl,
                    DocCss,
                    DocData,
                    DocLayout,
                    DocViews,
                    FieldKeys,
                    FieldTuples,
                    ForceServerWrite,
                    Height,
                    Highlight,
                    Initializing,
                    Self,
                    SelfProxy,
                    UpdatingFromServer,
                    Width,
                    '__LAYOUT__',
                ];
            },
            getOwnPropertyDescriptor: (target, prop) => {
                if (prop.toString() === '__LAYOUT__' || !(prop in target[FieldKeys])) {
                    return Reflect.getOwnPropertyDescriptor(target, prop);
                }
                return {
                    configurable: true, // TODO Should configurable be true?
                    enumerable: true,
                    value: 0, // () => target.__fieldTuples[prop])
                };
            },
            deleteProperty: deleteProperty,
            defineProperty: () => {
                throw new Error("Currently properties can't be defined on documents using Object.defineProperty");
            },
        });
        this[SelfProxy] = docProxy;
        if (!id || forceSave) {
            DocServer.CreateDocField(docProxy);
        }
        return docProxy; // need to return the proxy from the constructor so that all our added fields will get called
    }

    [key: string]: FieldResult;
    [key2: symbol]: unknown;

    @serializable(alias('fields', map(autoObject(), { afterDeserialize: afterDocDeserialize })))
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    get __fieldTuples(): any {
        // __fieldTuples does not follow the index signature pattern which requires a FieldResult return value -- so this hack suppresses the error
        return this[FieldTuples];
    }
    set __fieldTuples(value) {
        // called by deserializer to set all fields in one shot
        this[FieldTuples] = value;
        Object.keys(value).forEach(key => {
            if (Object.prototype.hasOwnProperty.call(value, key)) {
                const field = value[key];
                field !== undefined && (this[FieldKeys][key] = true);
                if (field instanceof ObjectField) {
                    field[Parent] = this[Self];
                    field[FieldChanged] = containedFieldChangedHandler(this[SelfProxy], key, field);
                }
            }
        });
    }

    @observable private [FieldTuples]: { [key: string]: FieldResult } = {};
    @observable private [FieldKeys]: { [key: string]: boolean } = {};
    /// all of the raw acl's that have been set on this document.  Use GetEffectiveAcl to determine the actual ACL of the doc for editing
    @observable public [DocAcl]: { [key: string]: symbol } = {};
    @observable public [DocCss]: number = 0; // incrementer denoting a change to CSS layout
    @observable public [DirectLinks] = new ObservableSet<Doc>();
    @observable public [AudioPlay]: unknown = undefined; // meant to store sound object from Howl
    @observable public [Animation]: Opt<Doc> = undefined;
    @observable public [Highlight]: boolean = false;
    @observable public [Brushed]: boolean = false;
    @observable public [DocViews] = new ObservableSet<unknown /* DocumentView */>();

    private [Self] = this;
    private [SelfProxy]: Doc;
    private [UpdatingFromServer]: boolean = false;
    private [ForceServerWrite]: boolean = false;
    private [CachedUpdates]: { [key: string]: () => void | Promise<void> } = {};

    public [Initializing]: boolean = false;
    public [FieldChanged] = (diff: { op: '$addToSet' | '$remFromSet' | '$set'; items: FieldType[] | undefined; length: number | undefined; hint?: { start: number; deleteCount: number } } | undefined, serverOp: serverOpType) => {
        if (!this[UpdatingFromServer] || this[ForceServerWrite]) {
            DocServer.UpdateField(this[Id], serverOp);
        }
    };
    public [Width] = () => NumCast(this[SelfProxy]._width);
    public [Height] = () => NumCast(this[SelfProxy]._height);
    public [TransitionTimer]: NodeJS.Timeout | undefined = undefined;
    public [ToJavascriptString] = () => `idToDoc("${this[Self][Id]}")`; // what should go here?
    public [ToScriptString] = () => `idToDoc("${this[Self][Id]}")`;
    public [ToString] = () => `Doc(${GetEffectiveAcl(this[SelfProxy]) === AclPrivate ? '-inaccessible-' : this[SelfProxy].title})`;
    public get [DocLayout]() { return this[SelfProxy].__LAYOUT__; } // prettier-ignore
    public get [DocData](): Doc {
        const self = this[SelfProxy];
        return self.resolvedDataDoc && !self.isTemplateForField ? self : Doc.GetProto(Cast(Doc.Layout(self).resolvedDataDoc, Doc, null) || self);
    }
    @computed get __LAYOUT__(): Doc | undefined {
        const self = this[SelfProxy];
        const templateLayoutDoc = Cast(Doc.LayoutField(self), Doc, null);
        if (templateLayoutDoc) {
            let renderFieldKey: string = '';
            const layoutField = templateLayoutDoc[StrCast(templateLayoutDoc.layout_fieldKey, 'layout')];
            if (typeof layoutField === 'string') {
                [renderFieldKey] = layoutField.split("fieldKey={'")[1].split("'"); // layoutField.split("'")[1];
            } else {
                return Cast(layoutField, Doc, null);
            }
            return Cast(self[renderFieldKey + '_layout[' + templateLayoutDoc[Id] + ']'], Doc, null) || templateLayoutDoc;
        }
        return undefined;
    }

    public async [HandleUpdate](diff: { $set: { [key: string]: FieldType } } | { $unset?: unknown }) {
        const $set = '$set' in diff ? diff.$set : undefined;
        const $unset = '$unset' in diff ? diff.$unset : undefined;
        const sameAuthor = this.author === ClientUtils.CurrentUserEmail();
        const fprefix = 'fields.';
        Object.keys($set ?? {})
            .filter(key => key.startsWith(fprefix))
            .forEach(async key => {
                const fKey = key.substring(fprefix.length);
                const fn = async () => {
                    const value = (await SerializationHelper.Deserialize($set?.[key])) as FieldType;
                    const prev = GetEffectiveAcl(this);
                    this[UpdatingFromServer] = true;
                    this[fKey] = value;
                    this[UpdatingFromServer] = false;
                    if (fKey.startsWith('acl_')) {
                        updateCachedAcls(this);
                    }
                    if (prev === AclPrivate && GetEffectiveAcl(this) !== AclPrivate) {
                        DocServer.GetRefField(this[Id], true);
                    }
                };
                const writeMode = DocServer.getFieldWriteMode(fKey);
                if (fKey.startsWith('acl_') || writeMode !== DocServer.WriteMode.Playground) {
                    delete this[CachedUpdates][fKey];
                    await fn();
                } else {
                    this[CachedUpdates][fKey] = fn;
                }
            });
        Object.keys($unset ?? {})
            .filter(key => key.startsWith(fprefix))
            .forEach(async key => {
                const fKey = key.substring(7);
                const fn = () => {
                    this[UpdatingFromServer] = true;
                    delete this[fKey];
                    this[UpdatingFromServer] = false;
                };
                if (sameAuthor || DocServer.getFieldWriteMode(fKey) !== DocServer.WriteMode.Playground) {
                    delete this[CachedUpdates][fKey];
                    await fn();
                } else {
                    this[CachedUpdates][fKey] = fn;
                }
            });
    }
}

export namespace Doc {
    export let SelectOnLoad: Doc | undefined;
    export function SetSelectOnLoad(doc: Doc | undefined) {
        SelectOnLoad = doc;
    }
    export let DocDragDataName: string = '';
    export function SetDocDragDataName(name: string) {
        DocDragDataName = name;
    }
    export function SetContainer(doc: Doc, container: Doc) {
        if (container !== Doc.MyRecentlyClosed) {
            doc.embedContainer = container;
            Doc.AddEmbedding(doc, doc);
        }
    }
    export function RunCachedUpdate(doc: Doc, field: string) {
        const update = doc[CachedUpdates][field];
        if (update) {
            update();
            delete doc[CachedUpdates][field];
        }
    }
    export function AddCachedUpdate(doc: Doc, field: string, oldValue: FieldType) {
        const val = oldValue;
        doc[CachedUpdates][field] = () => {
            doc[UpdatingFromServer] = true;
            doc[field] = val;
            doc[UpdatingFromServer] = false;
        };
    }
    export function MakeReadOnly(): { end(): void } {
        makeReadOnly();
        return {
            end() {
                makeEditable();
            },
        };
    }

    export function Get(doc: Doc, key: string, ignoreProto: boolean = false): FieldResult {
        try {
            return getField(doc[Self], key, ignoreProto) as FieldResult;
        } catch {
            return doc;
        }
    }
    export function GetT<T extends FieldType>(doc: Doc, key: string, ctor: ToConstructor<T>, ignoreProto: boolean = false): FieldResult<T> {
        return Cast(Get(doc, key, ignoreProto), ctor) as FieldResult<T>;
    }
    export function isTemplateDoc(doc: Doc) {
        return GetT(doc, 'isTemplateDoc', 'boolean', true);
    }
    export function isTemplateForField(doc: Doc) {
        return GetT(doc, 'isTemplateForField', 'string', true);
    }
    export function IsDataProto(doc: Doc) {
        return GetT(doc, 'isDataDoc', 'boolean', true);
    }
    export function IsBaseProto(doc: Doc) {
        return GetT(doc, 'isBaseProto', 'boolean', true);
    }
    export function IsSystem(doc: Doc) {
        return GetT(doc, 'isSystem', 'boolean', true);
    }
    export function IsDelegateField(doc: Doc, fieldKey: string) {
        return doc && Get(doc, fieldKey, true) !== undefined;
    }
    //
    // this will write the value to the key on either the data doc or the embedding doc.  The choice
    // of where to write it is based on:
    // 1) if the embedding Doc already has this field defined on it, then it will be written to the embedding
    // 2) if the data doc has the field, then it's written there.
    // 3) if neither already has the field, then 'defaultProto' determines whether to write it to the data doc (or the embedding)
    //
    export async function SetInPlace(doc: Doc, keyIn: string, value: FieldType | undefined, defaultProto: boolean) {
        const key = keyIn.startsWith('_') ? keyIn.substring(1) : keyIn;
        const hasProto = doc[DocData] !== doc ? doc[DocData] : undefined;
        const onDeleg = Object.getOwnPropertyNames(doc).indexOf(key) !== -1;
        const onProto = hasProto && Object.getOwnPropertyNames(hasProto).indexOf(key) !== -1;
        if (onDeleg || !hasProto || (!onProto && !defaultProto)) {
            doc[key] = value;
        } else hasProto[key] = value;
    }
    export function GetAllPrototypes(doc: Doc): Doc[] {
        const protos: Doc[] = [];
        let d: Opt<Doc> = doc;
        while (d) {
            protos.push(d);
            d = DocCast(FieldValue(d.proto));
        }
        return protos;
    }
    /**
     * This function is intended to model Object.assign({}, {}) [https://mzl.la/1Mo3l21], which copies
     * the values of the properties of a source object into the target.
     *
     * This is just a specific, Dash-authored version that serves the same role for our
     * Doc class.
     *
     * @param doc the target document into which you'd like to insert the new fields
     * @param fields the fields to project onto the target. Its type signature defines a mapping from some string key
     * to a potentially undefined field, where each entry in this mapping is optional.
     */
    export function assign<K extends string>(doc: Doc, fields: Partial<Record<K, Opt<FieldType>>>, skipUndefineds: boolean = false, isInitializing = false) {
        isInitializing && (doc[Initializing] = true);
        Object.keys(fields).forEach(key => {
            const value = (fields as { [key: string]: Opt<FieldType> })[key];
            if (!skipUndefineds || value !== undefined) {
                // Do we want to filter out undefineds?
                if (typeof value === 'object' && 'values' in value) {
                    console.log(value);
                }
                doc[key] = value;
            }
        });
        isInitializing && (doc[Initializing] = false);
        return doc;
    }

    // compare whether documents or their protos match
    export function AreProtosEqual(doc?: Doc, other?: Doc) {
        return doc && other && (doc === other || Doc.GetProto(doc) === Doc.GetProto(other));
    }

    // Gets the data document for the document.  Note: this is mis-named -- it does not specifically
    // return the doc's proto, but rather recursively searches through the proto inheritance chain
    // and returns the document who's proto is undefined or whose proto is marked as a data doc ('isDataDoc').
    export function GetProto(doc: Doc): Doc {
        const proto = doc && (Doc.GetT(doc, 'isDataDoc', 'boolean', true) ? doc : DocCast(doc.proto, doc));
        return proto === doc ? proto : Doc.GetProto(proto);
    }
    export function GetDataDoc(doc: Doc): Doc {
        const proto = Doc.GetProto(doc);
        return proto === doc ? proto : Doc.GetDataDoc(proto);
    }

    export function allKeys(doc: Doc): string[] {
        const results: Set<string> = new Set();

        let proto: Doc | undefined = doc;
        while (proto) {
            Object.keys(proto).forEach(key => results.add(key));
            proto = DocCast(FieldValue(proto.proto));
        }

        return Array.from(results);
    }

    /**
     * @returns the index of doc toFind in list of docs, -1 otherwise
     */
    export function IndexOf(toFind: Doc, list: Doc[]) {
        const index = list.indexOf(toFind);
        return index !== -1 ? index : list.findIndex(doc => Doc.AreProtosEqual(doc, toFind));
    }

    /**
     * Removes doc from the list of Docs at listDoc[fieldKey]
     * @returns true if successful, false otherwise.
     */
    export function RemoveDocFromList(listDoc: Doc, fieldKey: string | undefined, doc: Doc, ignoreProto = false) {
        const key = fieldKey || Doc.LayoutFieldKey(listDoc);
        const list = Doc.Get(listDoc, key, ignoreProto) === undefined ? (listDoc[DocData][key] = new List<Doc>()) : Cast(listDoc[key], listSpec(Doc));
        if (list) {
            const ind = list.indexOf(doc);
            if (ind !== -1) {
                list.splice(ind, 1);
                return true;
            }
        }
        return false;
    }

    /**
     * Adds doc to the list of Docs stored at listDoc[fieldKey].
     * @returns true if successful, false otherwise.
     */
    export function AddDocToList(listDoc: Doc, fieldKey: string | undefined, doc: Doc, relativeTo?: Doc, before?: boolean, first?: boolean, allowDuplicates?: boolean, reversed?: boolean, ignoreProto?: boolean) {
        const key = fieldKey || Doc.LayoutFieldKey(listDoc);
        const list = Doc.Get(listDoc, key, ignoreProto) === undefined ? (listDoc[DocData][key] = new List<Doc>()) : Cast(listDoc[key], listSpec(Doc));
        if (list) {
            if (!allowDuplicates) {
                const pind = list.findIndex(d => d instanceof Doc && d[Id] === doc[Id]);
                if (pind !== -1) {
                    return true;
                }
            }
            if (first) {
                list.splice(0, 0, doc);
            } else {
                const ind = relativeTo ? list.indexOf(relativeTo) : -1;
                if (ind === -1) {
                    if (reversed) list.splice(0, 0, doc);
                    else list.push(doc);
                } else {
                    if (reversed) list.splice(before ? list.length - ind + 1 : list.length - ind, 0, doc);
                    else list.splice(before ? ind : ind + 1, 0, doc);
                }
            }
            return true;
        }
        return false;
    }

    export function RemoveEmbedding(doc: Doc, embedding: Doc) {
        Doc.RemoveDocFromList(doc[DocData], 'proto_embeddings', embedding);
    }
    export function AddEmbedding(doc: Doc, embedding: Doc) {
        if (embedding === null) {
            console.log('WHAT?');
        }
        Doc.AddDocToList(doc[DocData], 'proto_embeddings', embedding, undefined, undefined, undefined, undefined, undefined, true);
    }
    export function GetEmbeddings(doc: Doc) {
        return DocListCast(Doc.Get(doc[DocData], 'proto_embeddings', true));
    }

    export function MakeEmbedding(doc: Doc, id?: string) {
        const embedding = (!GetT(doc, 'isDataDoc', 'boolean', true) && doc.proto) || doc.type === DocumentType.CONFIG ? Doc.MakeCopy(doc, undefined, id) : Doc.MakeDelegate(doc, id);
        const layout = Doc.LayoutField(embedding);
        if (layout instanceof Doc && layout !== embedding && layout === Doc.Layout(embedding)) {
            Doc.SetLayout(embedding, Doc.MakeEmbedding(layout));
        }
        embedding.createdFrom = doc;
        embedding.proto_embeddingId = doc[DocData].proto_embeddingId = Doc.GetEmbeddings(doc).length - 1;
        !Doc.GetT(embedding, 'title', 'string', true) && (embedding.title = ComputedField.MakeFunction(`renameEmbedding(this)`));
        embedding.author = ClientUtils.CurrentUserEmail();

        return embedding;
    }

    export function BestEmbedding(doc: Doc) {
        const dataDoc = doc[DocData];
        const availableEmbeddings = Doc.GetEmbeddings(dataDoc);
        const bestEmbedding = [...(dataDoc !== doc ? [doc] : []), ...availableEmbeddings].find(d => !d.embedContainer && d.author === ClientUtils.CurrentUserEmail());
        bestEmbedding && Doc.AddEmbedding(doc, doc);
        return bestEmbedding ?? Doc.MakeEmbedding(doc);
    }

    // this lists out all the tag ids that can be  in a RichTextField that might contain document ids.
    // if a document is cloned, we need to make sure to clone all of these referenced documents as well;
    export const DocsInTextFieldIds = ['audioId', 'textId', 'anchorId', 'docId'];
    export async function makeClone(
        doc: Doc,
        cloneMap: Map<string, Doc>,
        linkMap: Map<string, Doc>,
        rtfs: { copy: Doc; key: string; field: RichTextField }[],
        exclusions: string[],
        pruneDocs: Doc[],
        cloneLinks: boolean,
        cloneTemplates: boolean
    ): Promise<Doc> {
        if (Doc.IsBaseProto(doc) || ((Doc.isTemplateDoc(doc) || Doc.isTemplateForField(doc)) && !cloneTemplates)) {
            return doc;
        }
        if (cloneMap.get(doc[Id])) return cloneMap.get(doc[Id])!;
        const copy = new Doc(undefined, true);
        cloneMap.set(doc[Id], copy);
        const filter = [...exclusions, ...StrListCast(doc.cloneFieldFilter)];
        await Promise.all(
            Object.keys(doc).map(async key => {
                if (filter.includes(key)) return;
                const assignKey = (val: Opt<FieldType>) => {
                    copy[key] = val;
                };
                const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
                const field = ProxyField.WithoutProxy(() => doc[key]);
                const copyObjectField = async (objField: ObjectField) => {
                    const list = await Cast(doc[key], listSpec(Doc));
                    const docs = list && (await DocListCastAsync(list))?.filter(d => d instanceof Doc);
                    if (docs !== undefined && docs.length) {
                        const clones = await Promise.all(docs.map(async d => Doc.makeClone(d, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks, cloneTemplates)));
                        assignKey(new List<Doc>(clones));
                    } else {
                        assignKey(ObjectField.MakeCopy(objField));
                        if (objField instanceof RichTextField) {
                            if (DocsInTextFieldIds.some(id => objField.Data.includes(`"${id}":`))) {
                                const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => '(' + exp + ')').join('|') + ')":"([a-z-A-Z0-9_]*)"', 'g');
                                const rawdocids = objField.Data.match(docidsearch);
                                const docids = rawdocids?.map((str: string) =>
                                    DocsInTextFieldIds.reduce((output, exp) => output.replace(new RegExp(`${exp}":`, 'g'), ''), str)
                                        .replace(/"/g, '')
                                        .trim()
                                );
                                const results = docids && (await DocServer.GetRefFields(docids));
                                const rdocs = results && Array.from(Object.keys(results)).map(rkey => DocCast(results.get(rkey)));
                                rdocs?.map(d => d && Doc.makeClone(d, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks, cloneTemplates));
                                rtfs.push({ copy, key, field: objField });
                            }
                        }
                    }
                };
                const docAtKey = doc[key];
                if (key === 'author') {
                    assignKey(ClientUtils.CurrentUserEmail());
                } else if (docAtKey instanceof Doc) {
                    if (pruneDocs.includes(docAtKey)) {
                        // prune doc and do nothing
                    } else if (
                        !Doc.IsSystem(docAtKey) &&
                        (key.includes('layout[') ||
                            key.startsWith('layout') || //
                            ['embedContainer', 'annotationOn', 'proto'].includes(key) ||
                            (['link_anchor_1', 'link_anchor_2'].includes(key) && doc.author === ClientUtils.CurrentUserEmail()))
                    ) {
                        assignKey(await Doc.makeClone(docAtKey, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks, cloneTemplates));
                    } else {
                        assignKey(docAtKey);
                    }
                } else if (field instanceof RefField) {
                    assignKey(field);
                } else if (cfield instanceof ComputedField) {
                    assignKey(cfield[Copy]());
                } else if (field instanceof ObjectField) {
                    await copyObjectField(field);
                } else if (field instanceof Promise) {
                    // eslint-disable-next-line no-debugger
                    debugger; // This shouldn't happen...
                } else {
                    assignKey(field);
                }
            })
        );
        Array.from(doc[DirectLinks]).forEach(async link => {
            if (
                cloneLinks ||
                ((cloneMap.has(DocCast(link.link_anchor_1)?.[Id]) || cloneMap.has(DocCast(DocCast(link.link_anchor_1)?.annotationOn)?.[Id])) &&
                    (cloneMap.has(DocCast(link.link_anchor_2)?.[Id]) || cloneMap.has(DocCast(DocCast(link.link_anchor_2)?.annotationOn)?.[Id])))
            ) {
                linkMap.set(link[Id], await Doc.makeClone(link, cloneMap, linkMap, rtfs, exclusions, pruneDocs, cloneLinks, cloneTemplates));
            }
        });
        copy.cloneOf = doc;
        const cfield = ComputedField.WithoutComputed(() => FieldValue(doc.title));
        if (Doc.Get(copy, 'title', true) && !(cfield instanceof ComputedField)) copy.title = '>:' + doc.title;
        cloneMap.set(doc[Id], copy);

        return copy;
    }
    export function repairClone(clone: Doc, cloneMap: Map<string, Doc>, cloneTemplates: boolean, visited: Set<Doc>) {
        if (visited.has(clone)) return;
        visited.add(clone);
        Object.keys(clone)
            .filter(key => key !== 'cloneOf')
            .forEach(key => {
                const docAtKey = DocCast(clone[key]);
                if (docAtKey && !Doc.IsSystem(docAtKey)) {
                    if (!Array.from(cloneMap.values()).includes(docAtKey)) {
                        clone[key] = !cloneTemplates && (Doc.isTemplateDoc(docAtKey) || Doc.isTemplateForField(docAtKey)) ? docAtKey : cloneMap.get(docAtKey[Id]);
                    } else {
                        repairClone(docAtKey, cloneMap, cloneTemplates, visited);
                    }
                }
            });
    }
    export function MakeClones(docs: Doc[], cloneLinks: boolean, cloneTemplates: boolean) {
        const cloneMap = new Map<string, Doc>();
        return docs.map(doc => Doc.MakeClone(doc, cloneLinks, cloneTemplates, cloneMap));
    }

    export async function MakeClone(doc: Doc, cloneLinks = true, cloneTemplates = true, cloneMap: Map<string, Doc> = new Map()) {
        const linkMap = new Map<string, Doc>();
        const rtfMap: { copy: Doc; key: string; field: RichTextField }[] = [];
        const clone = await Doc.makeClone(doc, cloneMap, linkMap, rtfMap, ['cloneOf'], doc.embedContainer ? [DocCast(doc.embedContainer)] : [], cloneLinks, cloneTemplates);
        const repaired = new Set<Doc>();
        const linkedDocs = Array.from(linkMap.values());
        linkedDocs.forEach(link => Doc.AddLink?.(link, true));
        rtfMap.forEach(({ copy, key, field }) => {
            const replacer = (match: string, attr: string, id: string /* , offset: any, string: any */) => {
                const mapped = cloneMap.get(id);
                return attr + '"' + (mapped ? mapped[Id] : id) + '"';
            };
            const replacer2 = (match: string, href: string, id: string /* , offset: any, string: any */) => {
                const mapped = cloneMap.get(id);
                return href + (mapped ? mapped[Id] : id);
            };
            const re = new RegExp(`(${Doc.localServerPath()})([^"]*)`, 'g');
            const docidsearch = new RegExp('(' + DocsInTextFieldIds.map(exp => `"${exp}":`).join('|') + ')"([^"]+)"', 'g');
            copy[key] = new RichTextField(field.Data.replace(docidsearch, replacer).replace(re, replacer2), field.Text);
        });
        const clonedDocs = [...Array.from(cloneMap.values()), ...linkedDocs];
        clonedDocs.forEach(cloneDoc => Doc.repairClone(cloneDoc, cloneMap, cloneTemplates, repaired));
        return { clone, map: cloneMap, linkMap };
    }

    const _pendingMap = new Set<string>();
    //
    // Returns an expanded template layout for a target data document if there is a template relationship
    // between the two. If so, the layoutDoc is expanded into a new document that inherits the properties
    // of the original layout while allowing for individual layout properties to be overridden in the expanded layout.
    export function expandTemplateLayout(templateLayoutDoc: Doc, targetDoc?: Doc) {
        // nothing to do if the layout isn't a template or we don't have a target that's different than the template
        if (!targetDoc || templateLayoutDoc === targetDoc || (!Doc.isTemplateForField(templateLayoutDoc) && !Doc.isTemplateDoc(templateLayoutDoc))) {
            return templateLayoutDoc;
        }

        const templateField = StrCast(templateLayoutDoc.isTemplateForField, Doc.LayoutFieldKey(templateLayoutDoc)); // the field that the template renders
        // First it checks if an expanded layout already exists -- if so it will be stored on the dataDoc
        // using the template layout doc's id as the field key.
        // If it doesn't find the expanded layout, then it makes a delegate of the template layout and
        // saves it on the data doc indexed by the template layout's id.
        //
        const expandedLayoutFieldKey = templateField + '_layout[' + templateLayoutDoc[Id] + ']';
        let expandedTemplateLayout = targetDoc?.[expandedLayoutFieldKey];

        if (templateLayoutDoc.resolvedDataDoc instanceof Promise) {
            expandedTemplateLayout = undefined;
            _pendingMap.add(targetDoc[Id] + expandedLayoutFieldKey);
        } else if (expandedTemplateLayout === undefined && !_pendingMap.has(targetDoc[Id] + expandedLayoutFieldKey)) {
            if (templateLayoutDoc.resolvedDataDoc === targetDoc[DocData]) {
                expandedTemplateLayout = templateLayoutDoc; // reuse an existing template layout if its for the same document with the same params
            } else {
                // eslint-disable-next-line no-param-reassign
                templateLayoutDoc.resolvedDataDoc && (templateLayoutDoc = DocCast(templateLayoutDoc.proto, templateLayoutDoc)); // if the template has already been applied (ie, a nested template), then use the template's prototype
                if (!targetDoc[expandedLayoutFieldKey]) {
                    _pendingMap.add(targetDoc[Id] + expandedLayoutFieldKey);
                    setTimeout(
                        action(() => {
                            const newLayoutDoc = Doc.MakeDelegate(templateLayoutDoc, undefined, '[' + templateLayoutDoc.title + ']');
                            const dataDoc = Doc.GetProto(targetDoc);
                            newLayoutDoc.rootDocument = targetDoc;
                            newLayoutDoc.embedContainer = targetDoc;
                            newLayoutDoc.resolvedDataDoc = dataDoc;
                            newLayoutDoc.acl_Guest = SharingPermissions.Edit;
                            if (dataDoc[templateField] === undefined && (templateLayoutDoc[templateField] as List<Doc>)?.length) {
                                dataDoc[templateField] = ObjectField.MakeCopy(templateLayoutDoc[templateField] as List<Doc>);
                                // ComputedField.MakeFunction(`ObjectField.MakeCopy(templateLayoutDoc["${templateField}"])`, { templateLayoutDoc: Doc.name }, { templateLayoutDoc });
                            }
                            targetDoc[expandedLayoutFieldKey] = newLayoutDoc;

                            _pendingMap.delete(targetDoc[Id] + expandedLayoutFieldKey);
                        })
                    );
                }
            }
        }
        return expandedTemplateLayout instanceof Doc ? expandedTemplateLayout : undefined; // layout is undefined if the expandedTemplateLayout is pending.
    }

    // if the childDoc is a template for a field, then this will return the expanded layout with its data doc.
    // otherwise, it just returns the childDoc
    export function GetLayoutDataDocPair(containerDoc: Doc, containerDataDoc: Opt<Doc>, childDoc: Doc) {
        if (!childDoc || childDoc instanceof Promise || !Doc.GetProto(childDoc)) {
            console.log('Warning: GetLayoutDataDocPair childDoc not defined');
            return { layout: childDoc, data: childDoc };
        }
        const resolvedDataDoc = Doc.AreProtosEqual(containerDataDoc, containerDoc) || (!Doc.isTemplateDoc(childDoc) && !Doc.isTemplateForField(childDoc)) ? undefined : containerDataDoc;
        const templateRoot = DocCast(containerDoc?.rootDocument);
        return { layout: Doc.expandTemplateLayout(childDoc, templateRoot), data: resolvedDataDoc };
    }

    export function FindReferences(infield: Doc | List<Doc>, references: Set<Doc>, system: boolean | undefined) {
        if (infield instanceof Promise) return;
        if (!(infield instanceof Doc)) {
            infield?.forEach(val => (val instanceof Doc || val instanceof List) && FindReferences(val, references, system));
            return;
        }
        const doc = infield as Doc;
        if (references.has(doc)) {
            references.add(doc);
            return;
        }
        const excludeLists = [Doc.MyRecentlyClosed, Doc.MyHeaderBar, Doc.MyDashboards].includes(doc);
        if (system !== undefined && ((system && !Doc.IsSystem(doc)) || (!system && Doc.IsSystem(doc)))) return;
        references.add(doc);
        Object.keys(doc).forEach(key => {
            if (key === 'proto') {
                if (doc.proto instanceof Doc) {
                    Doc.FindReferences(doc.proto, references, system);
                }
            } else {
                const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
                const field = key === 'author' ? ClientUtils.CurrentUserEmail() : ProxyField.WithoutProxy(() => doc[key]);
                if (field instanceof RefField) {
                    if (field instanceof Doc) {
                        if (key === 'myLinkDatabase') {
                            field instanceof Doc && references.add(field);
                            // skip docs that have been closed and are scheduled for garbage collection
                        } else {
                            Doc.FindReferences(field, references, system);
                        }
                    }
                } else if (cfield instanceof ComputedField) {
                    /* empty */
                } else if (field instanceof ObjectField) {
                    if (field instanceof Doc) {
                        Doc.FindReferences(field, references, system);
                    } else if (field instanceof List) {
                        !excludeLists && Doc.FindReferences(field, references, system);
                    } else if (field instanceof ProxyField) {
                        if (key === 'myLinkDatabase') {
                            field instanceof Doc && references.add(field);
                            // skip docs that have been closed and are scheduled for garbage collection
                        } else {
                            Doc.FindReferences(field.value, references, system);
                        }
                    } else if (field instanceof PrefetchProxy) {
                        Doc.FindReferences(field.value, references, system);
                    }
                } else if (field instanceof Promise) {
                    // eslint-disable-next-line no-debugger
                    debugger; // This shouldn't happend...
                }
            }
        });
    }

    export function MakeCopy(doc: Doc, copyProto: boolean = false, copyProtoId?: string, retitle = false): Doc {
        const copy = runInAction(() => new Doc(copyProtoId, true));
        updateCachedAcls(copy);
        const exclude = [...StrListCast(doc.cloneFieldFilter), 'dragFactory_count', 'cloneFieldFilter'];
        Object.keys(doc)
            .filter(key => !exclude.includes(key))
            .forEach(key => {
                if (key === 'proto' && copyProto) {
                    if (doc.proto instanceof Doc) {
                        copy[key] = Doc.MakeCopy(doc.proto, false);
                    }
                } else {
                    const cfield = ComputedField.WithoutComputed(() => FieldValue(doc[key]));
                    const field = key === 'author' ? ClientUtils.CurrentUserEmail() : ProxyField.WithoutProxy(() => doc[key]);
                    if (field instanceof RefField) {
                        copy[key] = field;
                    } else if (cfield instanceof ComputedField) {
                        copy[key] = cfield[Copy](); // ComputedField.MakeFunction(cfield.script.originalScript);
                    } else if (field instanceof ObjectField) {
                        const docAtKey = doc[key];
                        copy[key] =
                            docAtKey instanceof Doc && key.includes('layout[')
                                ? new ProxyField(Doc.MakeCopy(docAtKey)) // copy the expanded render template
                                : ObjectField.MakeCopy(field);
                    } else if (field instanceof Promise) {
                        // eslint-disable-next-line no-debugger
                        debugger; // This shouldn't happend...
                    } else {
                        copy[key] = field;
                    }
                }
            });
        if (copyProto) {
            Doc.GetProto(copy).embedContainer = undefined;
            Doc.GetProto(copy).proto_embeddings = new List<Doc>([copy]);
        } else {
            Doc.AddEmbedding(copy, copy);
        }
        copy.embedContainer = undefined;
        if (retitle) {
            copy.title = incrementTitleCopy(StrCast(copy.title));
        }
        return copy;
    }

    export function MakeDelegate(doc: Doc, id?: string, title?: string): Doc;
    export function MakeDelegate(doc: Opt<Doc>, id?: string, title?: string): Opt<Doc>;
    export function MakeDelegate(doc: Opt<Doc>, id?: string, title?: string): Opt<Doc> {
        if (doc) {
            const delegate = new Doc(id, true);
            delegate[Initializing] = true;
            updateCachedAcls(delegate);
            delegate.proto = doc;
            delegate.author = ClientUtils.CurrentUserEmail();
            Object.keys(doc)
                .filter(key => key.startsWith('acl_'))
                .forEach(key => {
                    delegate[key] = doc[key];
                });
            title && (delegate.title = title);
            delegate[Initializing] = false;
            if (!Doc.IsSystem(doc)) Doc.AddEmbedding(doc, delegate);
            return delegate;
        }
        return undefined;
    }

    // Makes a delegate of a document by first creating a delegate where data should be stored
    // (ie, the 'data' doc), and then creates another delegate of that (ie, the 'layout' doc).
    // This is appropriate if you're trying to create a document that behaves like all
    // regularly created documents (e.g, text docs, pdfs, etc which all have data/layout docs)
    export function MakeDelegateWithProto(doc: Doc /* , id?: string, title?: string */) {
        const ndoc = Doc.ApplyTemplate(doc);
        if (ndoc) {
            Doc.GetProto(ndoc).isDataDoc = true;
            ndoc && (Doc.GetProto(ndoc).proto = doc);
        }
        return ndoc;
    }

    let _applyCount: number = 0;
    export function ApplyTemplate(templateDoc: Doc) {
        if (templateDoc) {
            const proto = new Doc();
            proto.author = ClientUtils.CurrentUserEmail();
            const target = Doc.MakeDelegate(proto);
            const targetKey = StrCast(templateDoc.layout_fieldKey, 'layout');
            const applied = ApplyTemplateTo(templateDoc, target, targetKey, templateDoc.title + '(...' + _applyCount++ + ')');
            target.layout_fieldKey = targetKey;
            applied && (Doc.GetProto(applied).type = templateDoc.type);
            return applied;
        }
        return undefined;
    }
    export function ApplyTemplateTo(templateDoc: Doc, target: Doc, targetKey: string, titleTarget: string | undefined) {
        if (!Doc.AreProtosEqual(target[targetKey] as Doc, templateDoc)) {
            if (target.resolvedDataDoc) {
                target[targetKey] = new PrefetchProxy(templateDoc);
            } else {
                titleTarget && (Doc.GetProto(target).title = titleTarget);
                const setDoc = [AclAdmin, AclEdit, AclAugment].includes(GetEffectiveAcl(Doc.GetProto(target))) ? Doc.GetProto(target) : target;
                setDoc[targetKey] = new PrefetchProxy(templateDoc);
            }
        }
        return target;
    }

    //
    //  This function converts a generic field layout display into a field layout that displays a specific
    // metadata field indicated by the title of the template field (not the default field that it was rendering)
    //
    export function MakeMetadataFieldTemplate(templateField: Doc, templateDoc: Opt<Doc>, keepFieldKey = false): boolean {
        // find the metadata field key that this template field doc will display (indicated by its title)
        const metadataFieldKey = keepFieldKey ? Doc.LayoutFieldKey(templateField) : StrCast(templateField.isTemplateForField) || StrCast(templateField.title).replace(/^-/, '') || Doc.LayoutFieldKey(templateField);

        // update the original template to mark it as a template
        templateField.isTemplateForField = metadataFieldKey;
        !keepFieldKey && (templateField.title = metadataFieldKey);

        const templateFieldValue = templateField[metadataFieldKey] || templateField[Doc.LayoutFieldKey(templateField)];
        // move any data that the template field had been rendering over to the template doc so that things will still be rendered
        // when the template field is adjusted to point to the new metadatafield key.
        // note 1: if the template field contained a list of documents, each of those documents will be converted to templates as well.
        // note 2: this will not overwrite any field that already exists on the template doc at the field key
        if (!templateDoc?.[metadataFieldKey] && templateFieldValue instanceof ObjectField) {
            Cast(templateFieldValue, listSpec(Doc), [])?.map(d => d instanceof Doc && MakeMetadataFieldTemplate(d, templateDoc));
            Doc.GetProto(templateField)[metadataFieldKey] = ObjectField.MakeCopy(templateFieldValue);
        }
        // get the layout string that the template uses to specify its layout
        const templateFieldLayoutString = StrCast(Doc.LayoutField(Doc.Layout(templateField)));

        // change it to render the target metadata field instead of what it was rendering before and assign it to the template field layout document.
        Doc.Layout(templateField).layout = templateFieldLayoutString.replace(/fieldKey={'[^']*'}/, `fieldKey={'${metadataFieldKey}'}`);

        return true;
    }

    // converts a document id to a url path on the server
    export function globalServerPath(doc: Doc | string = ''): string {
        return ClientUtils.prepend('/doc/' + (doc instanceof Doc ? doc[Id] : doc));
    }
    // converts a document id to a url path on the server
    export function localServerPath(doc?: Doc): string {
        return '/doc/' + (doc ? doc[Id] : '');
    }

    export function GetBrushHighlightStatus(doc: Doc) {
        return Doc.IsHighlighted(doc) ? DocBrushStatus.highlighted : Doc.GetBrushStatus(doc);
    }
    export class DocBrush {
        BrushedDoc = new Set<Doc>();
        SearchMatchDoc: ObservableMap<Doc, { searchMatch: number }> = new ObservableMap();
        brushDoc = action((doc: Doc, unbrush: boolean) => {
            unbrush ? this.BrushedDoc.delete(doc) : this.BrushedDoc.add(doc);
            doc[Brushed] = !unbrush;
        });
    }
    export const brushManager = new DocBrush();

    export class UserDocData {
        @observable _user_doc: Doc = undefined!;
        @observable _sharing_doc: Doc = undefined!;
        @observable _searchQuery: string = '';
    }

    // the document containing the view layout information - will be the Document itself unless the Document has
    // a layout field or 'layout' is given.
    export function Layout(doc: Doc, layout?: Doc): Doc {
        const overrideLayout = layout && Cast(doc[`${StrCast(layout.isTemplateForField, 'data')}_layout[` + layout[Id] + ']'], Doc, null);
        return overrideLayout || doc[DocLayout] || doc;
    }
    export function SetLayout(doc: Doc, layout: Doc | string) {
        doc[StrCast(doc.layout_fieldKey, 'layout')] = layout;
    }
    export function LayoutField(doc: Doc) {
        return doc[StrCast(doc.layout_fieldKey, 'layout')];
    }
    export function LayoutFieldKey(doc: Doc, templateLayoutString?: string): string {
        const match = StrCast(templateLayoutString || Doc.Layout(doc).layout).match(/fieldKey={'([^']+)'}/);
        return match?.[1] || ''; // bcz: TODO check on this .  used to always reference 'layout', now it uses the layout speicfied by the current layout_fieldKey
    }
    export function NativeAspect(doc: Doc, dataDoc?: Doc, useDim?: boolean) {
        return Doc.NativeWidth(doc, dataDoc, useDim) / (Doc.NativeHeight(doc, dataDoc, useDim) || 1);
    }
    export function NativeWidth(doc?: Doc, dataDoc?: Doc, useWidth?: boolean) {
        return !doc ? 0 : NumCast(doc._nativeWidth, NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_nativeWidth'], useWidth ? NumCast(doc._width) : 0));
    }
    export function NativeHeight(doc?: Doc, dataDoc?: Doc, useHeight?: boolean) {
        if (!doc) return 0;
        const nheight = (Doc.NativeWidth(doc, dataDoc, useHeight) / NumCast(doc._width)) * NumCast(doc._height); // divide before multiply to avoid floating point errrorin case nativewidth = width
        const dheight = NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_nativeHeight'], useHeight ? NumCast(doc._height) : 0);
        return NumCast(doc._nativeHeight, nheight || dheight);
    }
    export function SetNativeWidth(doc: Doc, width: number | undefined, fieldKey?: string) {
        doc[(fieldKey || Doc.LayoutFieldKey(doc)) + '_nativeWidth'] = width;
    }
    export function SetNativeHeight(doc: Doc, height: number | undefined, fieldKey?: string) {
        doc[(fieldKey || Doc.LayoutFieldKey(doc)) + '_nativeHeight'] = height;
    }

    const manager = new UserDocData();
    export function SearchQuery(): string {
        return manager._searchQuery;
    }
    export function SetSearchQuery(query: string) {
        runInAction(() => {
            manager._searchQuery = query;
        });
    }
    export function UserDoc(): Doc {
        return manager._user_doc;
    }
    export function SharingDoc(): Doc {
        return Doc.MySharedDocs;
    }
    export function LinkDBDoc(): Doc {
        return Cast(Doc.UserDoc().myLinkDatabase, Doc, null);
    }
    export function SetUserDoc(doc: Doc) {
        return (manager._user_doc = doc);
    }

    const isSearchMatchCache = computedFn((doc: Doc) =>
        (brushManager.SearchMatchDoc.has(doc) ? brushManager.SearchMatchDoc.get(doc) :
        brushManager.SearchMatchDoc.has(doc[DocData]) ? brushManager.SearchMatchDoc.get(doc[DocData]) : undefined)); // prettier-ignore
    export function IsSearchMatch(doc: Doc) {
        return isSearchMatchCache(doc);
    }
    export function IsSearchMatchUnmemoized(doc: Doc) {
        return brushManager.SearchMatchDoc.has(doc) ? brushManager.SearchMatchDoc.get(doc) : brushManager.SearchMatchDoc.has(doc[DocData]) ? brushManager.SearchMatchDoc.get(doc[DocData]) : undefined;
    }
    export function SetSearchMatch(doc: Doc, results: { searchMatch: number }) {
        if (doc && GetEffectiveAcl(doc) !== AclPrivate && GetEffectiveAcl(doc[DocData]) !== AclPrivate) {
            brushManager.SearchMatchDoc.set(doc, results);
        }
        return doc;
    }
    export function SearchMatchNext(doc: Doc, backward: boolean) {
        if (!doc || GetEffectiveAcl(doc) === AclPrivate || GetEffectiveAcl(doc[DocData]) === AclPrivate) return doc;
        const result = brushManager.SearchMatchDoc.get(doc);
        const num = Math.abs(result?.searchMatch || 0) + 1;
        runInAction(() => result && brushManager.SearchMatchDoc.set(doc, { searchMatch: backward ? -num : num }));
        return doc;
    }
    export function ClearSearchMatches() {
        brushManager.SearchMatchDoc.clear();
    }

    export enum DocBrushStatus {
        unbrushed = 0,
        protoBrushed = 1,
        selfBrushed = 2,
        highlighted = 3,
    }
    // returns 'how' a Doc has been brushed over - whether the document itself was brushed, it's prototype, or neither
    export function GetBrushStatus(doc: Doc) {
        if (!doc || GetEffectiveAcl(doc) === AclPrivate || GetEffectiveAcl(doc[DocData]) === AclPrivate || doc.opacity === 0) return DocBrushStatus.unbrushed;
        return doc[Brushed] ? DocBrushStatus.selfBrushed : doc[DocData][Brushed] ? DocBrushStatus.protoBrushed : DocBrushStatus.unbrushed;
    }
    export function BrushDoc(doc: Doc, unbrush = false) {
        if (doc && GetEffectiveAcl(doc) !== AclPrivate && GetEffectiveAcl(doc[DocData]) !== AclPrivate) {
            brushManager.brushDoc(doc, unbrush);
            brushManager.brushDoc(doc[DocData], unbrush);
        }
        return doc;
    }
    export function UnBrushDoc(doc: Doc) {
        return BrushDoc(doc, true);
    }
    export function UnBrushAllDocs() {
        Array.from(brushManager.BrushedDoc).forEach(
            action(doc => {
                doc[Brushed] = false;
            })
        );
    }

    const UnhighlightWatchers: (() => void)[] = [];
    let UnhighlightTimer: NodeJS.Timeout | undefined;
    export function IsUnhighlightTimerSet() { return UnhighlightTimer; } // prettier-ignore
    export function AddUnHighlightWatcher(watcher: () => void) {
        if (UnhighlightTimer) {
            UnhighlightWatchers.push(watcher);
        } else watcher();
    }
    export function linkFollowUnhighlight() {
        clearTimeout(UnhighlightTimer);
        UnhighlightTimer = undefined;
        UnhighlightWatchers.forEach(watcher => watcher());
        UnhighlightWatchers.length = 0;
        highlightedDocs.forEach(doc => Doc.UnHighlightDoc(doc));
        document.removeEventListener('pointerdown', linkFollowUnhighlight);
    }
    export function linkFollowHighlight(destDoc: Doc | Doc[], dataAndDisplayDocs = true, presentationEffect?: Doc) {
        linkFollowUnhighlight();
        toList(destDoc).forEach(doc => Doc.HighlightDoc(doc, dataAndDisplayDocs, presentationEffect));
        document.removeEventListener('pointerdown', linkFollowUnhighlight);
        document.addEventListener('pointerdown', linkFollowUnhighlight);
        if (UnhighlightTimer) clearTimeout(UnhighlightTimer);
        const presTransition = Number(presentationEffect?.presentation_transition);
        const duration = isNaN(presTransition) ? 5000 : presTransition;
        UnhighlightTimer = setTimeout(linkFollowUnhighlight, duration);
    }

    export const highlightedDocs = new ObservableSet<Doc>();
    export function IsHighlighted(doc: Doc) {
        if (!doc || GetEffectiveAcl(doc) === AclPrivate || GetEffectiveAcl(doc[DocData]) === AclPrivate || doc.opacity === 0) return false;
        return doc[Highlight] || doc[DocData][Highlight];
    }
    export function HighlightDoc(doc: Doc, dataAndDisplayDocs = true, presentationEffect?: Doc) {
        runInAction(() => {
            highlightedDocs.add(doc);
            doc[Highlight] = true;
            doc[Animation] = presentationEffect;
            if (dataAndDisplayDocs && !doc.resolvedDataDoc) {
                // if doc is a layout template then we don't want to highlight the proto since that will be the entire template, not just the specific layout field
                highlightedDocs.add(doc[DocData]);
                doc[DocData][Highlight] = true;
                // want to highlight the targets of presentation docs explicitly since following a pres target does not highlight PDf <Annotations> which are not DocumentViews
                if (doc.presentation_targetDoc) DocCast(doc.presentation_targetDoc)[Highlight] = true;
            }
        });
    }
    /// if doc is defined, then it is unhighlighted, otherwise all highlighted docs are unhighlighted
    export function UnHighlightDoc(docs?: Doc) {
        runInAction(() => {
            (docs ? [docs] : Array.from(highlightedDocs)).forEach(doc => {
                highlightedDocs.delete(doc);
                highlightedDocs.delete(doc[DocData]);
                doc[Highlight] = doc[DocData][Highlight] = false;
                doc[Animation] = undefined;
                if (doc.presentation_targetDoc) DocCast(doc.presentation_targetDoc)[Highlight] = false;
            });
        });
    }

    export function getDocTemplate(doc?: Doc) {
        return !doc
            ? undefined
            : doc.isTemplateDoc
              ? doc
              : Cast(doc.dragFactory, Doc, null)?.isTemplateDoc
                ? doc.dragFactory
                : Cast(Doc.Layout(doc), Doc, null)?.isTemplateDoc
                  ? Cast(Doc.Layout(doc), Doc, null).resolvedDataDoc
                      ? Doc.Layout(doc).proto
                      : Doc.Layout(doc)
                  : undefined;
    }

    export function toggleLockedPosition(doc: Doc) {
        doc._lockedPosition = !doc._lockedPosition;
        doc._pointerEvents = doc._lockedPosition ? 'none' : undefined;
    }

    export function deiconifyView(doc: Doc) {
        StrCast(doc.layout_fieldKey).split('_')[1] === 'icon' && setNativeView(doc);
    }

    export function setNativeView(doc: Doc) {
        const prevLayout = StrCast(doc.layout_fieldKey).split('_')[1];
        const deiconify = prevLayout === 'icon' && StrCast(doc.deiconifyLayout) ? 'layout_' + StrCast(doc.deiconifyLayout) : '';
        prevLayout === 'icon' && (doc.deiconifyLayout = undefined);
        doc.layout_fieldKey = deiconify || 'layout';
    }
    export function setDocRangeFilter(container: Opt<Doc>, key: string, range?: readonly number[], modifiers?: 'remove') {
        if (!container) return;

        const childFiltersByRanges = Cast(container._childFiltersByRanges, listSpec('string'), []);

        for (let i = 0; i < childFiltersByRanges.length; i += 3) {
            if (childFiltersByRanges[i] === key) {
                childFiltersByRanges.splice(i, 3);
                break;
            }
        }
        if (range !== undefined) {
            childFiltersByRanges.push(key);
            childFiltersByRanges.push(range[0].toString());
            childFiltersByRanges.push(range[1].toString());
            container._childFiltersByRanges = new List<string>(childFiltersByRanges);
        }

        if (modifiers) {
            childFiltersByRanges.splice(0, 3);
            container._childFiltersByRanges = new List<string>(childFiltersByRanges);
        }
    }

    export const FilterSep = '::';
    export const FilterAny = '--any--';
    export const FilterNone = '--undefined--';

    export function hasDocFilter(container: Opt<Doc>, key: string, value: string | undefined, fieldPrefix?: string) {
        if (!container) return;
        const filterField = '_' + (fieldPrefix ? fieldPrefix + '_' : '') + 'childFilters';
        const childFilters = StrListCast(container[filterField]);
        return childFilters.some(filter => filter.split(FilterSep)[0] === key && (value === undefined || value === Doc.FilterAny || filter.split(FilterSep)[1] === value));
    }

    // filters document in a container collection:
    // all documents with the specified value for the specified key are included/excluded
    // based on the modifiers :"check", "x", undefined
    export function setDocFilter(container: Opt<Doc>, key: string, value: FieldType | undefined, modifiers: 'remove' | 'match' | 'check' | 'x' | 'exists' | 'unset', toggle?: boolean, fieldPrefix?: string, append: boolean = true) {
        if (!container) return;
        const filterField = '_' + (fieldPrefix ? fieldPrefix + '_' : '') + 'childFilters';
        const childFilters = StrListCast(container[filterField]);
        runInAction(() => {
            for (let i = 0; i < childFilters.length; i++) {
                const fields = childFilters[i].split(FilterSep); // split key:value:modifier
                if (fields[0] === key && (fields[1] === value?.toString() || value === Doc.FilterAny || modifiers === 'match' || (fields[2] === 'match' && modifiers === 'remove'))) {
                    if (fields[2] === modifiers && modifiers && (fields[1] === value?.toString() || value === Doc.FilterAny)) {
                        // eslint-disable-next-line no-param-reassign
                        if (toggle) modifiers = 'remove';
                        else return;
                    }
                    childFilters.splice(i, 1);
                    container[filterField] = new List<string>(childFilters);
                    break;
                }
            }
            if (!childFilters.length && modifiers === 'match' && value === undefined) {
                container[filterField] = undefined;
            } else if (modifiers !== 'remove') {
                !append && (childFilters.length = 0);
                childFilters.push(key + FilterSep + value + FilterSep + modifiers);
                container[filterField] = new List<string>(childFilters);
            }
        });
    }
    export function readDocRangeFilter(doc: Doc, key: string) {
        const childFiltersByRanges = Cast(doc._childFiltersByRanges, listSpec('string'), []);
        for (let i = 0; i < childFiltersByRanges.length; i += 3) {
            if (childFiltersByRanges[i] === key) {
                return [Number(childFiltersByRanges[i + 1]), Number(childFiltersByRanges[i + 2])];
            }
        }
        return undefined;
    }
    export function assignDocToField(doc: Doc, field: string, id: string) {
        DocServer.GetRefField(id)?.then(layout => {
            layout instanceof Doc && (doc[field] = layout);
        });
        return id;
    }

    export function toggleNativeDimensions(layoutDoc: Doc, contentScale: number, panelWidth: number, panelHeight: number) {
        runInAction(() => {
            if (Doc.NativeWidth(layoutDoc) || Doc.NativeHeight(layoutDoc)) {
                layoutDoc._freeform_scale = NumCast(layoutDoc._freeform_scale, 1) * contentScale;
                layoutDoc._nativeWidth = undefined;
                layoutDoc._nativeHeight = undefined;
            } else {
                layoutDoc._layout_autoHeight = false;
                if (!Doc.NativeWidth(layoutDoc)) {
                    layoutDoc._nativeWidth = NumCast(layoutDoc._width, panelWidth);
                    layoutDoc._nativeHeight = NumCast(layoutDoc._height, panelHeight);
                }
            }
        });
    }

    export function Paste(docids: string[], clone: boolean, addDocument: (doc: Doc | Doc[]) => boolean, ptx?: number, pty?: number, newPoint?: number[]) {
        DocServer.GetRefFields(docids).then(async fieldlist => {
            const list = Array.from(fieldlist.values())
                .map(d => DocCast(d))
                .filter(d => d);
            const docs = clone ? (await Promise.all(Doc.MakeClones(list, false, false))).map(res => res.clone) : list;
            if (ptx !== undefined && pty !== undefined && newPoint !== undefined) {
                const firstx = list.length ? NumCast(list[0].x) + ptx - newPoint[0] : 0;
                const firsty = list.length ? NumCast(list[0].y) + pty - newPoint[1] : 0;
                docs.forEach(doc => {
                    doc.x = NumCast(doc.x) - firstx;
                    doc.y = NumCast(doc.y) - firsty;
                });
            }
            undoable(addDocument, 'Paste Doc')(docs); // embedContainer gets set in addDocument
        });
    }

    // prettier-ignore
    export function toIcon(doc?: Doc, isOpen?: Opt<boolean>) {
        if (isOpen) return doc?.isFolder ? 'chevron-down' : 'folder-open';
        switch (StrCast(doc?.type)) {
            case DocumentType.IMG:  return 'image';
            case DocumentType.COMPARISON: return 'columns';
            case DocumentType.RTF: return 'sticky-note';
            case DocumentType.COL:
                if (doc?.isFolder) {
                    switch (doc.type_collection) {
                        default: return isOpen === false ? 'chevron-right' : 'question';
                    } // prettier-ignore
                }
                switch (doc?.type_collection) {
                    case CollectionViewType.Freeform :   return 'object-group';
                    case CollectionViewType.NoteTaking : return 'chalkboard';
                    case CollectionViewType.Schema :     return 'table-cells';
                    case CollectionViewType.Docking:     return 'solar-panel';
                    default:                             return 'folder';
                } // prettier-ignore
            case DocumentType.WEB: return 'globe-asia';
            case DocumentType.SCREENSHOT: return 'photo-video';
            case DocumentType.WEBCAM: return 'video';
            case DocumentType.AUDIO: return 'microphone';
            case DocumentType.BUTTON: return 'bolt';
            case DocumentType.PRES: return 'route';
            case DocumentType.SCRIPTING: return 'terminal';
            case DocumentType.VID: return 'video';
            case DocumentType.INK: return 'pen-nib';
            case DocumentType.PDF: return 'file-pdf';
            case DocumentType.LINK: return 'link';
            case DocumentType.MAP: return 'map-marker-alt';
            case DocumentType.DATAVIZ: return 'chart-bar';
            case DocumentType.EQUATION: return 'calculator';
            case DocumentType.SIMULATION: return 'rocket';
            case DocumentType.CONFIG: return 'folder-closed';
            default:
        }
        return 'question';
    }

    ///
    // imports a previously exported zip file which contains a set of documents and their assets (eg, images, videos)
    // the 'remap' parameter determines whether the ids of the documents loaded should be kept as they were, or remapped to new ids
    // If they are not remapped, loading the file will overwrite any existing documents with those ids
    //
    export async function importDocument(file: File, remap = false) {
        const upload = ClientUtils.prepend('/uploadDoc');
        const formData = new FormData();
        if (file) {
            formData.append('file', file);
            formData.append('remap', remap.toString());
            const response = await fetch(upload, { method: 'POST', body: formData });
            const json = await response.json();
            if (json !== 'error') {
                // eslint-disable-next-line @typescript-eslint/no-unused-vars
                const docs = await DocServer.GetRefFields(json.docids as string[]);
                const doc = DocCast(await DocServer.GetRefField(json.id));
                const links = await DocServer.GetRefFields(json.linkids as string[]);
                Array.from(Object.keys(links))
                    .map(key => links.get(key))
                    .forEach(link => link instanceof Doc && Doc.AddLink?.(link));
                return doc;
            }
        }
        return undefined;
    }

    export namespace Get {
        const primitives = ['string', 'number', 'boolean'];

        export interface JsonConversionOpts {
            data: unknown;
            title?: string;
            appendToExisting?: { targetDoc: Doc; fieldKey?: string };
            excludeEmptyObjects?: boolean;
        }

        const defaultKey = 'json';

        /**
         * This function takes any valid JSON(-like) data, i.e. parsed or unparsed, and at arbitrarily
         * deep levels of nesting, converts the data and structure into nested documents with the appropriate fields.
         *
         * After building a hierarchy within / below a top-level document, it then returns that top-level parent.
         *
         * If we've received a string, treat it like valid JSON and try to parse it into an object. If this fails, the
         * string is invalid JSON, so we should assume that the input is the result of a JSON.parse()
         * call that returned a regular string value to be stored as a Field.
         *
         * If we've received something other than a string, since the caller might also pass in the results of a
         * JSON.parse() call, valid input might be an object, an array (still typeof object), a boolean or a number.
         * Anything else (like a function, etc. passed in naively as any) is meaningless for this operation.
         *
         * All TS/JS objects get converted directly to documents, directly preserving the key value structure. Everything else,
         * lacking the key value structure, gets stored as a field in a wrapper document.
         *
         * @param data for convenience and flexibility, either a valid JSON string to be parsed,
         * or the result of any JSON.parse() call.
         * @param title an optional title to give to the highest parent document in the hierarchy.
         * If whether this function creates a new document or appendToExisting is specified and that document already has a title,
         * because this title field can be left undefined for the opposite behavior, including a title will overwrite the existing title.
         * @param appendToExisting **if specified**, there are two cases, both of which return the target document:
         *
         * 1) the json to be converted can be represented as a document, in which case the target document will act as the root
         * of the tree and receive all the conversion results as new fields on itself
         * 2) the json can't be represented as a document, in which case the function will assign the field-level conversion
         * results to either the specified key on the target document, or to its "json" key by default.
         *
         * If not specified, the function creates and returns a new entirely generic document (different from the Doc.Create calls)
         * to act as the root of the tree.
         *
         * One might choose to specify this field if you want to write to a document returned from a Document.Create function call,
         * say a TreeView document that will be rendered, not just an untyped, identityless doc that would otherwise be created
         * from a default call to new Doc.
         *
         * @param excludeEmptyObjects whether non-primitive objects (TypeScript objects and arrays) should be converted even
         * if they contain no data. By default, empty objects and arrays are ignored.
         */
        export function FromJson({ data, title, appendToExisting, excludeEmptyObjects }: JsonConversionOpts): Opt<Doc> {
            if (excludeEmptyObjects === undefined) {
                // eslint-disable-next-line no-param-reassign
                excludeEmptyObjects = true;
            }
            if (data === undefined || data === null || ![...primitives, 'object'].includes(typeof data)) {
                return undefined;
            }
            let resolved: unknown;
            try {
                resolved = JSON.parse(typeof data === 'string' ? data : JSON.stringify(data));
            } catch (e) {
                console.error(e);
                return undefined;
            }
            let output: Opt<Doc>;
            if (typeof resolved === 'object' && !(resolved instanceof Array)) {
                output = convertObject(resolved as { [key: string]: FieldType }, excludeEmptyObjects, title, appendToExisting?.targetDoc);
            } else {
                // give the proper types to the data extracted from the JSON
                const result = toField(resolved, excludeEmptyObjects);
                if (appendToExisting) {
                    (output = appendToExisting.targetDoc)[appendToExisting.fieldKey || defaultKey] = result;
                } else {
                    (output = new Doc()).json = result;
                }
            }
            title && output && (output.title = title);
            return output;
        }

        /**
         * For each value of the object, recursively convert it to its appropriate field value
         * and store the field at the appropriate key in the document if it is not undefined
         * @param object the object to convert
         * @returns the object mapped from JSON to field values, where each mapping
         * might involve arbitrary recursion (since toField might itself call convertObject)
         */
        const convertObject = (object: { [key: string]: FieldType }, excludeEmptyObjects: boolean, title?: string, target?: Doc): Opt<Doc> => {
            const hasEntries = Object.keys(object).length;
            if (hasEntries || !excludeEmptyObjects) {
                const resolved = target ?? new Doc();
                if (hasEntries) {
                    Object.keys(object).forEach(key => {
                        // if excludeEmptyObjects is true, any qualifying conversions from toField will
                        // be undefined, and thus the results that would have
                        // otherwise been empty (List or Doc)s will just not be written
                        const result = toField(object[key], excludeEmptyObjects, key);
                        if (result) {
                            resolved[key] = result;
                        }
                    });
                }
                title && (resolved.title = title);
                return resolved;
            }
            return undefined;
        };

        /**
         * For each element in the list, recursively convert it to a document or other field
         * and push the field to the list if it is not undefined
         * @param list the list to convert
         * @returns the list mapped from JSON to field values, where each mapping
         * might involve arbitrary recursion (since toField might itself call convertList)
         */
        const convertList = (list: Array<unknown>, excludeEmptyObjects: boolean): Opt<List<FieldType>> => {
            const target = new List();
            let result: Opt<FieldType>;
            // if excludeEmptyObjects is true, any qualifying conversions from toField will
            // be undefined, and thus the results that would have
            // otherwise been empty (List or Doc)s will just not be written
            list.forEach(item => {
                (result = toField(item, excludeEmptyObjects)) && target.push(result);
            });
            if (target.length || !excludeEmptyObjects) {
                return target;
            }
            return undefined;
        };

        const toField = (data: unknown, excludeEmptyObjects: boolean, title?: string): Opt<FieldType> => {
            if (data === null || data === undefined) {
                return undefined;
            }
            if (primitives.includes(typeof data)) {
                return data as FieldType;
            }
            if (typeof data === 'object') {
                return data instanceof Array ? convertList(data, excludeEmptyObjects) : convertObject(data as { [key: string]: FieldType }, excludeEmptyObjects, title, undefined);
            }
            throw new Error(`How did ${data} of type ${typeof data} end up in JSON?`);
        };
    }
}

export function returnEmptyDoclist() {
    return [] as Doc[];
}

export function RTFIsFragment(html: string) {
    return html.indexOf('data-pm-slice') !== -1;
}
export function GetHrefFromHTML(html: string): string {
    const parser = new DOMParser();
    const parsedHtml = parser.parseFromString(html, 'text/html');
    if (parsedHtml.body.childNodes.length === 1 && parsedHtml.body.childNodes[0].childNodes.length === 1 && (parsedHtml.body.childNodes[0].childNodes[0] as HTMLAnchorElement).href) {
        return (parsedHtml.body.childNodes[0].childNodes[0] as HTMLAnchorElement).href;
    }
    return '';
}
export function GetDocFromUrl(url: string) {
    return url.startsWith(document.location.origin) ? new URL(url).pathname.split('doc/').lastElement() : ''; // docId
}

let activeAudioLinker: (f: () => Doc | undefined, broadcast?: boolean) => (Doc | undefined)[];
export function SetActiveAudioLinker(func: (f: () => Doc | undefined, broadcast?: boolean) => (Doc | undefined)[]) {
    activeAudioLinker = func;
}
export function CreateLinkToActiveAudio(func: () => Doc | undefined, broadcast?: boolean) {
    return activeAudioLinker(func, broadcast);
}

export function IdToDoc(id: string) {
    return DocCast(DocServer.GetCachedRefField(id));
}
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function idToDoc(id: string): Doc {
    return IdToDoc(id);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function renameEmbedding(doc: Doc) {
    return StrCast(doc[DocData].title).replace(/\([0-9]*\)/, '') + `(${doc.proto_embeddingId})`;
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function getProto(doc: Doc) {
    return Doc.GetProto(doc);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function getDocTemplate(doc?: Doc) {
    return Doc.getDocTemplate(doc);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function getEmbedding(doc: Doc) {
    return Doc.MakeEmbedding(doc);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function getCopy(doc: Doc, copyProto: boolean) {
    return doc.isTemplateDoc ? Doc.MakeDelegateWithProto(doc) : Doc.MakeCopy(doc, copyProto);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function copyField(field: FieldResult) {
    return Field.Copy(field);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function docList(field: FieldResult) {
    return DocListCast(field);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function addDocToList(doc: Doc, field: string, added: Doc) {
    return Doc.AddDocToList(doc, field, added);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function setInPlace(doc: Doc, field: string, value: string) {
    return Doc.SetInPlace(doc, field, value, false);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function sameDocs(doc1: Doc, doc2: Doc) {
    return Doc.AreProtosEqual(doc1, doc2);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function assignDoc(doc: Doc, field: string, id: string) {
    return Doc.assignDocToField(doc, field, id);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function docCastAsync(doc: FieldResult): FieldResult<Doc> {
    return Cast(doc, Doc);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function activePresentationItem() {
    const curPres = Doc.ActivePresentation;
    return curPres && DocListCast(curPres[Doc.LayoutFieldKey(curPres)])[NumCast(curPres._itemIndex)];
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function setDocFilter(container: Doc, key: string, value: string, modifiers: 'match' | 'check' | 'x' | 'remove') {
    Doc.setDocFilter(container, key, value, modifiers);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function setDocRangeFilter(container: Doc, key: string, range: number[]) {
    Doc.setDocRangeFilter(container, key, range);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function toJavascriptString(str: string) {
    return Field.toJavascriptString(str as FieldType);
});
// eslint-disable-next-line prefer-arrow-callback
ScriptingGlobals.add(function RtfField() {
    return RichTextField.RTFfield();
});