Merge branch 'master' into zach-starter

1 files changed, 100 insertions, 192 deletions

diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts
index ac865382d..c644308b7 100644
--- a/src/client/DocServer.ts
+++ b/src/client/DocServer.ts
@@ -1,14 +1,14 @@
-import { runInAction } from 'mobx';
+/* eslint-disable @typescript-eslint/no-namespace */
+import { action } from 'mobx';
 import { Socket, io } from 'socket.io-client';
 import { ClientUtils } from '../ClientUtils';
 import { Utils, emptyFunction } from '../Utils';
-import { Doc, Opt } from '../fields/Doc';
+import { Doc, FieldType, Opt, SetObjGetRefField, SetObjGetRefFields } from '../fields/Doc';
 import { UpdatingFromServer } from '../fields/DocSymbols';
 import { FieldLoader } from '../fields/FieldLoader';
 import { HandleUpdate, Id, Parent } from '../fields/FieldSymbols';
-import { ObjectField, SetObjGetRefField, SetObjGetRefFields } from '../fields/ObjectField';
-import { RefField } from '../fields/RefField';
-import { GestureContent, Message, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, YoutubeQueryTypes } from '../server/Message';
+import { ObjectField, serverOpType } from '../fields/ObjectField';
+import { Message, MessageStore } from '../server/Message';
 import { SerializationHelper } from './util/SerializationHelper';
 
 /**
@@ -25,8 +25,7 @@ import { SerializationHelper } from './util/SerializationHelper';
  * or update ourselves based on the server's update message, that occurs here
  */
 export namespace DocServer {
-    // eslint-disable-next-line import/no-mutable-exports
-    let _cache: { [id: string]: RefField | Promise<Opt<RefField>> } = {};
+    let _cache: { [id: string]: Doc | Promise<Opt<Doc>> } = {};
     export function Cache() {
         return _cache;
     }
@@ -34,24 +33,24 @@ export namespace DocServer {
     function errorFunc(): never {
         throw new Error("Can't use DocServer without calling init first");
     }
-    let _UpdateField: (id: string, diff: any) => void = errorFunc;
-    let _CreateField: (field: RefField) => void = errorFunc;
+    let _UpdateField: (id: string, diff: serverOpType) => void = errorFunc;
+    let _CreateDocField: (field: Doc) => void = errorFunc;
 
-    export function AddServerHandler<T>(socket: Socket, message: Message<T>, handler: (args: T) => any) {
+    export function AddServerHandler<T>(socket: Socket, message: Message<T>, handler: (args: T) => void) {
         socket.on(message.Message, Utils.loggingCallback('Incoming', handler, message.Name));
     }
     export function Emit<T>(socket: Socket, message: Message<T>, args: T) {
         // log('Emit', message.Name, args, false);
         socket.emit(message.Message, args);
     }
-    export function EmitCallback<T>(socket: Socket, message: Message<T>, args: T): Promise<any>;
-    export function EmitCallback<T>(socket: Socket, message: Message<T>, args: T, fn: (args: any) => any): void;
-    export function EmitCallback<T>(socket: Socket, message: Message<T>, args: T, fn?: (args: any) => any): void | Promise<any> {
+    export function EmitCallback<T>(socket: Socket, message: Message<T>, args: T): Promise<unknown>;
+    export function EmitCallback<T>(socket: Socket, message: Message<T>, args: T, fn: (args: unknown) => unknown): void;
+    export function EmitCallback<T>(socket: Socket, message: Message<T>, args: T, fn?: (args: unknown) => unknown): void | Promise<unknown> {
         // log('Emit', message.Name, args, false);
         if (fn) {
             socket.emit(message.Message, args, Utils.loggingCallback('Receiving', fn, message.Name));
         } else {
-            return new Promise<any>(res => {
+            return new Promise<unknown>(res => {
                 socket.emit(message.Message, args, Utils.loggingCallback('Receiving', res, message.Name));
             });
         }
@@ -99,7 +98,7 @@ export namespace DocServer {
         return ClientUtils.CurrentUserEmail() === 'guest' ? WriteMode.LivePlayground : fieldWriteModes[field] || WriteMode.Default;
     }
 
-    export function registerDocWithCachedUpdate(doc: Doc, field: string, oldValue: any) {
+    export function registerDocWithCachedUpdate(doc: Doc, field: string, oldValue: FieldType) {
         let list = docsWithUpdates[field];
         if (!list) {
             list = docsWithUpdates[field] = new Set();
@@ -110,25 +109,6 @@ export namespace DocServer {
         }
     }
 
-    export namespace Mobile {
-        export function dispatchGesturePoints(content: GestureContent) {
-            DocServer.Emit(_socket, MessageStore.GesturePoints, content);
-        }
-
-        export function dispatchOverlayTrigger(content: MobileInkOverlayContent) {
-            // _socket.emit("dispatchBoxTrigger");
-            DocServer.Emit(_socket, MessageStore.MobileInkOverlayTrigger, content);
-        }
-
-        export function dispatchOverlayPositionUpdate(content: UpdateMobileInkOverlayPositionContent) {
-            DocServer.Emit(_socket, MessageStore.UpdateMobileInkOverlayPosition, content);
-        }
-
-        export function dispatchMobileDocumentUpload(content: MobileDocumentUploadContent) {
-            DocServer.Emit(_socket, MessageStore.MobileDocumentUpload, content);
-        }
-    }
-
     const instructions = 'This page will automatically refresh after this alert is closed. Expect to reconnect after about 30 seconds.';
     function alertUser(connectionTerminationReason: string) {
         switch (connectionTerminationReason) {
@@ -152,7 +132,7 @@ export namespace DocServer {
         export function makeReadOnly() {
             if (!_isReadOnly) {
                 _isReadOnly = true;
-                _CreateField = field => {
+                _CreateDocField = field => {
                     _cache[field[Id]] = field;
                 };
                 _UpdateField = emptyFunction;
@@ -203,7 +183,7 @@ export namespace DocServer {
      * the server if the document has not been cached.
      * @param id the id of the requested document
      */
-    const _GetRefFieldImpl = (id: string, force: boolean = false): Promise<Opt<RefField>> => {
+    const _GetRefFieldImpl = (id: string, force: boolean = false): Promise<Opt<Doc>> => {
         // an initial pass through the cache to determine whether the document needs to be fetched,
         // is already in the process of being fetched or already exists in the
         // cache
@@ -221,7 +201,7 @@ export namespace DocServer {
             // future .proto calls on the Doc won't have to go farther than the cache to get their actual value.
             const deserializeField = getSerializedField.then(async fieldJson => {
                 // deserialize
-                const field = await SerializationHelper.Deserialize(fieldJson);
+                const field = (await SerializationHelper.Deserialize(fieldJson)) as Doc;
                 if (force && field && cached instanceof Doc) {
                     cached[UpdatingFromServer] = true;
                     Array.from(Object.keys(field)).forEach(key => {
@@ -247,7 +227,7 @@ export namespace DocServer {
             // here, indicate that the document associated with this id is currently
             // being retrieved and cached
             !force && (_cache[id] = deserializeField);
-            return force ? (cached as any) : deserializeField;
+            return force ? (cached instanceof Promise ? cached : new Promise<Doc>(res => res(cached))) : deserializeField;
         }
         if (cached instanceof Promise) {
             // BEING RETRIEVED AND CACHED => some other caller previously (likely recently) called GetRefField(s),
@@ -261,7 +241,7 @@ export namespace DocServer {
             // (field instanceof Doc) && fetchProto(field);
         );
     };
-    const _GetCachedRefFieldImpl = (id: string): Opt<RefField> => {
+    const _GetCachedRefFieldImpl = (id: string): Opt<Doc> => {
         const cached = _cache[id];
         if (cached !== undefined && !(cached instanceof Promise)) {
             return cached;
@@ -269,174 +249,102 @@ export namespace DocServer {
         return undefined;
     };
 
-    let _GetRefField: (id: string, force: boolean) => Promise<Opt<RefField>> = errorFunc;
-    let _GetCachedRefField: (id: string) => Opt<RefField> = errorFunc;
+    let _GetRefField: (id: string, force: boolean) => Promise<Opt<Doc>> = errorFunc;
+    let _GetCachedRefField: (id: string) => Opt<Doc> = errorFunc;
 
-    export function GetRefField(id: string, force = false): Promise<Opt<RefField>> {
+    export function GetRefField(id: string, force = false): Promise<Opt<Doc>> {
         return _GetRefField(id, force);
     }
-    export function GetCachedRefField(id: string): Opt<RefField> {
+    export function GetCachedRefField(id: string): Opt<Doc> {
         return _GetCachedRefField(id);
     }
 
-    export async function getYoutubeChannels() {
-        return DocServer.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.Channels });
-    }
-
-    export function getYoutubeVideos(videoTitle: string, callBack: (videos: any[]) => void) {
-        DocServer.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.SearchVideo, userInput: videoTitle }, callBack);
-    }
-
-    export function getYoutubeVideoDetails(videoIds: string, callBack: (videoDetails: any[]) => void) {
-        DocServer.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.VideoDetails, videoIds: videoIds }, callBack);
-    }
-
     /**
      * Given a list of Doc GUIDs, this utility function will asynchronously attempt to each id's associated
      * field, first looking in the RefField cache and then communicating with
      * the server if the document has not been cached.
      * @param ids the ids that map to the reqested documents
      */
-    const _GetRefFieldsImpl = async (ids: string[]): Promise<{ [id: string]: Opt<RefField> }> => {
-        const requestedIds: string[] = [];
-        const promises: Promise<any>[] = [];
-
-        let defaultRes: any;
-        const defaultPromise = new Promise<any>(res => {
-            defaultRes = res;
+    const _GetRefFieldsImpl = async (ids: string[]): Promise<Map<string, Opt<Doc>>> => {
+        const uncachedRequestedIds: string[] = [];
+        const deserializeDocPromises: Promise<Opt<Doc>>[] = [];
+
+        // setup a Promise that we will resolve after all cached Docs have been acquired.
+        let allCachesFilledResolver!: (value: Opt<Doc> | PromiseLike<Opt<Doc>>) => void;
+        const allCachesFilledPromise = new Promise<Opt<Doc>>(res => {
+            allCachesFilledResolver = res;
         });
-        const defaultPromises: { p: Promise<any>; id: string }[] = [];
-        // 1) an initial pass through the cache to determine
-        // i) which documents need to be fetched
-        // ii) which are already in the process of being fetched
-        // iii) which already exist in the cache
+
+        const fetchDocPromises: Map<string, Promise<Opt<Doc>>> = new Map(); // { p: Promise<Doc>; id: string }[] = []; // promises to fetch the value for a requested Doc
+        // Determine which requested documents need to be fetched
         // eslint-disable-next-line no-restricted-syntax
         for (const id of ids.filter(filterid => filterid)) {
-            const cached = _cache[id];
-            if (cached === undefined) {
-                defaultPromises.push({
-                    id,
-                    // eslint-disable-next-line no-loop-func
-                    p: (_cache[id] = new Promise<any>(res => {
-                        defaultPromise.then(() => res(_cache[id]));
-                    })),
-                });
-                // NOT CACHED => we'll have to send a request to the server
-                requestedIds.push(id);
-            } else if (cached instanceof Promise) {
-                // BEING RETRIEVED AND CACHED => some other caller previously (likely recently) called GetRefField(s),
-                // and requested one of the documents I'm looking for. Shouldn't fetch again, just
-                // wait until this promise is resolved (see 7)
-                promises.push(cached);
-                // waitingIds.push(id);
-            } else {
-                // CACHED => great, let's just add it to the field map
-                // map[id] = cached;
+            if (_cache[id] === undefined) {
+                // EMPTY CACHE - make promise that we resolve after all batch-requested Docs have been fetched and deserialized and we know we have this Doc
+                const fetchPromise = new Promise<Opt<Doc>>(res =>
+                    allCachesFilledPromise.then(() => {
+                        // if all Docs have been cached, then we can be sure the fetched Doc has been found and cached. So return it to anyone who had been awaiting it.
+                        const cache = _cache[id];
+                        if (!(cache instanceof Doc)) console.log('CACHE WAS NEVER FILLED!!');
+                        res(cache instanceof Doc ? cache : undefined);
+                    })
+                );
+                // eslint-disable-next-line no-loop-func
+                fetchDocPromises.set(id, (_cache[id] = fetchPromise));
+                uncachedRequestedIds.push(id); // add to list of Doc requests from server
             }
+            // else CACHED => do nothing, Doc or promise of Doc is already in cache
         }
 
-        if (requestedIds.length) {
-            // 2) synchronously, we emit a single callback to the server requesting the serialized (i.e. represented by a string)
-            // fields for the given ids. This returns a promise, which, when resolved, indicates that all the JSON serialized versions of
-            // the fields have been returned from the server
-            console.log('Requesting ' + requestedIds.length);
-            setTimeout(() =>
-                runInAction(() => {
-                    FieldLoader.ServerLoadStatus.requested = requestedIds.length;
-                })
-            );
-            const serializedFields = await DocServer.EmitCallback(_socket, MessageStore.GetRefFields, requestedIds);
-
-            // 3) when the serialized RefFields have been received, go head and begin deserializing them into objects.
-            // Here, once deserialized, we also invoke .proto to 'load' the documents' prototypes, which ensures that all
-            // future .proto calls on the Doc won't have to go farther than the cache to get their actual value.
+        if (uncachedRequestedIds.length) {
+            console.log('Requesting ' + uncachedRequestedIds.length);
+            setTimeout(action(() => { FieldLoader.ServerLoadStatus.requested = uncachedRequestedIds.length; })); // prettier-ignore
+
+            // Synchronously emit a single server request for the serialized (i.e. represented by a string) Doc ids
+            // This returns a promise, that resolves when all the JSON serialized Docs have been retrieved
+            const serializedFields = (await DocServer.EmitCallback(_socket, MessageStore.GetRefFields, uncachedRequestedIds)) as { id: string; fields: unknown[]; __type: string }[];
+
             let processed = 0;
-            console.log('deserializing ' + serializedFields.length + ' fields');
+            console.log('Retrieved ' + serializedFields.length + ' fields');
+            // After the serialized Docs have been received, deserialize them into objects.
             // eslint-disable-next-line no-restricted-syntax
             for (const field of serializedFields) {
-                processed++;
-                if (processed % 150 === 0) {
+                // eslint-disable-next-line no-await-in-loop
+                ++processed % 150 === 0 &&
+                    (await new Promise<number>(
+                        res =>
+                        setTimeout(action(() => res(FieldLoader.ServerLoadStatus.retrieved = processed))) // prettier-ignore
+                    )); // force loading to yield to splash screen rendering to update progress
+
+                if (fetchDocPromises.has(field.id)) {
+                    // Doc hasn't started deserializing yet - the cache still has the fetch promise
                     // eslint-disable-next-line no-loop-func
-                    runInAction(() => {
-                        FieldLoader.ServerLoadStatus.retrieved = processed;
+                    const deserializePromise = SerializationHelper.Deserialize(field).then((deserialized: unknown) => {
+                        const doc = deserialized as Doc;
+                        // overwrite any fetch or deserialize cache promise with deserialized value.
+                        // fetch promises wait to resolve until after all deserializations; deserialize promises resolve upon deserializaton
+                        if (deserialized !== undefined) _cache[field.id] = doc;
+                        else delete _cache[field.id];
+
+                        return doc;
                     });
-                    // eslint-disable-next-line no-await-in-loop
-                    await new Promise(res => {
-                        setTimeout(res);
-                    }); // force loading to yield to splash screen rendering to update progress
-                }
-                const cached = _cache[field.id];
-                if (!cached || (cached instanceof Promise && defaultPromises.some(dp => dp.p === cached))) {
-                    // deserialize
-                    // adds to a list of promises that will be awaited asynchronously
-                    promises.push(
-                        // eslint-disable-next-line no-loop-func
-                        (_cache[field.id] = SerializationHelper.Deserialize(field).then(deserialized => {
-                            // overwrite or delete any promises (that we inserted as flags
-                            // to indicate that the field was in the process of being fetched). Now everything
-                            // should be an actual value within or entirely absent from the cache.
-                            if (deserialized !== undefined) {
-                                _cache[field.id] = deserialized;
-                            } else {
-                                delete _cache[field.id];
-                            }
-                            const promInd = defaultPromises.findIndex(dp => dp.id === field.id);
-                            promInd !== -1 && defaultPromises.splice(promInd, 1);
-                            return deserialized;
-                        }))
-                    );
-                    // 4) here, for each of the documents we've requested *ourselves* (i.e. weren't promises or found in the cache)
-                    // we set the value at the field's id to a promise that will resolve to the field.
-                    // When we find that promises exist at keys in the cache, THIS is where they were set, just by some other caller (method).
-                    // The mapping in the .then call ensures that when other callers await these promises, they'll
-                    // get the resolved field
-                } else if (cached instanceof Promise) {
+                    deserializeDocPromises.push((_cache[field.id] = deserializePromise)); // replace the cache's placeholder fetch promise with the deserializePromise
+                    fetchDocPromises.delete(field.id);
+                } else if (_cache[field.id] instanceof Promise) {
                     console.log('.');
-                    // promises.push(cached);
-                } else if (field) {
-                    // console.log('-');
                 }
             }
         }
 
-        await Promise.all(promises);
-        defaultPromises.forEach(df => delete _cache[df.id]);
-        defaultRes();
-
-        // 5) at this point, all fields have a) been returned from the server and b) been deserialized into actual Field objects whose
-        // prototype documents, if any, have also been fetched and cached.
-        console.log('Deserialized ' + (requestedIds.length - defaultPromises.length) + ' fields');
-        // 6) with this confidence, we can now go through and update the cache at the ids of the fields that
-        // we explicitly had to fetch. To finish it off, we add whatever value we've come up with for a given
-        // id to the soon-to-be-returned field mapping.
-        // ids.forEach(id => (map[id] = _cache[id] as any));
-
-        // 7) those promises we encountered in the else if of 1), which represent
-        // other callers having already submitted a request to the server for (a) document(s)
-        // in which we're interested, must still be awaited so that we can return the proper
-        // values for those as well.
-        //
-        // fortunately, those other callers will also hit their own version of 6) and clean up
-        // the shared cache when these promises resolve, so all we have to do is...
-        // const otherCallersFetching = await Promise.all(promises);
-        // ...extract the RefFields returned from the resolution of those promises and add them to our
-        // own map.
-        // waitingIds.forEach((id, index) => (map[id] = otherCallersFetching[index]));
-
-        // now, we return our completed mapping from all of the ids that were passed into the method
-        // to their actual RefField | undefined values. This return value either becomes the input
-        // argument to the caller's promise (i.e. GetRefFields(["_id1_", "_id2_", "_id3_"]).then(map => //do something with map...))
-        // or it is the direct return result if the promise is awaited (i.e. let fields = await GetRefFields(["_id1_", "_id2_", "_id3_"])).
-        return ids.reduce(
-            (map, id) => {
-                map[id] = _cache[id] as any;
-                return map;
-            },
-            {} as { [id: string]: Opt<RefField> }
-        );
+        await Promise.all(deserializeDocPromises); // promise resolves when cache is up-to-date with all requested Docs
+        Array.from(fetchDocPromises).forEach(([id]) => delete _cache[id]);
+        allCachesFilledResolver(undefined); // notify anyone who was promised a Doc fron when it was just being fetched (since all requested Docs have now been fetched and deserialized)
+
+        console.log('Deserialized ' + (uncachedRequestedIds.length - fetchDocPromises.size) + ' fields');
+        return new Map<string, Opt<Doc>>(ids.map(id => [id, _cache[id] instanceof Doc ? (_cache[id] as Doc) : undefined]) as [string, Opt<Doc>][]);
     };
 
-    let _GetRefFields: (ids: string[]) => Promise<{ [id: string]: Opt<RefField> }> = errorFunc;
+    let _GetRefFields: (ids: string[]) => Promise<Map<string, Opt<Doc>>> = errorFunc;
 
     export function GetRefFields(ids: string[]) {
         return _GetRefFields(ids);
@@ -449,20 +357,20 @@ export namespace DocServer {
     }
 
     /**
-     * A wrapper around the function local variable _createField.
+     * A wrapper around the function local variable _CreateDocField.
      * This allows us to swap in different executions while comfortably
      * calling the same function throughout the code base (such as in Util.makeReadonly())
      * @param field the [RefField] to be serialized and sent to the server to be stored in the database
      */
-    export function CreateField(field: RefField) {
+    export function CreateDocField(field: Doc) {
         _cacheNeedsUpdate = true;
-        _CreateField(field);
+        _CreateDocField(field);
     }
 
-    function _CreateFieldImpl(field: RefField) {
+    function _CreateDocFieldImpl(field: Doc) {
         _cache[field[Id]] = field;
         const initialState = SerializationHelper.Serialize(field);
-        ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Emit(_socket, MessageStore.CreateField, initialState);
+        ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Emit(_socket, MessageStore.CreateDocField, initialState);
     }
 
     // NOTIFY THE SERVER OF AN UPDATE TO A DOC'S STATE
@@ -475,22 +383,22 @@ export namespace DocServer {
      * @param updatedState the new value of the document. At some point, this
      * should actually be a proper diff, to improve efficiency
      */
-    export function UpdateField(id: string, updatedState: any) {
+    export function UpdateField(id: string, updatedState: serverOpType) {
         _UpdateField(id, updatedState);
     }
 
-    function _UpdateFieldImpl(id: string, diff: any) {
+    function _UpdateFieldImpl(id: string, diff: serverOpType) {
         !DocServer.Control.isReadOnly() && ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Emit(_socket, MessageStore.UpdateField, { id, diff });
     }
 
-    function _respondToUpdateImpl(diff: any) {
-        const { id } = diff;
+    function _respondToUpdateImpl(change: { id: string; diff: serverOpType }) {
+        const { id } = change;
         // to be valid, the Diff object must reference
         // a document's id
         if (id === undefined) {
             return;
         }
-        const update = (f: Opt<RefField>) => {
+        const update = (f: Opt<Doc>) => {
             // if the RefField is absent from the cache or
             // its promise in the cache resolves to undefined, there
             // can't be anything to update
@@ -500,7 +408,7 @@ export namespace DocServer {
             // extract this Doc's update handler
             const handler = f[HandleUpdate];
             if (handler) {
-                handler.call(f, diff.diff);
+                handler.call(f, change.diff as { $set: { [key: string]: FieldType } } | { $unset: unknown });
             }
         };
         // check the cache for the field
@@ -536,8 +444,8 @@ export namespace DocServer {
     const _RespondToUpdate = _respondToUpdateImpl;
     const _respondToDelete = _respondToDeleteImpl;
 
-    function respondToUpdate(diff: any) {
-        _RespondToUpdate(diff);
+    function respondToUpdate(change: { id: string; diff: serverOpType }) {
+        _RespondToUpdate(change);
     }
 
     function respondToDelete(ids: string | string[]) {
@@ -548,13 +456,13 @@ export namespace DocServer {
         _cache = {};
         USER_ID = identifier;
         _socket = io(`${protocol.startsWith('https') ? 'wss' : 'ws'}://${hostname}:${port}`, { transports: ['websocket'], rejectUnauthorized: false });
-        _socket.on('connect_error', (err: any) => console.log(err));
+        _socket.on('connect_error', (err: unknown) => console.log(err));
         // io.connect(`https://7f079dda.ngrok.io`);// if using ngrok, create a special address for the websocket
 
         _GetCachedRefField = _GetCachedRefFieldImpl;
         SetObjGetRefField((_GetRefField = _GetRefFieldImpl));
         SetObjGetRefFields((_GetRefFields = _GetRefFieldsImpl));
-        _CreateField = _CreateFieldImpl;
+        _CreateDocField = _CreateDocFieldImpl;
         _UpdateField = _UpdateFieldImpl;
 
         /**