From 4574b7f03ccc85c4bebdbfd9475788456086704f Mon Sep 17 00:00:00 2001 From: bobzel Date: Thu, 8 Aug 2024 12:27:40 -0400 Subject: many changes to add typing in place of 'any's etc --- src/client/DocServer.ts | 263 +++++++++++++++++------------------------------- 1 file changed, 95 insertions(+), 168 deletions(-) (limited to 'src/client/DocServer.ts') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index ac865382d..2bf3a6f9f 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -1,14 +1,14 @@ -import { runInAction } from 'mobx'; +/* eslint-disable @typescript-eslint/no-namespace */ +import { action } from 'mobx'; import { Socket, io } from 'socket.io-client'; import { ClientUtils } from '../ClientUtils'; import { Utils, emptyFunction } from '../Utils'; -import { Doc, Opt } from '../fields/Doc'; +import { Doc, FieldType, Opt, SetObjGetRefField, SetObjGetRefFields } from '../fields/Doc'; import { UpdatingFromServer } from '../fields/DocSymbols'; import { FieldLoader } from '../fields/FieldLoader'; import { HandleUpdate, Id, Parent } from '../fields/FieldSymbols'; -import { ObjectField, SetObjGetRefField, SetObjGetRefFields } from '../fields/ObjectField'; -import { RefField } from '../fields/RefField'; -import { GestureContent, Message, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, YoutubeQueryTypes } from '../server/Message'; +import { ObjectField, serverOpType } from '../fields/ObjectField'; +import { GestureContent, Message, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent } from '../server/Message'; import { SerializationHelper } from './util/SerializationHelper'; /** @@ -25,8 +25,7 @@ import { SerializationHelper } from './util/SerializationHelper'; * or update ourselves based on the server's update message, that occurs here */ export namespace DocServer { - // eslint-disable-next-line import/no-mutable-exports - let _cache: { [id: string]: RefField | Promise> } = {}; + let _cache: { [id: string]: Doc | Promise> } = {}; export function Cache() { return _cache; } @@ -34,24 +33,24 @@ export namespace DocServer { function errorFunc(): never { throw new Error("Can't use DocServer without calling init first"); } - let _UpdateField: (id: string, diff: any) => void = errorFunc; - let _CreateField: (field: RefField) => void = errorFunc; + let _UpdateField: (id: string, diff: serverOpType) => void = errorFunc; + let _CreateField: (field: Doc) => void = errorFunc; - export function AddServerHandler(socket: Socket, message: Message, handler: (args: T) => any) { + export function AddServerHandler(socket: Socket, message: Message, handler: (args: T) => void) { socket.on(message.Message, Utils.loggingCallback('Incoming', handler, message.Name)); } export function Emit(socket: Socket, message: Message, args: T) { // log('Emit', message.Name, args, false); socket.emit(message.Message, args); } - export function EmitCallback(socket: Socket, message: Message, args: T): Promise; - export function EmitCallback(socket: Socket, message: Message, args: T, fn: (args: any) => any): void; - export function EmitCallback(socket: Socket, message: Message, args: T, fn?: (args: any) => any): void | Promise { + export function EmitCallback(socket: Socket, message: Message, args: T): Promise; + export function EmitCallback(socket: Socket, message: Message, args: T, fn: (args: unknown) => unknown): void; + export function EmitCallback(socket: Socket, message: Message, args: T, fn?: (args: unknown) => unknown): void | Promise { // log('Emit', message.Name, args, false); if (fn) { socket.emit(message.Message, args, Utils.loggingCallback('Receiving', fn, message.Name)); } else { - return new Promise(res => { + return new Promise(res => { socket.emit(message.Message, args, Utils.loggingCallback('Receiving', res, message.Name)); }); } @@ -99,7 +98,7 @@ export namespace DocServer { return ClientUtils.CurrentUserEmail() === 'guest' ? WriteMode.LivePlayground : fieldWriteModes[field] || WriteMode.Default; } - export function registerDocWithCachedUpdate(doc: Doc, field: string, oldValue: any) { + export function registerDocWithCachedUpdate(doc: Doc, field: string, oldValue: FieldType) { let list = docsWithUpdates[field]; if (!list) { list = docsWithUpdates[field] = new Set(); @@ -203,7 +202,7 @@ export namespace DocServer { * the server if the document has not been cached. * @param id the id of the requested document */ - const _GetRefFieldImpl = (id: string, force: boolean = false): Promise> => { + const _GetRefFieldImpl = (id: string, force: boolean = false): Promise> => { // an initial pass through the cache to determine whether the document needs to be fetched, // is already in the process of being fetched or already exists in the // cache @@ -221,7 +220,7 @@ export namespace DocServer { // future .proto calls on the Doc won't have to go farther than the cache to get their actual value. const deserializeField = getSerializedField.then(async fieldJson => { // deserialize - const field = await SerializationHelper.Deserialize(fieldJson); + const field = (await SerializationHelper.Deserialize(fieldJson)) as Doc; if (force && field && cached instanceof Doc) { cached[UpdatingFromServer] = true; Array.from(Object.keys(field)).forEach(key => { @@ -247,7 +246,7 @@ export namespace DocServer { // here, indicate that the document associated with this id is currently // being retrieved and cached !force && (_cache[id] = deserializeField); - return force ? (cached as any) : deserializeField; + return force ? (cached instanceof Promise ? cached : new Promise(res => res(cached))) : deserializeField; } if (cached instanceof Promise) { // BEING RETRIEVED AND CACHED => some other caller previously (likely recently) called GetRefField(s), @@ -261,7 +260,7 @@ export namespace DocServer { // (field instanceof Doc) && fetchProto(field); ); }; - const _GetCachedRefFieldImpl = (id: string): Opt => { + const _GetCachedRefFieldImpl = (id: string): Opt => { const cached = _cache[id]; if (cached !== undefined && !(cached instanceof Promise)) { return cached; @@ -269,174 +268,102 @@ export namespace DocServer { return undefined; }; - let _GetRefField: (id: string, force: boolean) => Promise> = errorFunc; - let _GetCachedRefField: (id: string) => Opt = errorFunc; + let _GetRefField: (id: string, force: boolean) => Promise> = errorFunc; + let _GetCachedRefField: (id: string) => Opt = errorFunc; - export function GetRefField(id: string, force = false): Promise> { + export function GetRefField(id: string, force = false): Promise> { return _GetRefField(id, force); } - export function GetCachedRefField(id: string): Opt { + export function GetCachedRefField(id: string): Opt { return _GetCachedRefField(id); } - export async function getYoutubeChannels() { - return DocServer.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.Channels }); - } - - export function getYoutubeVideos(videoTitle: string, callBack: (videos: any[]) => void) { - DocServer.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.SearchVideo, userInput: videoTitle }, callBack); - } - - export function getYoutubeVideoDetails(videoIds: string, callBack: (videoDetails: any[]) => void) { - DocServer.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.VideoDetails, videoIds: videoIds }, callBack); - } - /** * Given a list of Doc GUIDs, this utility function will asynchronously attempt to each id's associated * field, first looking in the RefField cache and then communicating with * the server if the document has not been cached. * @param ids the ids that map to the reqested documents */ - const _GetRefFieldsImpl = async (ids: string[]): Promise<{ [id: string]: Opt }> => { - const requestedIds: string[] = []; - const promises: Promise[] = []; - - let defaultRes: any; - const defaultPromise = new Promise(res => { - defaultRes = res; + const _GetRefFieldsImpl = async (ids: string[]): Promise>> => { + const uncachedRequestedIds: string[] = []; + const deserializeDocPromises: Promise>[] = []; + + // setup a Promise that we will resolve after all cached Docs have been acquired. + let allCachesFilledResolver!: (value: Opt | PromiseLike>) => void; + const allCachesFilledPromise = new Promise>(res => { + allCachesFilledResolver = res; }); - const defaultPromises: { p: Promise; id: string }[] = []; - // 1) an initial pass through the cache to determine - // i) which documents need to be fetched - // ii) which are already in the process of being fetched - // iii) which already exist in the cache + + const fetchDocPromises: Map>> = new Map(); // { p: Promise; id: string }[] = []; // promises to fetch the value for a requested Doc + // Determine which requested documents need to be fetched // eslint-disable-next-line no-restricted-syntax for (const id of ids.filter(filterid => filterid)) { - const cached = _cache[id]; - if (cached === undefined) { - defaultPromises.push({ - id, - // eslint-disable-next-line no-loop-func - p: (_cache[id] = new Promise(res => { - defaultPromise.then(() => res(_cache[id])); - })), - }); - // NOT CACHED => we'll have to send a request to the server - requestedIds.push(id); - } else if (cached instanceof Promise) { - // BEING RETRIEVED AND CACHED => some other caller previously (likely recently) called GetRefField(s), - // and requested one of the documents I'm looking for. Shouldn't fetch again, just - // wait until this promise is resolved (see 7) - promises.push(cached); - // waitingIds.push(id); - } else { - // CACHED => great, let's just add it to the field map - // map[id] = cached; + if (_cache[id] === undefined) { + // EMPTY CACHE - make promise that we resolve after all batch-requested Docs have been fetched and deserialized and we know we have this Doc + const fetchPromise = new Promise>(res => + allCachesFilledPromise.then(() => { + // if all Docs have been cached, then we can be sure the fetched Doc has been found and cached. So return it to anyone who had been awaiting it. + const cache = _cache[id]; + if (!(cache instanceof Doc)) console.log('CACHE WAS NEVER FILLED!!'); + res(cache instanceof Doc ? cache : undefined); + }) + ); + // eslint-disable-next-line no-loop-func + fetchDocPromises.set(id, (_cache[id] = fetchPromise)); + uncachedRequestedIds.push(id); // add to list of Doc requests from server } + // else CACHED => do nothing, Doc or promise of Doc is already in cache } - if (requestedIds.length) { - // 2) synchronously, we emit a single callback to the server requesting the serialized (i.e. represented by a string) - // fields for the given ids. This returns a promise, which, when resolved, indicates that all the JSON serialized versions of - // the fields have been returned from the server - console.log('Requesting ' + requestedIds.length); - setTimeout(() => - runInAction(() => { - FieldLoader.ServerLoadStatus.requested = requestedIds.length; - }) - ); - const serializedFields = await DocServer.EmitCallback(_socket, MessageStore.GetRefFields, requestedIds); - - // 3) when the serialized RefFields have been received, go head and begin deserializing them into objects. - // Here, once deserialized, we also invoke .proto to 'load' the documents' prototypes, which ensures that all - // future .proto calls on the Doc won't have to go farther than the cache to get their actual value. + if (uncachedRequestedIds.length) { + console.log('Requesting ' + uncachedRequestedIds.length); + setTimeout(action(() => { FieldLoader.ServerLoadStatus.requested = uncachedRequestedIds.length; })); // prettier-ignore + + // Synchronously emit a single server request for the serialized (i.e. represented by a string) Doc ids + // This returns a promise, that resolves when all the JSON serialized Docs have been retrieved + const serializedFields = (await DocServer.EmitCallback(_socket, MessageStore.GetRefFields, uncachedRequestedIds)) as { id: string; fields: unknown[]; __type: string }[]; + let processed = 0; - console.log('deserializing ' + serializedFields.length + ' fields'); + console.log('Retrieved ' + serializedFields.length + ' fields'); + // After the serialized Docs have been received, deserialize them into objects. // eslint-disable-next-line no-restricted-syntax for (const field of serializedFields) { - processed++; - if (processed % 150 === 0) { + // eslint-disable-next-line no-await-in-loop + ++processed % 150 === 0 && + (await new Promise( + res => + setTimeout(action(() => res(FieldLoader.ServerLoadStatus.retrieved = processed))) // prettier-ignore + )); // force loading to yield to splash screen rendering to update progress + + if (fetchDocPromises.has(field.id)) { + // Doc hasn't started deserializing yet - the cache still has the fetch promise // eslint-disable-next-line no-loop-func - runInAction(() => { - FieldLoader.ServerLoadStatus.retrieved = processed; + const deserializePromise = SerializationHelper.Deserialize(field).then((deserialized: unknown) => { + const doc = deserialized as Doc; + // overwrite any fetch or deserialize cache promise with deserialized value. + // fetch promises wait to resolve until after all deserializations; deserialize promises resolve upon deserializaton + if (deserialized !== undefined) _cache[field.id] = doc; + else delete _cache[field.id]; + + return doc; }); - // eslint-disable-next-line no-await-in-loop - await new Promise(res => { - setTimeout(res); - }); // force loading to yield to splash screen rendering to update progress - } - const cached = _cache[field.id]; - if (!cached || (cached instanceof Promise && defaultPromises.some(dp => dp.p === cached))) { - // deserialize - // adds to a list of promises that will be awaited asynchronously - promises.push( - // eslint-disable-next-line no-loop-func - (_cache[field.id] = SerializationHelper.Deserialize(field).then(deserialized => { - // overwrite or delete any promises (that we inserted as flags - // to indicate that the field was in the process of being fetched). Now everything - // should be an actual value within or entirely absent from the cache. - if (deserialized !== undefined) { - _cache[field.id] = deserialized; - } else { - delete _cache[field.id]; - } - const promInd = defaultPromises.findIndex(dp => dp.id === field.id); - promInd !== -1 && defaultPromises.splice(promInd, 1); - return deserialized; - })) - ); - // 4) here, for each of the documents we've requested *ourselves* (i.e. weren't promises or found in the cache) - // we set the value at the field's id to a promise that will resolve to the field. - // When we find that promises exist at keys in the cache, THIS is where they were set, just by some other caller (method). - // The mapping in the .then call ensures that when other callers await these promises, they'll - // get the resolved field - } else if (cached instanceof Promise) { + deserializeDocPromises.push((_cache[field.id] = deserializePromise)); // replace the cache's placeholder fetch promise with the deserializePromise + fetchDocPromises.delete(field.id); + } else if (_cache[field.id] instanceof Promise) { console.log('.'); - // promises.push(cached); - } else if (field) { - // console.log('-'); } } } - await Promise.all(promises); - defaultPromises.forEach(df => delete _cache[df.id]); - defaultRes(); - - // 5) at this point, all fields have a) been returned from the server and b) been deserialized into actual Field objects whose - // prototype documents, if any, have also been fetched and cached. - console.log('Deserialized ' + (requestedIds.length - defaultPromises.length) + ' fields'); - // 6) with this confidence, we can now go through and update the cache at the ids of the fields that - // we explicitly had to fetch. To finish it off, we add whatever value we've come up with for a given - // id to the soon-to-be-returned field mapping. - // ids.forEach(id => (map[id] = _cache[id] as any)); - - // 7) those promises we encountered in the else if of 1), which represent - // other callers having already submitted a request to the server for (a) document(s) - // in which we're interested, must still be awaited so that we can return the proper - // values for those as well. - // - // fortunately, those other callers will also hit their own version of 6) and clean up - // the shared cache when these promises resolve, so all we have to do is... - // const otherCallersFetching = await Promise.all(promises); - // ...extract the RefFields returned from the resolution of those promises and add them to our - // own map. - // waitingIds.forEach((id, index) => (map[id] = otherCallersFetching[index])); - - // now, we return our completed mapping from all of the ids that were passed into the method - // to their actual RefField | undefined values. This return value either becomes the input - // argument to the caller's promise (i.e. GetRefFields(["_id1_", "_id2_", "_id3_"]).then(map => //do something with map...)) - // or it is the direct return result if the promise is awaited (i.e. let fields = await GetRefFields(["_id1_", "_id2_", "_id3_"])). - return ids.reduce( - (map, id) => { - map[id] = _cache[id] as any; - return map; - }, - {} as { [id: string]: Opt } - ); + await Promise.all(deserializeDocPromises); // promise resolves when cache is up-to-date with all requested Docs + Array.from(fetchDocPromises).forEach(([id]) => delete _cache[id]); + allCachesFilledResolver(undefined); // notify anyone who was promised a Doc fron when it was just being fetched (since all requested Docs have now been fetched and deserialized) + + console.log('Deserialized ' + (uncachedRequestedIds.length - fetchDocPromises.size) + ' fields'); + return new Map>(ids.map(id => [id, _cache[id] instanceof Doc ? (_cache[id] as Doc) : undefined]) as [string, Opt][]); }; - let _GetRefFields: (ids: string[]) => Promise<{ [id: string]: Opt }> = errorFunc; + let _GetRefFields: (ids: string[]) => Promise>> = errorFunc; export function GetRefFields(ids: string[]) { return _GetRefFields(ids); @@ -454,12 +381,12 @@ export namespace DocServer { * calling the same function throughout the code base (such as in Util.makeReadonly()) * @param field the [RefField] to be serialized and sent to the server to be stored in the database */ - export function CreateField(field: RefField) { + export function CreateField(field: Doc) { _cacheNeedsUpdate = true; _CreateField(field); } - function _CreateFieldImpl(field: RefField) { + function _CreateFieldImpl(field: Doc) { _cache[field[Id]] = field; const initialState = SerializationHelper.Serialize(field); ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Emit(_socket, MessageStore.CreateField, initialState); @@ -475,22 +402,22 @@ export namespace DocServer { * @param updatedState the new value of the document. At some point, this * should actually be a proper diff, to improve efficiency */ - export function UpdateField(id: string, updatedState: any) { + export function UpdateField(id: string, updatedState: serverOpType) { _UpdateField(id, updatedState); } - function _UpdateFieldImpl(id: string, diff: any) { + function _UpdateFieldImpl(id: string, diff: serverOpType) { !DocServer.Control.isReadOnly() && ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Emit(_socket, MessageStore.UpdateField, { id, diff }); } - function _respondToUpdateImpl(diff: any) { - const { id } = diff; + function _respondToUpdateImpl(change: { id: string; diff: serverOpType }) { + const { id } = change; // to be valid, the Diff object must reference // a document's id if (id === undefined) { return; } - const update = (f: Opt) => { + const update = (f: Opt) => { // if the RefField is absent from the cache or // its promise in the cache resolves to undefined, there // can't be anything to update @@ -500,7 +427,7 @@ export namespace DocServer { // extract this Doc's update handler const handler = f[HandleUpdate]; if (handler) { - handler.call(f, diff.diff); + handler.call(f, change.diff as { $set: { [key: string]: FieldType } } | { $unset: unknown }); } }; // check the cache for the field @@ -536,8 +463,8 @@ export namespace DocServer { const _RespondToUpdate = _respondToUpdateImpl; const _respondToDelete = _respondToDeleteImpl; - function respondToUpdate(diff: any) { - _RespondToUpdate(diff); + function respondToUpdate(change: { id: string; diff: serverOpType }) { + _RespondToUpdate(change); } function respondToDelete(ids: string | string[]) { @@ -548,7 +475,7 @@ export namespace DocServer { _cache = {}; USER_ID = identifier; _socket = io(`${protocol.startsWith('https') ? 'wss' : 'ws'}://${hostname}:${port}`, { transports: ['websocket'], rejectUnauthorized: false }); - _socket.on('connect_error', (err: any) => console.log(err)); + _socket.on('connect_error', (err: unknown) => console.log(err)); // io.connect(`https://7f079dda.ngrok.io`);// if using ngrok, create a special address for the websocket _GetCachedRefField = _GetCachedRefFieldImpl; -- cgit v1.2.3-70-g09d2 From 4d45f8a046ce5300f0b046457a381d219eef3363 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 13 Aug 2024 15:14:05 -0400 Subject: cleaning up database types --- src/client/DocServer.ts | 21 +------- src/client/views/GestureOverlay.tsx | 5 -- src/fields/Doc.ts | 3 +- src/fields/ObjectField.ts | 15 ++++-- src/fields/util.ts | 20 ++++---- src/server/IDatabase.ts | 8 ++-- src/server/MemoryDatabase.ts | 6 +-- src/server/Message.ts | 66 +------------------------- src/server/apis/google/GoogleApiServerUtils.ts | 1 + src/server/database.ts | 12 +++-- src/server/server_Initialization.ts | 7 ++- 11 files changed, 44 insertions(+), 120 deletions(-) (limited to 'src/client/DocServer.ts') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 2bf3a6f9f..33fa928f2 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -8,7 +8,7 @@ import { UpdatingFromServer } from '../fields/DocSymbols'; import { FieldLoader } from '../fields/FieldLoader'; import { HandleUpdate, Id, Parent } from '../fields/FieldSymbols'; import { ObjectField, serverOpType } from '../fields/ObjectField'; -import { GestureContent, Message, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent } from '../server/Message'; +import { Message, MessageStore } from '../server/Message'; import { SerializationHelper } from './util/SerializationHelper'; /** @@ -109,25 +109,6 @@ export namespace DocServer { } } - export namespace Mobile { - export function dispatchGesturePoints(content: GestureContent) { - DocServer.Emit(_socket, MessageStore.GesturePoints, content); - } - - export function dispatchOverlayTrigger(content: MobileInkOverlayContent) { - // _socket.emit("dispatchBoxTrigger"); - DocServer.Emit(_socket, MessageStore.MobileInkOverlayTrigger, content); - } - - export function dispatchOverlayPositionUpdate(content: UpdateMobileInkOverlayPositionContent) { - DocServer.Emit(_socket, MessageStore.UpdateMobileInkOverlayPosition, content); - } - - export function dispatchMobileDocumentUpload(content: MobileDocumentUploadContent) { - DocServer.Emit(_socket, MessageStore.MobileDocumentUpload, content); - } - } - const instructions = 'This page will automatically refresh after this alert is closed. Expect to reconnect after about 30 seconds.'; function alertUser(connectionTerminationReason: string) { switch (connectionTerminationReason) { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index bcd4d1ee5..3a2738c3b 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -21,10 +21,8 @@ import { SetActiveInkColor, SetActiveInkWidth, } from './nodes/DocumentView'; -// import MobileInkOverlay from '../../mobile/MobileInkOverlay'; import { Gestures } from '../../pen-gestures/GestureTypes'; import { GestureUtils } from '../../pen-gestures/GestureUtils'; -// import { MobileInkOverlayContent } from '../../server/Message'; import { InteractionUtils } from '../util/InteractionUtils'; import { ScriptingGlobals } from '../util/ScriptingGlobals'; import { Transform } from '../util/Transform'; @@ -71,8 +69,6 @@ export class GestureOverlay extends ObservableReactComponent(); private _d1: Doc | undefined; private _inkToTextDoc: Doc | undefined; @@ -485,7 +481,6 @@ export class GestureOverlay extends ObservableReactComponent - {/* {this.showMobileInkOverlay ? : null} */} {this.elements}
void | Promise } = {}; public [Initializing]: boolean = false; - public [FieldChanged] = (diff: { op: '$addToSet' | '$remFromSet' | '$set'; items: FieldType[] | undefined; length: number | undefined; hint?: unknown } | undefined, serverOp: serverOpType) => { + public [FieldChanged] = (diff: { op: '$addToSet' | '$remFromSet' | '$set'; items: FieldType[] | undefined; length: number | undefined; hint?: { start: number; deleteCount: number } } | undefined, serverOp: serverOpType) => { if (!this[UpdatingFromServer] || this[ForceServerWrite]) { DocServer.UpdateField(this[Id], serverOp); } diff --git a/src/fields/ObjectField.ts b/src/fields/ObjectField.ts index 21c4af608..5f31208eb 100644 --- a/src/fields/ObjectField.ts +++ b/src/fields/ObjectField.ts @@ -2,11 +2,18 @@ import { ScriptingGlobals } from '../client/util/ScriptingGlobals'; import { Copy, FieldChanged, Parent, ToJavascriptString, ToScriptString, ToString } from './FieldSymbols'; import { RefField } from './RefField'; +export type serializedFieldType = { fieldId: string; heading?: string; __type: string }; +export type serializedFieldsType = { [key: string]: { fields: serializedFieldType[] } }; +export interface serializedDoctype { + readonly id: string; + readonly fields?: serializedFieldsType; +} + export type serverOpType = { - $set?: { [key: string]: unknown }; // + $set?: serializedFieldsType; // $unset?: { [key: string]: unknown }; - $remFromSet?: { [key: string]: unknown }; - $addToSet?: { [key: string]: unknown }; + $remFromSet?: { [key: string]: { fields: serializedFieldType[] } | { deleteCount: number; start: number } | undefined; hint?: { deleteCount: number; start: number } }; + $addToSet?: serializedFieldsType; length?: number; }; export abstract class ObjectField { @@ -15,7 +22,7 @@ export abstract class ObjectField { // eslint-disable-next-line no-use-before-define items: FieldType[] | undefined; length: number | undefined; - hint?: unknown }, + hint?: { deleteCount: number, start: number} }, serverOp?: serverOpType) => void; // eslint-disable-next-line no-use-before-define public [Parent]?: RefField | ObjectField; diff --git a/src/fields/util.ts b/src/fields/util.ts index 69ece82a2..a5c56607c 100644 --- a/src/fields/util.ts +++ b/src/fields/util.ts @@ -8,7 +8,7 @@ import { Doc, DocListCast, FieldType, FieldResult, HierarchyMapping, ReverseHier import { AclAdmin, AclAugment, AclEdit, AclPrivate, DirectLinks, DocAcl, DocData, DocLayout, FieldKeys, ForceServerWrite, Height, Initializing, SelfProxy, UpdatingFromServer, Width } from './DocSymbols'; import { FieldChanged, Id, Parent, ToValue } from './FieldSymbols'; import { List, ListImpl } from './List'; -import { ObjectField } from './ObjectField'; +import { ObjectField, serializedFieldType, serverOpType } from './ObjectField'; import { PrefetchProxy, ProxyField } from './Proxy'; import { RefField } from './RefField'; import { RichTextField } from './RichTextField'; @@ -112,9 +112,9 @@ const _setterImpl = action((target: Doc | ListImpl, prop: string | sy if (writeToServer) { // prettier-ignore - if (value === undefined) + if (value === undefined || value === null) (target as Doc|ObjectField)[FieldChanged]?.(undefined, { $unset: { ['fields.' + prop]: '' } }); - else (target as Doc|ObjectField)[FieldChanged]?.(undefined, { $set: { ['fields.' + prop]: value instanceof ObjectField ? SerializationHelper.Serialize(value) :value}}); + else (target as Doc|ObjectField)[FieldChanged]?.(undefined, { $set: { ['fields.' + prop]: (value instanceof ObjectField ? SerializationHelper.Serialize(value) :value) as { fields: serializedFieldType[]}}}); if (prop === 'author' || prop.toString().startsWith('acl_')) updateCachedAcls(target); } else if (receiver instanceof Doc) { DocServer.registerDocWithCachedUpdate(receiver, prop as string, curValue); @@ -393,15 +393,15 @@ export function deleteProperty(target: Doc | ListImpl, prop: string | // able to undo and redo the partial change. // export function containedFieldChangedHandler(container: ListImpl | Doc, prop: string | number, liveContainedField: ObjectField) { - let lastValue: FieldResult = liveContainedField instanceof ObjectField ? ObjectField.MakeCopy(liveContainedField) : liveContainedField; - return (diff?: { op: '$addToSet' | '$remFromSet' | '$set'; items: (FieldType & { value?: FieldType })[] | undefined; length: number | undefined; hint?: unknown } /* , dummyServerOp?: any */) => { - const serializeItems = () => ({ __type: 'list', fields: diff?.items?.map((item: FieldType) => SerializationHelper.Serialize(item)) }); + let lastValue = ObjectField.MakeCopy(liveContainedField); + return (diff?: { op: '$addToSet' | '$remFromSet' | '$set'; items: (FieldType & { value?: FieldType })[] | undefined; length: number | undefined; hint?: { start: number; deleteCount: number } } /* , dummyServerOp?: any */) => { + const serializeItems = () => ({ __type: 'list', fields: diff?.items?.map((item: FieldType) => SerializationHelper.Serialize(item) as serializedFieldType) ?? [] }); // prettier-ignore - const serverOp = diff?.op === '$addToSet' + const serverOp: serverOpType = diff?.op === '$addToSet' ? { $addToSet: { ['fields.' + prop]: serializeItems() }, length: diff.length } : diff?.op === '$remFromSet' ? { $remFromSet: { ['fields.' + prop]: serializeItems(), hint: diff.hint}, length: diff.length } - : { $set: { ['fields.' + prop]: liveContainedField ? SerializationHelper.Serialize(liveContainedField) as FieldType : undefined } }; + : { $set: { ['fields.' + prop]: SerializationHelper.Serialize(liveContainedField) as {fields: serializedFieldType[]}} }; if (!(container instanceof Doc) || !container[UpdatingFromServer]) { const cont = container as { [key: string | number]: FieldType }; @@ -477,13 +477,13 @@ export function containedFieldChangedHandler(container: ListImpl | Do // console.log('redo list: ' + prop, fieldVal()); // bcz: uncomment to log undo setFieldVal(ObjectField.MakeCopy(newValue)); const containerProp = cont[prop]; - lastValue = containerProp instanceof ObjectField && ObjectField.MakeCopy(containerProp); + if (containerProp instanceof ObjectField) lastValue = ObjectField.MakeCopy(containerProp); }, undo: () => { // console.log('undo list: ' + prop, fieldVal()); // bcz: uncomment to log undo setFieldVal(ObjectField.MakeCopy(prevValue)); const containerProp = cont[prop]; - lastValue = containerProp instanceof ObjectField && ObjectField.MakeCopy(containerProp); + if (containerProp instanceof ObjectField) lastValue = ObjectField.MakeCopy(containerProp); }, prop: 'set list field', }, diff --git a/src/server/IDatabase.ts b/src/server/IDatabase.ts index 2274792b3..481b64d4a 100644 --- a/src/server/IDatabase.ts +++ b/src/server/IDatabase.ts @@ -1,5 +1,5 @@ import * as mongodb from 'mongodb'; -import { Transferable } from './Message'; +import { serializedDoctype } from '../fields/ObjectField'; export const DocumentsCollection = 'documents'; export interface IDatabase { @@ -13,10 +13,10 @@ export interface IDatabase { dropSchema(...schemaNames: string[]): Promise; - insert(value: any, collectionName?: string): Promise; + insert(value: { _id: string }, collectionName?: string): Promise; - getDocument(id: string, fn: (result?: Transferable) => void, collectionName?: string): void; - getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName?: string): void; + getDocument(id: string, fn: (result?: serializedDoctype) => void, collectionName?: string): void; + getDocuments(ids: string[], fn: (result: serializedDoctype[]) => void, collectionName?: string): void; getCollectionNames(): Promise; visit(ids: string[], fn: (result: any) => string[] | Promise, collectionName?: string): Promise; diff --git a/src/server/MemoryDatabase.ts b/src/server/MemoryDatabase.ts index 1432d91c4..b838cb61b 100644 --- a/src/server/MemoryDatabase.ts +++ b/src/server/MemoryDatabase.ts @@ -1,6 +1,6 @@ import * as mongodb from 'mongodb'; +import { serializedDoctype } from '../fields/ObjectField'; import { DocumentsCollection, IDatabase } from './IDatabase'; -import { Transferable } from './Message'; export class MemoryDatabase implements IDatabase { private db: { [collectionName: string]: { [id: string]: any } } = {}; @@ -81,10 +81,10 @@ export class MemoryDatabase implements IDatabase { return Promise.resolve(); } - public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = DocumentsCollection): void { + public getDocument(id: string, fn: (result?: serializedDoctype) => void, collectionName = DocumentsCollection): void { fn(this.getCollection(collectionName)[id]); } - public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = DocumentsCollection): void { + public getDocuments(ids: string[], fn: (result: serializedDoctype[]) => void, collectionName = DocumentsCollection): void { fn(ids.map(id => this.getCollection(collectionName)[id])); } diff --git a/src/server/Message.ts b/src/server/Message.ts index 03150c841..26b41539b 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -1,5 +1,6 @@ import * as uuid from 'uuid'; import { Point } from '../pen-gestures/ndollar'; +import { serverOpType } from '../fields/ObjectField'; function GenerateDeterministicGuid(seed: string): string { return uuid.v5(seed, uuid.v5.URL); @@ -22,52 +23,12 @@ export class Message { } } -export enum Types { - Number, - List, - Key, - Image, - Web, - Document, - Text, - Icon, - RichText, - DocumentReference, - Html, - Video, - Audio, - Ink, - PDF, - Tuple, - Boolean, - Script, - Templates, -} - -export interface Transferable { - readonly id: string; - readonly type: Types; - readonly data?: any; -} - -export enum YoutubeQueryTypes { - Channels, - SearchVideo, - VideoDetails, -} - -export interface YoutubeQueryInput { - readonly type: YoutubeQueryTypes; - readonly userInput?: string; - readonly videoIds?: string; -} - export interface Reference { readonly id: string; } export interface Diff extends Reference { - readonly diff: any; + readonly diff: serverOpType; } export interface GestureContent { @@ -77,23 +38,6 @@ export interface GestureContent { readonly color?: string; } -export interface MobileInkOverlayContent { - readonly enableOverlay: boolean; - readonly width?: number; - readonly height?: number; - readonly text?: string; -} - -export interface UpdateMobileInkOverlayPositionContent { - readonly dx?: number; - readonly dy?: number; - readonly dsize?: number; -} - -export interface MobileDocumentUploadContent { - readonly docId: string; -} - export interface RoomMessage { readonly message: string; readonly room: string; @@ -102,17 +46,11 @@ export interface RoomMessage { export namespace MessageStore { export const Foo = new Message('Foo'); export const Bar = new Message('Bar'); - export const SetField = new Message('Set Field'); // send Transferable (no reply) - export const GetField = new Message('Get Field'); // send string 'id' get Transferable back - export const GetFields = new Message('Get Fields'); // send string[] of 'id' get Transferable[] back export const GetDocument = new Message('Get Document'); export const DeleteAll = new Message('Delete All'); export const ConnectionTerminated = new Message('Connection Terminated'); export const GesturePoints = new Message('Gesture Points'); - export const MobileInkOverlayTrigger = new Message('Trigger Mobile Ink Overlay'); - export const UpdateMobileInkOverlayPosition = new Message('Update Mobile Ink Overlay Position'); - export const MobileDocumentUpload = new Message('Upload Document From Mobile'); export const GetRefField = new Message('Get Ref Field'); export const GetRefFields = new Message('Get Ref Fields'); diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index d3acc968b..47206f415 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -21,6 +21,7 @@ const scope = ['documents.readonly', 'documents', 'presentations', 'presentation * This namespace manages server side authentication for Google API queries, either * from the standard v1 APIs or the Google Photos REST API. */ +// eslint-disable-next-line @typescript-eslint/no-namespace export namespace GoogleApiServerUtils { /** * As we expand out to more Google APIs that are accessible from diff --git a/src/server/database.ts b/src/server/database.ts index ff8584cd7..a93117349 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-namespace */ import * as mongodb from 'mongodb'; import * as mongoose from 'mongoose'; import { Opt } from '../fields/Doc'; @@ -5,11 +6,11 @@ import { emptyFunction, Utils } from '../Utils'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { DocumentsCollection, IDatabase } from './IDatabase'; import { MemoryDatabase } from './MemoryDatabase'; -import { Transferable } from './Message'; import { Upload } from './SharedMediaTypes'; +import { serializedDoctype } from '../fields/ObjectField'; export namespace Database { - export let disconnect: Function; + export let disconnect: () => void; class DocSchema implements mongodb.BSON.Document { _id!: string; @@ -84,6 +85,7 @@ export namespace Database { if (this.db) { const collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; + // eslint-disable-next-line prefer-const let newProm: Promise; const run = (): Promise => new Promise(resolve => { @@ -112,6 +114,7 @@ export namespace Database { if (this.db) { const collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; + // eslint-disable-next-line prefer-const let newProm: Promise; const run = (): Promise => new Promise(resolve => { @@ -196,6 +199,7 @@ export namespace Database { const id = value._id; const collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; + // eslint-disable-next-line prefer-const let newProm: Promise; const run = (): Promise => new Promise(resolve => { @@ -219,7 +223,7 @@ export namespace Database { return undefined; } - public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = DocumentsCollection) { + public getDocument(id: string, fn: (result?: serializedDoctype) => void, collectionName = DocumentsCollection) { if (this.db) { const collection = this.db.collection(collectionName); collection.findOne({ _id: id }).then(resultIn => { @@ -237,7 +241,7 @@ export namespace Database { } } - public async getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = DocumentsCollection) { + public async getDocuments(ids: string[], fn: (result: serializedDoctype[]) => void, collectionName = DocumentsCollection) { if (this.db) { const found = await this.db .collection(collectionName) diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index 9183688c6..2190e27c7 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -29,7 +29,6 @@ import { WebSocket } from './websocket'; export type RouteSetter = (server: RouteManager) => void; // export let disconnect: Function; -// eslint-disable-next-line import/no-mutable-exports export let resolvedServerUrl: string; const week = 7 * 24 * 60 * 60 * 1000; @@ -115,12 +114,12 @@ function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { } function proxyServe(req: any, requrl: string, response: any) { - // eslint-disable-next-line global-require + // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires const htmlBodyMemoryStream = new (require('memorystream'))(); let wasinBrFormat = false; const sendModifiedBody = () => { const header = response.headers['content-encoding']; - const refToCors = (match: any, tag: string, sym: string, href: string) => `${tag}=${sym + resolvedServerUrl}/corsProxy/${href + sym}`; + const refToCors = (match: string, tag: string, sym: string, href: string) => `${tag}=${sym + resolvedServerUrl}/corsProxy/${href + sym}`; // const relpathToCors = (match: any, href: string, offset: any, string: any) => `="${resolvedServerUrl + '/corsProxy/' + decodeURIComponent(req.originalUrl.split('/corsProxy/')[1].match(/https?:\/\/[^\/]*/)?.[0] ?? '') + '/' + href}"`; if (header) { try { @@ -238,7 +237,7 @@ function registerAuthenticationRoutes(server: express.Express) { export default async function InitializeServer(routeSetter: RouteSetter) { const isRelease = determineEnvironment(); const app = buildWithMiddleware(express()); - const compiler = webpack(config as any); + const compiler = webpack(config as webpack.Configuration); // route table managed by express. routes are tested sequentially against each of these map rules. when a match is found, the handler is called to process the request app.use(wdm(compiler, { publicPath: config.output.publicPath })); -- cgit v1.2.3-70-g09d2 From 5960fa9635c28c2b609826005cb7595ec6b9fb75 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 13 Aug 2024 16:31:13 -0400 Subject: fixed bugs introduced cleaning up database list ops. --- src/client/DocServer.ts | 16 ++++++++-------- src/fields/Doc.ts | 2 +- src/server/Message.ts | 2 +- src/server/websocket.ts | 7 ++++++- 4 files changed, 16 insertions(+), 11 deletions(-) (limited to 'src/client/DocServer.ts') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 33fa928f2..c644308b7 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -34,7 +34,7 @@ export namespace DocServer { throw new Error("Can't use DocServer without calling init first"); } let _UpdateField: (id: string, diff: serverOpType) => void = errorFunc; - let _CreateField: (field: Doc) => void = errorFunc; + let _CreateDocField: (field: Doc) => void = errorFunc; export function AddServerHandler(socket: Socket, message: Message, handler: (args: T) => void) { socket.on(message.Message, Utils.loggingCallback('Incoming', handler, message.Name)); @@ -132,7 +132,7 @@ export namespace DocServer { export function makeReadOnly() { if (!_isReadOnly) { _isReadOnly = true; - _CreateField = field => { + _CreateDocField = field => { _cache[field[Id]] = field; }; _UpdateField = emptyFunction; @@ -357,20 +357,20 @@ export namespace DocServer { } /** - * A wrapper around the function local variable _createField. + * A wrapper around the function local variable _CreateDocField. * This allows us to swap in different executions while comfortably * calling the same function throughout the code base (such as in Util.makeReadonly()) * @param field the [RefField] to be serialized and sent to the server to be stored in the database */ - export function CreateField(field: Doc) { + export function CreateDocField(field: Doc) { _cacheNeedsUpdate = true; - _CreateField(field); + _CreateDocField(field); } - function _CreateFieldImpl(field: Doc) { + function _CreateDocFieldImpl(field: Doc) { _cache[field[Id]] = field; const initialState = SerializationHelper.Serialize(field); - ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Emit(_socket, MessageStore.CreateField, initialState); + ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Emit(_socket, MessageStore.CreateDocField, initialState); } // NOTIFY THE SERVER OF AN UPDATE TO A DOC'S STATE @@ -462,7 +462,7 @@ export namespace DocServer { _GetCachedRefField = _GetCachedRefFieldImpl; SetObjGetRefField((_GetRefField = _GetRefFieldImpl)); SetObjGetRefFields((_GetRefFields = _GetRefFieldsImpl)); - _CreateField = _CreateFieldImpl; + _CreateDocField = _CreateDocFieldImpl; _UpdateField = _UpdateFieldImpl; /** diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index ad7609895..2ae9dbf02 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -329,7 +329,7 @@ export class Doc extends RefField { }); this[SelfProxy] = docProxy; if (!id || forceSave) { - DocServer.CreateField(docProxy); + DocServer.CreateDocField(docProxy); } // eslint-disable-next-line no-constructor-return return docProxy; // need to return the proxy from the constructor so that all our added fields will get called diff --git a/src/server/Message.ts b/src/server/Message.ts index 4599708c9..b904a5ba3 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -56,7 +56,7 @@ export namespace MessageStore { export const GetRefField = new Message('Get Ref Field'); export const GetRefFields = new Message('Get Ref Fields'); export const UpdateField = new Message('Update Ref Field'); - export const CreateField = new Message('Create Ref Field'); + export const CreateDocField = new Message('Create Ref Field'); export const DeleteField = new Message('Delete field'); export const DeleteFields = new Message('Delete fields'); diff --git a/src/server/websocket.ts b/src/server/websocket.ts index f10455680..f588151a5 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -240,7 +240,7 @@ export namespace WebSocket { // if the client and server have different versions of the data after // deletion, they will have different lengths and the server will // send its version of the data to the client - const sendBack = diff.diff.length !== remListItems.length; + const sendBack = diff.diff.length !== diff.diff.$set[updatefield].fields.length; delete diff.diff.length; Database.Instance.update( diff.id, @@ -306,6 +306,10 @@ export namespace WebSocket { }); } + function CreateDocField(newValue: serializedDoctype) { + Database.Instance.insert(newValue); + } + export async function initialize(isRelease: boolean, credentials: SecureContextOptions) { let io: Server; if (isRelease) { @@ -367,6 +371,7 @@ export namespace WebSocket { ServerUtils.AddServerHandler(socket, MessageStore.DeleteAll, () => doDelete(false)); } + ServerUtils.AddServerHandler(socket, MessageStore.CreateDocField, CreateDocField); ServerUtils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); ServerUtils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); ServerUtils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); -- cgit v1.2.3-70-g09d2