path: root/src/client/util/SearchUtil.ts

import * as rp from 'request-promise';
import { DocServer } from '../DocServer';
import { Doc, DocListCast, Field, Opt } from '../../fields/Doc';
import { Id } from '../../fields/FieldSymbols';
import { Utils } from '../../Utils';
import { DocumentType } from '../documents/DocumentTypes';
import { StrCast } from '../../fields/Types';

export namespace SearchUtil {
    export type HighlightingResult = { [id: string]: { [key: string]: string[] } };

    export function SearchCollection(rootDoc: Opt<Doc>, query: string) {
        const blockedTypes = [DocumentType.PRESELEMENT, DocumentType.CONFIG, DocumentType.KVP, DocumentType.FONTICON, DocumentType.BUTTON, DocumentType.SCRIPTING];
        const blockedKeys = [
            'x',
            'y',
            'proto',
            'width',
            'layout_autoHeight',
            'acl-Override',
            'acl-Guest',
            'embedContainer',
            'zIndex',
            'height',
            'text_scrollHeight',
            'text_height',
            'cloneFieldFilter',
            'isDataDoc',
            'text_annotations',
            'dragFactory_count',
            'text_noTemplate',
            'proto_embeddings',
            'isSystem',
            'layout_fieldKey',
            'isBaseProto',
            'xMargin',
            'yMargin',
            'links',
            'layout',
            'layout_keyValue',
            'layout_fitWidth',
            'type_collection',
            'title_custom',
            'freeform_panX',
            'freeform_panY',
            'freeform_scale',
        ];
        query = query.toLowerCase();

        const results = new Map<Doc, string[]>();
        if (rootDoc) {
            const docs = DocListCast(rootDoc[Doc.LayoutFieldKey(rootDoc)]);
            const docIDs: String[] = [];
            SearchUtil.foreachRecursiveDoc(docs, (depth: number, doc: Doc) => {
                const dtype = StrCast(doc.type) as DocumentType;
                if (dtype && !blockedTypes.includes(dtype) && !docIDs.includes(doc[Id]) && depth >= 0) {
                    const hlights = new Set<string>();
                    SearchUtil.documentKeys(doc).forEach(
                        key =>
                            (Field.toString(doc[key] as Field) + Field.toScriptString(doc[key] as Field))
                                .toLowerCase() //
                                .includes(query) && hlights.add(key)
                    );
                    blockedKeys.forEach(key => hlights.delete(key));

                    if (Array.from(hlights.keys()).length > 0) {
                        results.set(doc, Array.from(hlights.keys()));
                    }
                }
                docIDs.push(doc[Id]);
            });
        }
        return results;
    }
    /**
     * @param {Doc} doc - doc for which keys are returned
     *
     * This method returns a list of a document doc's keys.
     */
    export function documentKeys(doc: Doc) {
        const keys: { [key: string]: boolean } = {};
        Doc.GetAllPrototypes(doc).map(proto => Object.keys(proto).forEach(key => (keys[key] = false)));
        return Array.from(Object.keys(keys));
    }

    /**
     * @param {Doc[]} docs - docs to be searched through recursively
     * @param {number, Doc => void} func - function to be called on each doc
     *
     * This method iterates through an array of docs and all docs within those docs, calling
     * the function func on each doc.
     */
    export function foreachRecursiveDoc(docs: Doc[], func: (depth: number, doc: Doc) => void) {
        let newarray: Doc[] = [];
        var depth = 0;
        const visited: Doc[] = [];
        while (docs.length > 0) {
            newarray = [];
            docs.filter(d => d && !visited.includes(d)).forEach(d => {
                visited.push(d);
                const fieldKey = Doc.LayoutFieldKey(d);
                const annos = !Field.toString(Doc.LayoutField(d) as Field).includes('CollectionView');
                const data = d[annos ? fieldKey + '_annotations' : fieldKey];
                data && newarray.push(...DocListCast(data));
                const sidebar = d[fieldKey + '_sidebar'];
                sidebar && newarray.push(...DocListCast(sidebar));
                func(depth, d);
            });
            docs = newarray;
            depth++;
        }
    }
    export interface IdSearchResult {
        ids: string[];
        lines: string[][];
        numFound: number;
        highlighting: HighlightingResult | undefined;
    }

    export interface DocSearchResult {
        docs: Doc[];
        lines: string[][];
        numFound: number;
        highlighting: HighlightingResult | undefined;
    }

    export interface SearchParams {
        hl?: string;
        'hl.fl'?: string;
        start?: number;
        rows?: number;
        fq?: string;
        sort?: string;
        allowEmbeddings?: boolean;
        onlyEmbeddings?: boolean;
        facet?: string;
        'facet.field'?: string;
    }
    export function Search(query: string, returnDocs: true, options?: SearchParams): Promise<DocSearchResult>;
    export function Search(query: string, returnDocs: false, options?: SearchParams): Promise<IdSearchResult>;
    export async function Search(query: string, returnDocs: boolean, options: SearchParams = {}) {
        query = query || '*'; //If we just have a filter query, search for * as the query
        const rpquery = Utils.prepend('/dashsearch');
        let replacedQuery = query.replace(/type_t:([^ )])/g, (substring, arg) => `{!join from=id to=proto_i}*:* AND ${arg}`);
        if (options.onlyEmbeddings) {
            const header = query.match(/_[atnb]?:/) ? replacedQuery : 'DEFAULT:' + replacedQuery;
            replacedQuery = `{!join from=id to=proto_i}* AND ${header}`;
        }
        //console.log("Q: " + replacedQuery + " fq: " + options.fq);
        const gotten = await rp.get(rpquery, { qs: { ...options, q: replacedQuery } });
        const result: IdSearchResult = gotten.startsWith('<') ? { ids: [], docs: [], numFound: 0, lines: [] } : JSON.parse(gotten);
        if (!returnDocs) {
            return result;
        }

        const { ids, highlighting } = result;

        const txtresult =
            query !== '*' &&
            JSON.parse(
                await rp.get(Utils.prepend('/textsearch'), {
                    qs: { ...options, q: query.replace(/^[ \+\?\*\|]*/, '') }, // a leading '+' leads to a server crash since findInFiles doesn't handle regex failures
                })
            );

        const fileids = txtresult ? txtresult.ids : [];
        const newIds: string[] = [];
        const newLines: string[][] = [];
        // bcz: we stopped storing fileUpload id's, so this won't find anything
        // if (fileids) {
        //     await Promise.all(
        //         fileids.map(async (tr: string, i: number) => {
        //             const docQuery = 'fileUpload_t:' + tr.substr(0, 7); //If we just have a filter query, search for * as the query
        //             const docResult = JSON.parse(await rp.get(Utils.prepend('/dashsearch'), { qs: { ...options, q: docQuery } }));
        //             newIds.push(...docResult.ids);
        //             newLines.push(...docResult.ids.map((dr: any) => txtresult.lines[i]));
        //         })
        //     );
        // }

        const theDocs: Doc[] = [];
        const theLines: string[][] = [];
        const textDocMap = await DocServer.GetRefFields(newIds);
        const textDocs = newIds.map((id: string) => textDocMap[id]).map(doc => doc as Doc);
        for (let i = 0; i < textDocs.length; i++) {
            const testDoc = textDocs[i];
            if (testDoc instanceof Doc && testDoc.type !== DocumentType.KVP && theDocs.findIndex(d => Doc.AreProtosEqual(d, testDoc)) === -1) {
                theDocs.push(Doc.GetProto(testDoc));
                theLines.push(newLines[i].map(line => line.replace(query, query.toUpperCase())));
            }
        }

        const docMap = await DocServer.GetRefFields(ids);
        const docs = ids.map((id: string) => docMap[id]).map(doc => doc as Doc);
        for (let i = 0; i < ids.length; i++) {
            const testDoc = docs[i];
            if (testDoc instanceof Doc && testDoc.type !== DocumentType.KVP && (options.allowEmbeddings || testDoc.proto === undefined || theDocs.findIndex(d => Doc.AreProtosEqual(d, testDoc)) === -1)) {
                theDocs.push(testDoc);
                theLines.push([]);
            } else {
                result.numFound--;
            }
        }

        return { docs: theDocs, numFound: Math.max(0, result.numFound), highlighting, lines: theLines };
    }

    export async function GetEmbeddingsOfDocument(doc: Doc): Promise<Doc[]>;
    export async function GetEmbeddingsOfDocument(doc: Doc, returnDocs: false): Promise<string[]>;
    export async function GetEmbeddingsOfDocument(doc: Doc, returnDocs = true): Promise<Doc[] | string[]> {
        const proto = Doc.GetProto(doc);
        const protoId = proto[Id];
        if (returnDocs) {
            return (await Search('', returnDocs, { fq: `proto_i:"${protoId}"`, allowEmbeddings: true })).docs;
        } else {
            return (await Search('', returnDocs, { fq: `proto_i:"${protoId}"`, allowEmbeddings: true })).ids;
        }
        // return Search(`{!join from=id to=proto_i}id:${protoId}`, true);
    }

    export async function GetViewsOfDocument(doc: Doc): Promise<Doc[]> {
        const results = await Search('', true, { fq: `proto_i:"${doc[Id]}"` });
        return results.docs;
    }

    export async function GetContextsOfDocument(doc: Doc): Promise<{ contexts: Doc[]; embeddingContexts: Doc[] }> {
        const docContexts = (await Search('', true, { fq: `data_l:"${doc[Id]}"` })).docs;
        const embeddings = await GetEmbeddingsOfDocument(doc, false);
        const embeddingContexts = await Promise.all(embeddings.map(doc => Search('', true, { fq: `data_l:"${doc}"` })));
        const contexts = { contexts: docContexts, embeddingContexts: [] as Doc[] };
        embeddingContexts.forEach(result => contexts.embeddingContexts.push(...result.docs));
        return contexts;
    }

    export async function GetContextIdsOfDocument(doc: Doc): Promise<{ contexts: string[]; embeddingContexts: string[] }> {
        const docContexts = (await Search('', false, { fq: `data_l:"${doc[Id]}"` })).ids;
        const embeddings = await GetEmbeddingsOfDocument(doc, false);
        const embeddingContexts = await Promise.all(embeddings.map(doc => Search('', false, { fq: `data_l:"${doc}"` })));
        const contexts = { contexts: docContexts, embeddingContexts: [] as string[] };
        embeddingContexts.forEach(result => contexts.embeddingContexts.push(...result.ids));
        return contexts;
    }

    export async function GetAllDocs() {
        const query = '*';
        const response = await rp.get(Utils.prepend('/dashsearch'), {
            qs: { start: 0, rows: 10000, q: query },
        });
        const result: IdSearchResult = JSON.parse(response);
        const { ids, numFound, highlighting } = result;
        const docMap = await DocServer.GetRefFields(ids);
        const docs: Doc[] = [];
        for (const id of ids) {
            const field = docMap[id];
            if (field instanceof Doc) {
                docs.push(field);
            }
        }
        return docs;
        // const docs = ids.map((id: string) => docMap[id]).filter((doc: any) => doc instanceof Doc);
        // return docs as Doc[];
    }
}