aboutsummaryrefslogtreecommitdiff
path: root/src/client/util/SearchUtil.ts
diff options
context:
space:
mode:
Diffstat (limited to 'src/client/util/SearchUtil.ts')
-rw-r--r--src/client/util/SearchUtil.ts213
1 files changed, 17 insertions, 196 deletions
diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts
index e51770c25..218667d3e 100644
--- a/src/client/util/SearchUtil.ts
+++ b/src/client/util/SearchUtil.ts
@@ -1,53 +1,25 @@
-import * as rp from 'request-promise';
-import { DocServer } from '../DocServer';
+import { ObservableMap } from 'mobx';
import { Doc, DocListCast, Field, Opt } from '../../fields/Doc';
import { Id } from '../../fields/FieldSymbols';
-import { Utils } from '../../Utils';
-import { DocumentType } from '../documents/DocumentTypes';
import { StrCast } from '../../fields/Types';
+import { DocumentType } from '../documents/DocumentTypes';
+import { DocOptions, FInfo } from '../documents/Documents';
export namespace SearchUtil {
export type HighlightingResult = { [id: string]: { [key: string]: string[] } };
- export function SearchCollection(collectionDoc: Opt<Doc>, query: string) {
+ export function SearchCollection(collectionDoc: Opt<Doc>, query: string, matchKeyNames: boolean) {
const blockedTypes = [DocumentType.PRESELEMENT, DocumentType.CONFIG, DocumentType.KVP, DocumentType.FONTICON, DocumentType.BUTTON, DocumentType.SCRIPTING];
- const blockedKeys = [
- 'x',
- 'y',
- 'proto',
- 'width',
- 'layout_autoHeight',
- 'acl-Override',
- 'acl-Guest',
- 'embedContainer',
- 'zIndex',
- 'height',
- 'text_scrollHeight',
- 'text_height',
- 'cloneFieldFilter',
- 'isDataDoc',
- 'text_annotations',
- 'dragFactory_count',
- 'text_noTemplate',
- 'proto_embeddings',
- 'isSystem',
- 'layout_fieldKey',
- 'isBaseProto',
- 'xMargin',
- 'yMargin',
- 'links',
- 'layout',
- 'layout_keyValue',
- 'layout_fitWidth',
- 'type_collection',
- 'title_custom',
- 'freeform_panX',
- 'freeform_panY',
- 'freeform_scale',
- ];
- query = query.toLowerCase();
+ const blockedKeys = matchKeyNames
+ ? []
+ : Object.entries(DocOptions)
+ .filter(([key, info]: [string, FInfo]) => !info?.searchable())
+ .map(([key]) => key);
- const results = new Map<Doc, string[]>();
+ const exact = query.startsWith('=');
+ query = query.toLowerCase().split('=').lastElement();
+
+ const results = new ObservableMap<Doc, string[]>();
if (collectionDoc) {
const docs = DocListCast(collectionDoc[Doc.LayoutFieldKey(collectionDoc)]);
const docIDs: String[] = [];
@@ -56,11 +28,10 @@ export namespace SearchUtil {
if (dtype && !blockedTypes.includes(dtype) && !docIDs.includes(doc[Id]) && depth >= 0) {
const hlights = new Set<string>();
SearchUtil.documentKeys(doc).forEach(
- key =>
- (Field.toString(doc[key] as Field) + Field.toScriptString(doc[key] as Field))
- .toLowerCase() //
- .includes(query) && hlights.add(key)
- );
+ key => (val => (exact ? val === query.toLowerCase() : val.includes(query.toLowerCase())))(
+ matchKeyNames ? key : Field.toString(doc[key] as Field))
+ && hlights.add(key)
+ ); // prettier-ignore
blockedKeys.forEach(key => hlights.delete(key));
if (Array.from(hlights.keys()).length > 0) {
@@ -110,154 +81,4 @@ export namespace SearchUtil {
depth++;
}
}
- export interface IdSearchResult {
- ids: string[];
- lines: string[][];
- numFound: number;
- highlighting: HighlightingResult | undefined;
- }
-
- export interface DocSearchResult {
- docs: Doc[];
- lines: string[][];
- numFound: number;
- highlighting: HighlightingResult | undefined;
- }
-
- export interface SearchParams {
- hl?: string;
- 'hl.fl'?: string;
- start?: number;
- rows?: number;
- fq?: string;
- sort?: string;
- allowEmbeddings?: boolean;
- onlyEmbeddings?: boolean;
- facet?: string;
- 'facet.field'?: string;
- }
- export function Search(query: string, returnDocs: true, options?: SearchParams): Promise<DocSearchResult>;
- export function Search(query: string, returnDocs: false, options?: SearchParams): Promise<IdSearchResult>;
- export async function Search(query: string, returnDocs: boolean, options: SearchParams = {}) {
- query = query || '*'; //If we just have a filter query, search for * as the query
- const rpquery = Utils.prepend('/dashsearch');
- let replacedQuery = query.replace(/type_t:([^ )])/g, (substring, arg) => `{!join from=id to=proto_i}*:* AND ${arg}`);
- if (options.onlyEmbeddings) {
- const header = query.match(/_[atnb]?:/) ? replacedQuery : 'DEFAULT:' + replacedQuery;
- replacedQuery = `{!join from=id to=proto_i}* AND ${header}`;
- }
- //console.log("Q: " + replacedQuery + " fq: " + options.fq);
- const gotten = await rp.get(rpquery, { qs: { ...options, q: replacedQuery } });
- const result: IdSearchResult = gotten.startsWith('<') ? { ids: [], docs: [], numFound: 0, lines: [] } : JSON.parse(gotten);
- if (!returnDocs) {
- return result;
- }
-
- const { ids, highlighting } = result;
-
- const txtresult =
- query !== '*' &&
- JSON.parse(
- await rp.get(Utils.prepend('/textsearch'), {
- qs: { ...options, q: query.replace(/^[ \+\?\*\|]*/, '') }, // a leading '+' leads to a server crash since findInFiles doesn't handle regex failures
- })
- );
-
- const fileids = txtresult ? txtresult.ids : [];
- const newIds: string[] = [];
- const newLines: string[][] = [];
- // bcz: we stopped storing fileUpload id's, so this won't find anything
- // if (fileids) {
- // await Promise.all(
- // fileids.map(async (tr: string, i: number) => {
- // const docQuery = 'fileUpload_t:' + tr.substr(0, 7); //If we just have a filter query, search for * as the query
- // const docResult = JSON.parse(await rp.get(Utils.prepend('/dashsearch'), { qs: { ...options, q: docQuery } }));
- // newIds.push(...docResult.ids);
- // newLines.push(...docResult.ids.map((dr: any) => txtresult.lines[i]));
- // })
- // );
- // }
-
- const theDocs: Doc[] = [];
- const theLines: string[][] = [];
- const textDocMap = await DocServer.GetRefFields(newIds);
- const textDocs = newIds.map((id: string) => textDocMap[id]).map(doc => doc as Doc);
- for (let i = 0; i < textDocs.length; i++) {
- const testDoc = textDocs[i];
- if (testDoc instanceof Doc && testDoc.type !== DocumentType.KVP && theDocs.findIndex(d => Doc.AreProtosEqual(d, testDoc)) === -1) {
- theDocs.push(Doc.GetProto(testDoc));
- theLines.push(newLines[i].map(line => line.replace(query, query.toUpperCase())));
- }
- }
-
- const docMap = await DocServer.GetRefFields(ids);
- const docs = ids.map((id: string) => docMap[id]).map(doc => doc as Doc);
- for (let i = 0; i < ids.length; i++) {
- const testDoc = docs[i];
- if (testDoc instanceof Doc && testDoc.type !== DocumentType.KVP && (options.allowEmbeddings || testDoc.proto === undefined || theDocs.findIndex(d => Doc.AreProtosEqual(d, testDoc)) === -1)) {
- theDocs.push(testDoc);
- theLines.push([]);
- } else {
- result.numFound--;
- }
- }
-
- return { docs: theDocs, numFound: Math.max(0, result.numFound), highlighting, lines: theLines };
- }
-
- export async function GetEmbeddingsOfDocument(doc: Doc): Promise<Doc[]>;
- export async function GetEmbeddingsOfDocument(doc: Doc, returnDocs: false): Promise<string[]>;
- export async function GetEmbeddingsOfDocument(doc: Doc, returnDocs = true): Promise<Doc[] | string[]> {
- const proto = Doc.GetProto(doc);
- const protoId = proto[Id];
- if (returnDocs) {
- return (await Search('', returnDocs, { fq: `proto_i:"${protoId}"`, allowEmbeddings: true })).docs;
- } else {
- return (await Search('', returnDocs, { fq: `proto_i:"${protoId}"`, allowEmbeddings: true })).ids;
- }
- // return Search(`{!join from=id to=proto_i}id:${protoId}`, true);
- }
-
- export async function GetViewsOfDocument(doc: Doc): Promise<Doc[]> {
- const results = await Search('', true, { fq: `proto_i:"${doc[Id]}"` });
- return results.docs;
- }
-
- export async function GetContextsOfDocument(doc: Doc): Promise<{ contexts: Doc[]; embeddingContexts: Doc[] }> {
- const docContexts = (await Search('', true, { fq: `data_l:"${doc[Id]}"` })).docs;
- const embeddings = await GetEmbeddingsOfDocument(doc, false);
- const embeddingContexts = await Promise.all(embeddings.map(doc => Search('', true, { fq: `data_l:"${doc}"` })));
- const contexts = { contexts: docContexts, embeddingContexts: [] as Doc[] };
- embeddingContexts.forEach(result => contexts.embeddingContexts.push(...result.docs));
- return contexts;
- }
-
- export async function GetContextIdsOfDocument(doc: Doc): Promise<{ contexts: string[]; embeddingContexts: string[] }> {
- const docContexts = (await Search('', false, { fq: `data_l:"${doc[Id]}"` })).ids;
- const embeddings = await GetEmbeddingsOfDocument(doc, false);
- const embeddingContexts = await Promise.all(embeddings.map(doc => Search('', false, { fq: `data_l:"${doc}"` })));
- const contexts = { contexts: docContexts, embeddingContexts: [] as string[] };
- embeddingContexts.forEach(result => contexts.embeddingContexts.push(...result.ids));
- return contexts;
- }
-
- export async function GetAllDocs() {
- const query = '*';
- const response = await rp.get(Utils.prepend('/dashsearch'), {
- qs: { start: 0, rows: 10000, q: query },
- });
- const result: IdSearchResult = JSON.parse(response);
- const { ids, numFound, highlighting } = result;
- const docMap = await DocServer.GetRefFields(ids);
- const docs: Doc[] = [];
- for (const id of ids) {
- const field = docMap[id];
- if (field instanceof Doc) {
- docs.push(field);
- }
- }
- return docs;
- // const docs = ids.map((id: string) => docMap[id]).filter((doc: any) => doc instanceof Doc);
- // return docs as Doc[];
- }
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-07 23:08:21 +0000