From 4d6da44b4eb484d79fc263e5d36c02ea50265767 Mon Sep 17 00:00:00 2001 From: Andy Rickert Date: Wed, 12 Aug 2020 03:55:26 -0400 Subject: filtering in solr and pagination --- .../views/collections/CollectionSchemaHeaders.tsx | 13 +- src/client/views/collections/CollectionSubView.tsx | 2 - src/client/views/search/SearchBox.tsx | 269 +++++++++++++-------- src/fields/Doc.ts | 1 + 4 files changed, 177 insertions(+), 108 deletions(-) (limited to 'src') diff --git a/src/client/views/collections/CollectionSchemaHeaders.tsx b/src/client/views/collections/CollectionSchemaHeaders.tsx index c1c4eb628..fc2463c97 100644 --- a/src/client/views/collections/CollectionSchemaHeaders.tsx +++ b/src/client/views/collections/CollectionSchemaHeaders.tsx @@ -387,8 +387,7 @@ export class KeysDropdown extends React.Component { }); // if search term does not already exist as a group type, give option to create new group type - console.log("Start here"); - console.log(this._key, this._searchTerm.slice(0, this._key.length)); + if (this._key !== this._searchTerm.slice(0, this._key.length)) { console.log("little further"); if (!exactFound && this._searchTerm !== "" && this.props.canAddNew) { @@ -414,16 +413,20 @@ export class KeysDropdown extends React.Component { } return options; } + + docSafe: Doc[] = [] + @action renderFilterOptions = (): JSX.Element[] | JSX.Element => { - console.log("we here"); if (!this._isOpen) { this.defaultMenuHeight = 0; return <>; } let keyOptions: string[] = []; - let docs = DocListCast(this.props.dataDoc![this.props.fieldKey!]); - + if (this.docSafe.length === 0) { + this.docSafe = DocListCast(this.props.dataDoc![this.props.fieldKey!]); + } + let docs = this.docSafe; docs.forEach((doc) => { const key = StrCast(doc[this._key]); if (keyOptions.includes(key) === false) { diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 806ae8ffa..a4493897c 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -172,8 +172,6 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T, moreProps?: }); return docsforFilter; } - console.log("you fool"); - console.log(childDocs); const docRangeFilters = this.props.ignoreFields?.includes("_docRangeFilters") ? [] : Cast(this.props.Document._docRangeFilters, listSpec("string"), []); return this.props.Document.dontRegisterView ? childDocs : DocUtils.FilterDocs(childDocs, this.docFilters(), docRangeFilters, viewSpecScript); } diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index da298e54c..39dd8d4bf 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -1,6 +1,6 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { Tooltip } from '@material-ui/core'; -import { action, computed, observable, runInAction } from 'mobx'; +import { action, computed, observable, runInAction, reaction, IReactionDisposer } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import * as rp from 'request-promise'; @@ -24,6 +24,8 @@ import { ViewBoxBaseComponent } from "../DocComponent"; import { DocumentView } from '../nodes/DocumentView'; import { FieldView, FieldViewProps } from '../nodes/FieldView'; import "./SearchBox.scss"; +import { stringifyUrl } from "query-string"; +import { filterSeries } from "async"; export const searchSchema = createSchema({ id: "string", @@ -86,7 +88,19 @@ export class SearchBox extends ViewBoxBaseComponent { + this._disposers.filters = reaction(() => Cast(this.props.Document._docFilters, listSpec("string")), // if a link is deleted, then remove all hyperlinks that reference it from the text's marks + newFilters => { + if (this.searchFullDB) { + runInAction(() => this._pageStart = 0); + this.submitSearch(); + // newFilters?.forEach(f => { + // console.log(f); + // }) + } + }); if (this.setupButtons === false) { runInAction(() => this.setupButtons = true); @@ -113,6 +127,10 @@ export class SearchBox extends ViewBoxBaseComponent disposer?.()); + } + @action getViews = (doc: Doc) => SearchUtil.GetViewsOfDocument(doc) @@ -123,8 +141,6 @@ export class SearchBox extends ViewBoxBaseComponent) { this.layoutDoc._searchString = e.target.value; this.newsearchstring = e.target.value; - - if (e.target.value === "") { if (this.currentSelectedCollection !== undefined) { let newarray: Doc[] = []; @@ -176,6 +192,7 @@ export class SearchBox extends ViewBoxBaseComponent { if (e.key === "Enter") { this.layoutDoc._searchString = this.newsearchstring; + runInAction(() => this._pageStart = 0); if (StrCast(this.layoutDoc._searchString) !== "" || !this.searchFullDB) { runInAction(() => this.open = true); @@ -228,26 +245,102 @@ export class SearchBox extends ViewBoxBaseComponent { + i === 0 ? newWords.push(key + mod + word) : newWords.push("&& " + key + mod + word) + }); + let v = "(" + newWords.join(" ") + ")"; + query = query + " && " + v; + } + else { + for (let i = 0; i < values.length; i++) { + let mod = "_t:" + const newWords: string[] = []; + const oldWords = values[i].split(" "); + oldWords.forEach((word, i) => { + i === 0 ? newWords.push(key + mod + word) : newWords.push("&& " + key + mod + word) + }); + let v = "(" + newWords.join(" ") + ")"; + if (i === 0) { + query = query + " && (" + v; + if (values.length === 1) { + query = query + ")"; + } + } + else if (i === values.length - 1) { + query = query + " " + v + ")"; + } + else { + query = query + " " + v; + } + } + } } + + + // let limit = typepos.length + // typepos.forEach(i => { + // if (i === 0) { + // if (i + 1 === limit) { + // query = query + " && " + filters[i] + "_t:" + filters; + // } + // else if (filters[i] === filters[i + 3]) { + // query = query + " && (" + filters[i] + "_t:" + filters; + // } + // else { + // query = query + " && " + filters[i] + "_t:" + filters; + // } + + // } + // else if (i + 3 > filters.length) { + + // } + // else { + + // } + + // }); + + // query = this.applyBasicFieldFilters(query); + + + + query = query.replace(/-\s+/g, ''); + query = query.replace(/-/g, ""); + console.log(query); return query; } @@ -400,15 +493,9 @@ export class SearchBox extends ViewBoxBaseComponent { if (d.data !== undefined) { d._searchDocs = new List(); - d._docFilters = new List(); + //d._docFilters = new List(); const newdocs = DocListCast(d.data); newdocs.forEach((newdoc) => { newarray.push(newdoc); @@ -463,7 +549,7 @@ export class SearchBox extends ViewBoxBaseComponent([]); @@ -471,9 +557,9 @@ export class SearchBox extends ViewBoxBaseComponent { Doc.UnBrushDoc(result[0]); result[0].searchMatch = undefined; @@ -542,11 +628,12 @@ export class SearchBox extends ViewBoxBaseComponent { while (this._results.length <= this._endIndex && (this._numTotalResults === -1 || this._maxSearchIndex < this._numTotalResults)) { - this._curRequest = SearchUtil.Search(query, true, { fq: this.filterQuery, start: this._maxSearchIndex, rows: this.NumResults, hl: true, "hl.fl": "*", }).then(action(async (res: SearchUtil.DocSearchResult) => { + this._curRequest = SearchUtil.Search(query, true, { fq: this.filterQuery, start: this._maxSearchIndex, rows: 10000000, hl: true, "hl.fl": "*", }).then(action(async (res: SearchUtil.DocSearchResult) => { // happens at the beginning if (res.numFound !== this._numTotalResults && this._numTotalResults === -1) { this._numTotalResults = res.numFound; } + console.log(res.numFound); const highlighting = res.highlighting || {}; const highlightList = res.docs.map(doc => highlighting[doc[Id]]); const lines = new Map(); @@ -558,31 +645,23 @@ export class SearchBox extends ViewBoxBaseComponent { filteredDocs.forEach((doc, i) => { + console.log(i); const index = this._resultsSet.get(doc); const highlight = highlights[doc[Id]]; const line = lines.get(doc[Id]) || []; const hlights = highlight ? Object.keys(highlight).map(key => key.substring(0, key.length - 2)).filter(k => k) : []; doc ? console.log(Cast(doc.context, Doc)) : null; - if (this.findCommonElements(hlights)) { - } - else { - const layoutresult = Cast(doc.type, "string"); - if (layoutresult) { - if (this.new_buckets[layoutresult] === undefined) { - this.new_buckets[layoutresult] = 1; - } - else { - this.new_buckets[layoutresult] = this.new_buckets[layoutresult] + 1; - } - } - if (index === undefined) { - this._resultsSet.set(doc, this._results.length); - this._results.push([doc, hlights, line]); - } else { - this._results[index][1].push(...hlights); - this._results[index][2].push(...line); - } + // if (this.findCommonElements(hlights)) { + // } + if (index === undefined) { + this._resultsSet.set(doc, this._results.length); + this._results.push([doc, hlights, line]); + console.log(i); + } else { + this._results[index][1].push(...hlights); + this._results[index][2].push(...line); } + }); }); @@ -663,15 +742,16 @@ export class SearchBox extends ViewBoxBaseComponent) => { if (!this._resultsRef.current) return; - this.props.Document._schemaHeaders = new List([]); const scrollY = e ? e.currentTarget.scrollTop : this._resultsRef.current ? this._resultsRef.current.scrollTop : 0; const itemHght = 53; - const startIndex = Math.floor(Math.max(0, scrollY / itemHght)); //const endIndex = Math.ceil(Math.min(this._numTotalResults - 1, startIndex + (this._resultsRef.current.getBoundingClientRect().height / itemHght))); const endIndex = 30; //this._endIndex = endIndex === -1 ? 12 : endIndex; @@ -699,62 +779,42 @@ export class SearchBox extends ViewBoxBaseComponent(this._numTotalResults === -1 ? 0 : this._numTotalResults); } - for (let i = 0; i < this._numTotalResults; i++) { + let max = this._pageStart + this._pageCount; + max > this._results.length ? max = this._results.length : console.log(""); + for (let i = this._pageStart; i < max; i++) { //if the index is out of the window then put a placeholder in //should ones that have already been found get set to placeholders? - if (i < startIndex || i > endIndex) { - if (this._isSearch[i] !== "placeholder") { - this._isSearch[i] = "placeholder"; - this._isSorted[i] = "placeholder"; - this._visibleElements[i] =
Loading...
; - } - } - else { - if (this._isSearch[i] !== "search") { - let result: [Doc, string[], string[]] | undefined = undefined; - if (i >= this._results.length) { - this.getResults(StrCast(this.layoutDoc._searchString)); - if (i < this._results.length) result = this._results[i]; - if (result) { - const highlights = Array.from([...Array.from(new Set(result[1]).values())]); - const lines = new List(result[2]); - result[0].lines = lines; - result[0].highlighting = highlights.join(", "); - highlights.forEach((item) => headers.add(item)); - this._visibleDocuments[i] = result[0]; - this._isSearch[i] = "search"; - Doc.BrushDoc(result[0]); - result[0].searchMatch = true; - Doc.AddDocToList(this.dataDoc, this.props.fieldKey, result[0]); - this.children++; - } - } - else { - result = this._results[i]; - if (result) { - const highlights = Array.from([...Array.from(new Set(result[1]).values())]); - const lines = new List(result[2]); - highlights.forEach((item) => headers.add(item)); - result[0].lines = lines; - result[0].highlighting = highlights.join(", "); - result[0].searchMatch = true; - if (i < this._visibleDocuments.length) { - this._visibleDocuments[i] = result[0]; - this._isSearch[i] = "search"; - Doc.BrushDoc(result[0]); - Doc.AddDocToList(this.dataDoc, this.props.fieldKey, result[0]); - this.children++; - } - } + + if (this._isSearch[i] !== "search") { + let result: [Doc, string[], string[]] | undefined = undefined; + + result = this._results[i]; + if (result) { + const highlights = Array.from([...Array.from(new Set(result[1]).values())]); + const lines = new List(result[2]); + highlights.forEach((item) => headers.add(item)); + result[0].lines = lines; + result[0].highlighting = highlights.join(", "); + result[0].searchMatch = true; + if (i < this._visibleDocuments.length) { + this._visibleDocuments[i] = result[0]; + this._isSearch[i] = "search"; + Doc.BrushDoc(result[0]); + Doc.AddDocToList(this.dataDoc, this.props.fieldKey, result[0]); + this.children++; } + } + } } const schemaheaders: SchemaHeaderField[] = []; this.headerscale = headers.size; headers.forEach((item) => schemaheaders.push(new SchemaHeaderField(item, "#f1efeb"))); this.headercount = schemaheaders.length; - this.props.Document._schemaHeaders = new List(schemaheaders); + if (Cast(this.props.Document._docFilters, listSpec("string"), []).length === 0) { + this.props.Document._schemaHeaders = new List(schemaheaders); + } if (this._maxSearchIndex >= this._numTotalResults) { this._visibleElements.length = this._results.length; this._visibleDocuments.length = this._results.length; @@ -799,6 +859,11 @@ export class SearchBox extends ViewBoxBaseComponent([]); + this.resultsScrolled(); + } render() { this.props.Document._chromeStatus === "disabled"; this.props.Document._searchDoc = true; @@ -807,14 +872,16 @@ export class SearchBox extends ViewBoxBaseComponent 5 ? length = 1076 : length = cols * 205 + 51; let height = 0; const rows = this.children; - rows > 6 ? height = 31 + 31 * 6 : height = 31 * rows + 31; + height = 31 + 31 * 6; return (
{Doc.CurrentUserEmail}
drag search results as collection
} >
- StrCast(this.layoutDoc._searchString) ? this.startDragCollection() : undefined)} icon={"search"} size="lg" + {/* StrCast(this.layoutDoc._searchString) ? this.startDragCollection() : undefined)} icon={"search"} size="lg" + style={{ cursor: "hand", color: "black", padding: 1, left: 35, position: "relative" }} /> */} + { this.newpage(); }} icon={"search"} size="lg" style={{ cursor: "hand", color: "black", padding: 1, left: 35, position: "relative" }} />
diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index b535fea5a..927ff4422 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -1044,6 +1044,7 @@ export namespace Doc { if (docFilters[i] === key && (docFilters[i + 1] === value || modifiers === "match")) { if (docFilters[i + 2] === modifiers && modifiers && docFilters[i + 1] === value) return; docFilters.splice(i, 3); + container._docFilters = new List(docFilters); break; } } -- cgit v1.2.3-70-g09d2