From 8c801b3c98e1eaae297b0f1682b42fc478a1b887 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Sun, 14 Apr 2019 00:44:02 -0400 Subject: Got a basic version of search working --- src/server/Search.ts | 27 +++++++++++++++++++++++++++ src/server/ServerUtil.ts | 4 ++-- src/server/database.ts | 12 ++++++++++++ src/server/index.ts | 12 +++++++++++- 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 src/server/Search.ts (limited to 'src/server') diff --git a/src/server/Search.ts b/src/server/Search.ts new file mode 100644 index 000000000..f9babc433 --- /dev/null +++ b/src/server/Search.ts @@ -0,0 +1,27 @@ +import * as rp from 'request-promise'; +import { Database } from './database'; + +export class Search { + public static Instance = new Search(); + private url = 'http://localhost:8983/solr/'; + + public updateDocument(document: any): rp.RequestPromise { + return rp.post(this.url + "dash/update/json/docs", { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(document) + }); + } + + public async search(query: string) { + const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { + qs: { + q: query + } + })); + const fields = searchResults.response.docs; + const ids = fields.map((field: any) => field.id); + const docs = await Database.Instance.searchQuery(ids); + const docIds = docs.map((doc: any) => doc._id); + return docIds; + } +} \ No newline at end of file diff --git a/src/server/ServerUtil.ts b/src/server/ServerUtil.ts index 0973f82b1..dc8687b7e 100644 --- a/src/server/ServerUtil.ts +++ b/src/server/ServerUtil.ts @@ -73,9 +73,9 @@ export class ServerUtils { return InkField.FromJson(id, data); case Types.Document: let doc: Document = new Document(id, false); - let fields: [string, string][] = data as [string, string][]; + let fields: { key: string, field: string }[] = data as { key: string, field: string }[]; fields.forEach(element => { - doc._proxies.set(element[0], element[1]); + doc._proxies.set(element.key, element.field); }); return doc; default: diff --git a/src/server/database.ts b/src/server/database.ts index 3290edde0..4011f26bd 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -71,6 +71,18 @@ export class Database { }); } + public searchQuery(ids: string[], collectionName = Database.DocumentsCollection): Promise { + return new Promise(resolve => { + this.db && this.db.collection(collectionName).find({ "data.field": { "$in": ids } }).toArray((err, docs) => { + if (err) { + console.log(err.message); + console.log(err.errmsg); + } + resolve(docs); + }); + }); + } + public print() { console.log("db says hi!"); } diff --git a/src/server/index.ts b/src/server/index.ts index a6fe6fa2c..fef26f78a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -22,7 +22,7 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; -import { MessageStore, Transferable } from "./Message"; +import { MessageStore, Transferable, Types } from "./Message"; import { RouteStore } from './RouteStore'; const app = express(); const config = require('../../webpack.config'); @@ -32,6 +32,7 @@ const serverPort = 4321; import expressFlash = require('express-flash'); import flash = require('connect-flash'); import c = require("crypto"); +import { Search } from './Search'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -118,6 +119,12 @@ app.get("/pull", (req, res) => // GETTERS +app.get("/search", async (req, res) => { + let query = req.query.query || "hello"; + let results = await Search.Instance.search(query); + res.send(results); +}); + // anyone attempting to navigate to localhost at this port will // first have to login addSecureRoute( @@ -258,6 +265,9 @@ function getFields([ids, callback]: [string[], (result: any) => void]) { function setField(socket: Socket, newValue: Transferable) { Database.Instance.update(newValue._id, newValue, () => socket.broadcast.emit(MessageStore.SetField.Message, newValue)); + if (newValue.type === Types.Text) { + Search.Instance.updateDocument({ id: newValue._id, data: (newValue as any).data }); + } } server.listen(serverPort); -- cgit v1.2.3-70-g09d2 From be1976fb0ba33064978ee973993b3a2316cdf43c Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Sun, 14 Apr 2019 01:02:25 -0400 Subject: deleting database now also clears Solr indexes --- src/server/Search.ts | 11 +++++++++++ src/server/index.ts | 7 +++++-- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/src/server/Search.ts b/src/server/Search.ts index f9babc433..7d8602346 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -24,4 +24,15 @@ export class Search { const docIds = docs.map((doc: any) => doc._id); return docIds; } + + public async clear() { + return rp.post(this.url + "dash/update", { + body: { + delete: { + query: "*:*" + } + }, + json: true + }); + } } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index bea84c6ed..cb4268a2d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -11,6 +11,7 @@ import { ObservableMap } from 'mobx'; import * as passport from 'passport'; import * as path from 'path'; import * as request from 'request'; +import * as rp from 'request-promise'; import * as io from 'socket.io'; import { Socket } from 'socket.io'; import * as webpack from 'webpack'; @@ -241,14 +242,16 @@ server.on("connection", function (socket: Socket) { Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); }); -function deleteFields() { - return Database.Instance.deleteAll(); +async function deleteFields() { + await Database.Instance.deleteAll(); + await Search.Instance.clear(); } async function deleteAll() { await Database.Instance.deleteAll(); await Database.Instance.deleteAll('sessions'); await Database.Instance.deleteAll('users'); + await Search.Instance.clear(); } function barReceived(guid: String) { -- cgit v1.2.3-70-g09d2 From 94fa065ae2a0f3ccb6a16141a45f11543add3b63 Mon Sep 17 00:00:00 2001 From: ab Date: Mon, 22 Apr 2019 19:56:49 -0400 Subject: exploratory sprint --- src/server/Search.ts | 1 + src/server/index.ts | 9 +++++++++ 2 files changed, 10 insertions(+) (limited to 'src/server') diff --git a/src/server/Search.ts b/src/server/Search.ts index 7d8602346..bcea03d5c 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -6,6 +6,7 @@ export class Search { private url = 'http://localhost:8983/solr/'; public updateDocument(document: any): rp.RequestPromise { + console.log(JSON.stringify(document)); return rp.post(this.url + "dash/update/json/docs", { headers: { 'content-type': 'application/json' }, body: JSON.stringify(document) diff --git a/src/server/index.ts b/src/server/index.ts index a68dabc0c..b3df90199 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -34,6 +34,7 @@ import expressFlash = require('express-flash'); import flash = require('connect-flash'); import c = require("crypto"); import { Search } from './Search'; +import { debug } from 'util'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -276,6 +277,7 @@ function setField(socket: Socket, newValue: Transferable) { socket.broadcast.emit(MessageStore.SetField.Message, newValue)); if (newValue.type === Types.Text) { Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); + console.log("set field"); } } @@ -286,10 +288,17 @@ function GetRefField([id, callback]: [string, (result?: Transferable) => void]) function UpdateField(socket: Socket, diff: Diff) { Database.Instance.update(diff.id, diff.diff, () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); + //if (diff.diff === Types.Text) { + Search.Instance.updateDocument({ name: "john", burns: "true" }); + Search.Instance.updateDocument({ id: diff.id, data: diff.diff.data }); + //console.log("set field"); + //} + console.log("updated field", diff.diff); } function CreateField(newValue: any) { Database.Instance.insert(newValue, "newDocuments"); + console.log("created field"); } server.listen(serverPort); -- cgit v1.2.3-70-g09d2 From 85ce4c9a3cd665983067d7783e20eb7701376503 Mon Sep 17 00:00:00 2001 From: ab Date: Tue, 23 Apr 2019 19:00:15 -0400 Subject: idk --- src/server/index.ts | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index b3df90199..464d3f68f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -35,6 +35,7 @@ import flash = require('connect-flash'); import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; +import _ = require('lodash'); const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -289,11 +290,32 @@ function UpdateField(socket: Socket, diff: Diff) { Database.Instance.update(diff.id, diff.diff, () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); //if (diff.diff === Types.Text) { - Search.Instance.updateDocument({ name: "john", burns: "true" }); - Search.Instance.updateDocument({ id: diff.id, data: diff.diff.data }); + //Search.Instance.updateDocument({ name: "john", burns: "true" }); + //Search.Instance.updateDocument({ id: diff.id, data: diff.diff.data }); //console.log("set field"); //} - console.log("updated field", diff.diff); + const docid = { id: diff.id }; + const docfield = diff.diff; + console.log("FIELD: ", docfield); + var dynfield = false; + for (var key in docfield) { + const val = docfield[key]; + if (typeof val === 'number') { + key = key + "_n"; + dynfield = true; + } + else if (typeof val === 'string') { + key = key + "_t"; + dynfield = true; + } + console.log(key); + } + var merged = {}; + _.extend(merged, docid, docfield); + if (dynfield) { + console.log("dynamic field detected!"); + Search.Instance.updateDocument(merged); + } } function CreateField(newValue: any) { -- cgit v1.2.3-70-g09d2 From df9fa3e362a6bdd616bc0b46ef9b425cfc5a010d Mon Sep 17 00:00:00 2001 From: Monika Hedman Date: Tue, 30 Apr 2019 16:17:39 -0400 Subject: before pull --- src/client/views/SearchBox.scss | 1 - src/fields/Document.ts | 15 +++++++-------- src/server/Search.ts | 38 ++++++++++++++++++++++++++++++++++++++ src/server/database.ts | 12 ++++++++++++ src/server/index.ts | 19 ++++++++++++++++--- 5 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 src/server/Search.ts (limited to 'src/server') diff --git a/src/client/views/SearchBox.scss b/src/client/views/SearchBox.scss index e1a1de142..92363e681 100644 --- a/src/client/views/SearchBox.scss +++ b/src/client/views/SearchBox.scss @@ -9,7 +9,6 @@ align-items: center; .submit-search { - display: inline-block; text-align: right; color: $dark-color; -webkit-transition: right 0.4s; diff --git a/src/fields/Document.ts b/src/fields/Document.ts index 2797efc09..6f6533754 100644 --- a/src/fields/Document.ts +++ b/src/fields/Document.ts @@ -29,15 +29,14 @@ export class Document extends Field { } static FromJson(data: any, id: string, save: boolean): Document { let doc = new Document(id, save); - let fields = data as [string, string][]; - fields.forEach(pair => doc._proxies.set(pair[0], pair[1])); + let fields = data as { key: string, field: string }[]; + fields.forEach(pair => doc._proxies.set(pair.key, pair.field)); return doc; } - UpdateFromServer(data: [string, string][]) { - for (const key in data) { - const element = data[key]; - this._proxies.set(element[0], element[1]); + UpdateFromServer(data: { key: string, field: string }[]) { + for (const kv of data) { + this._proxies.set(kv.key, kv.field); } } @@ -448,9 +447,9 @@ export class Document extends Field { } ToJson() { - let fields: [string, string][] = []; + let fields: { key: string, field: string }[] = []; this._proxies.forEach((field, key) => - field && fields.push([key, field])); + field && fields.push({ key, field })); return { type: Types.Document, diff --git a/src/server/Search.ts b/src/server/Search.ts new file mode 100644 index 000000000..7d8602346 --- /dev/null +++ b/src/server/Search.ts @@ -0,0 +1,38 @@ +import * as rp from 'request-promise'; +import { Database } from './database'; + +export class Search { + public static Instance = new Search(); + private url = 'http://localhost:8983/solr/'; + + public updateDocument(document: any): rp.RequestPromise { + return rp.post(this.url + "dash/update/json/docs", { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(document) + }); + } + + public async search(query: string) { + const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { + qs: { + q: query + } + })); + const fields = searchResults.response.docs; + const ids = fields.map((field: any) => field.id); + const docs = await Database.Instance.searchQuery(ids); + const docIds = docs.map((doc: any) => doc._id); + return docIds; + } + + public async clear() { + return rp.post(this.url + "dash/update", { + body: { + delete: { + query: "*:*" + } + }, + json: true + }); + } +} \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index 5457e4dd5..d5905c7b3 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -72,6 +72,18 @@ export class Database { }); } + public searchQuery(ids: string[], collectionName = Database.DocumentsCollection): Promise { + return new Promise(resolve => { + this.db && this.db.collection(collectionName).find({ "data.field": { "$in": ids } }).toArray((err, docs) => { + if (err) { + console.log(err.message); + console.log(err.errmsg); + } + resolve(docs); + }); + }); + } + public print() { console.log("db says hi!"); } diff --git a/src/server/index.ts b/src/server/index.ts index 70a7d266c..cb4268a2d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -11,6 +11,7 @@ import { ObservableMap } from 'mobx'; import * as passport from 'passport'; import * as path from 'path'; import * as request from 'request'; +import * as rp from 'request-promise'; import * as io from 'socket.io'; import { Socket } from 'socket.io'; import * as webpack from 'webpack'; @@ -22,7 +23,7 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; -import { MessageStore, Transferable } from "./Message"; +import { MessageStore, Transferable, Types } from "./Message"; import { RouteStore } from './RouteStore'; const app = express(); const config = require('../../webpack.config'); @@ -32,6 +33,7 @@ const serverPort = 4321; import expressFlash = require('express-flash'); import flash = require('connect-flash'); import c = require("crypto"); +import { Search } from './Search'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -120,6 +122,12 @@ app.get("/pull", (req, res) => // GETTERS +app.get("/search", async (req, res) => { + let query = req.query.query || "hello"; + let results = await Search.Instance.search(query); + res.send(results); +}); + // anyone attempting to navigate to localhost at this port will // first have to login addSecureRoute( @@ -234,14 +242,16 @@ server.on("connection", function (socket: Socket) { Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); }); -function deleteFields() { - return Database.Instance.deleteAll(); +async function deleteFields() { + await Database.Instance.deleteAll(); + await Search.Instance.clear(); } async function deleteAll() { await Database.Instance.deleteAll(); await Database.Instance.deleteAll('sessions'); await Database.Instance.deleteAll('users'); + await Search.Instance.clear(); } function barReceived(guid: String) { @@ -260,6 +270,9 @@ function getFields([ids, callback]: [string[], (result: Transferable[]) => void] function setField(socket: Socket, newValue: Transferable) { Database.Instance.update(newValue.id, newValue, () => socket.broadcast.emit(MessageStore.SetField.Message, newValue)); + if (newValue.type === Types.Text) { + Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); + } } server.listen(serverPort); -- cgit v1.2.3-70-g09d2 From da792d534024d8e0d266e291f92b9a1e83be51c2 Mon Sep 17 00:00:00 2001 From: ab Date: Tue, 30 Apr 2019 18:38:05 -0400 Subject: set not working --- src/debug/Test.tsx | 50 +++++++++++++++++++++++++------------------------- src/server/index.ts | 14 ++++++++++---- 2 files changed, 35 insertions(+), 29 deletions(-) (limited to 'src/server') diff --git a/src/debug/Test.tsx b/src/debug/Test.tsx index 6a677f80f..47cfac2c1 100644 --- a/src/debug/Test.tsx +++ b/src/debug/Test.tsx @@ -46,34 +46,34 @@ class Test extends React.Component { doc.fields = "test"; doc.test = "hello doc"; doc.url = url; - doc.testDoc = doc2; + //doc.testDoc = doc2; - const test1: TestDoc = TestDoc(doc); - const test2: Test2Doc = Test2Doc(doc); - assert(test1.hello === 5); - assert(test1.fields === undefined); - assert(test1.test === "hello doc"); - assert(test1.url === url); - assert(test1.testDoc === doc2); - test1.myField = 20; - assert(test1.myField === 20); + // const test1: TestDoc = TestDoc(doc); + // const test2: Test2Doc = Test2Doc(doc); + // assert(test1.hello === 5); + // assert(test1.fields === undefined); + // assert(test1.test === "hello doc"); + // assert(test1.url === url); + // //assert(test1.testDoc === doc2); + // test1.myField = 20; + // assert(test1.myField === 20); - assert(test2.hello === undefined); - // assert(test2.fields === "test"); - assert(test2.test === undefined); - assert(test2.url === undefined); - assert(test2.testDoc === undefined); - test2.url = 35; - assert(test2.url === 35); - const l = new List(); - //TODO push, and other array functions don't go through the proxy - l.push(1); - //TODO currently length, and any other string fields will get serialized - l.length = 3; - l[2] = 5; - console.log(l.slice()); - console.log(SerializationHelper.Serialize(l)); + // assert(test2.hello === undefined); + // // assert(test2.fields === "test"); + // assert(test2.test === undefined); + // assert(test2.url === undefined); + // assert(test2.testDoc === undefined); + // test2.url = 35; + // assert(test2.url === 35); + // const l = new List(); + // //TODO push, and other array functions don't go through the proxy + // l.push(1); + // //TODO currently length, and any other string fields will get serialized + // l.length = 3; + // l[2] = 5; + // console.log(l.slice()); + // console.log(SerializationHelper.Serialize(l)); } render() { diff --git a/src/server/index.ts b/src/server/index.ts index 464d3f68f..b57e5c482 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -295,23 +295,29 @@ function UpdateField(socket: Socket, diff: Diff) { //console.log("set field"); //} const docid = { id: diff.id }; - const docfield = diff.diff; + var docfield = diff.diff; + docfield = JSON.parse(JSON.stringify(docfield).split("fields.").join("")); console.log("FIELD: ", docfield); var dynfield = false; for (var key in docfield) { const val = docfield[key]; if (typeof val === 'number') { - key = key + "_n"; + const new_key: string = key + "_n"; + docfield = JSON.parse(JSON.stringify(docfield).split(key).join(new_key)); + //docfield[new_key] = { 'set': val }; dynfield = true; } else if (typeof val === 'string') { - key = key + "_t"; + const new_key: string = key + "_t"; + docfield = JSON.parse(JSON.stringify(docfield).split(key).join(new_key)); + docfield[new_key] = { 'set': val }; dynfield = true; } - console.log(key); } var merged = {}; _.extend(merged, docid, docfield); + console.log(merged); + console.log(docfield); if (dynfield) { console.log("dynamic field detected!"); Search.Instance.updateDocument(merged); -- cgit v1.2.3-70-g09d2 From eed0866d6148dfdb29c3f87b42afe365231f258c Mon Sep 17 00:00:00 2001 From: Monika Hedman Date: Tue, 30 Apr 2019 21:47:03 -0400 Subject: i am CONFUSED --- src/client/views/SearchBox.tsx | 31 ++++++++++++++++++++++++------- src/server/Search.ts | 2 ++ 2 files changed, 26 insertions(+), 7 deletions(-) (limited to 'src/server') diff --git a/src/client/views/SearchBox.tsx b/src/client/views/SearchBox.tsx index 7ceaf1da6..0670360a2 100644 --- a/src/client/views/SearchBox.tsx +++ b/src/client/views/SearchBox.tsx @@ -9,6 +9,10 @@ import { faSearch } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { library } from '@fortawesome/fontawesome-svg-core'; import { actionFieldDecorator } from 'mobx/lib/internal'; +// const app = express(); +// import * as express from 'express'; +import { Search } from '../../server/Search'; + library.add(faSearch); @@ -26,12 +30,25 @@ export class SearchBox extends React.Component { } submitSearch = () => { - Utils.EmitCallback(Server.Socket, MessageStore.SearchFor, this.searchString, (results: string[]) => { - for (const result of results) { - console.log(result); - //Utils.GetQueryVariable(); - } - }); + // Utils.EmitCallback(Server.Socket, MessageStore.SearchFor, this.searchString, (results: string[]) => { + // for (const result of results) { + // console.log(result); + // //Utils.GetQueryVariable(); + // } + // }); + + let query = this.searchString; + console.log(query); + //something bad is happening here + let results = Search.Instance.search(query); + console.log(results); + + // app.get("/search", async (req, res) => { + // //let query = req.query.query || "hello"; + // let query = this.searchString; + // let results = await Search.Instance.search(query); + // res.send(results); + // }); } @action @@ -73,7 +90,7 @@ export class SearchBox extends React.Component { map(prop => )} */} -
+
diff --git a/src/server/Search.ts b/src/server/Search.ts index 7d8602346..8ae996e9e 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -13,6 +13,8 @@ export class Search { } public async search(query: string) { + console.log("____________________________"); + console.log(query); const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { qs: { q: query -- cgit v1.2.3-70-g09d2 From 0c28cabf0d496be24da3e5ee414a8fcd925250ab Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Fri, 3 May 2019 01:42:54 -0400 Subject: Got part of search working --- src/server/Search.ts | 13 ++++++------- src/server/database.ts | 12 ------------ src/server/index.ts | 4 ++-- 3 files changed, 8 insertions(+), 21 deletions(-) (limited to 'src/server') diff --git a/src/server/Search.ts b/src/server/Search.ts index bcea03d5c..9e462f0ae 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -1,15 +1,16 @@ import * as rp from 'request-promise'; import { Database } from './database'; +import { thisExpression } from 'babel-types'; export class Search { public static Instance = new Search(); private url = 'http://localhost:8983/solr/'; - public updateDocument(document: any): rp.RequestPromise { - console.log(JSON.stringify(document)); - return rp.post(this.url + "dash/update/json/docs", { + public async updateDocument(document: any) { + console.log("UPDATE: ", JSON.stringify(document)); + return rp.post(this.url + "dash/update", { headers: { 'content-type': 'application/json' }, - body: JSON.stringify(document) + body: JSON.stringify([document]) }); } @@ -21,9 +22,7 @@ export class Search { })); const fields = searchResults.response.docs; const ids = fields.map((field: any) => field.id); - const docs = await Database.Instance.searchQuery(ids); - const docIds = docs.map((doc: any) => doc._id); - return docIds; + return ids; } public async clear() { diff --git a/src/server/database.ts b/src/server/database.ts index 1e8004328..a61b4d823 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -74,18 +74,6 @@ export class Database { }); } - public searchQuery(ids: string[], collectionName = Database.DocumentsCollection): Promise { - return new Promise(resolve => { - this.db && this.db.collection(collectionName).find({ "data.field": { "$in": ids } }).toArray((err, docs) => { - if (err) { - console.log(err.message); - console.log(err.errmsg); - } - resolve(docs); - }); - }); - } - public print() { console.log("db says hi!"); } diff --git a/src/server/index.ts b/src/server/index.ts index b57e5c482..b4252c2a1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -316,8 +316,8 @@ function UpdateField(socket: Socket, diff: Diff) { } var merged = {}; _.extend(merged, docid, docfield); - console.log(merged); - console.log(docfield); + console.log("MERGED: ", merged); + console.log("DOC_FIELD: ", docfield); if (dynfield) { console.log("dynamic field detected!"); Search.Instance.updateDocument(merged); -- cgit v1.2.3-70-g09d2 From 189eeefbb3e7f3d96d1d140beff74fcffe305786 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Fri, 3 May 2019 20:28:22 -0400 Subject: Added debugging support and added removing of unused keys in Solr --- .vscode/launch.json | 9 +++++++++ package.json | 3 ++- src/server/index.ts | 40 +++++++++++++++------------------------- 3 files changed, 26 insertions(+), 26 deletions(-) (limited to 'src/server') diff --git a/.vscode/launch.json b/.vscode/launch.json index fb91a1080..e92a4949a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,6 +13,15 @@ "url": "http://localhost:1050/login", "webRoot": "${workspaceFolder}", }, + { + "type": "node", + "request": "attach", + "name": "Typescript Server", + "protocol": "inspector", + "port": 9229, + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + }, { "type": "node", "request": "launch", diff --git a/package.json b/package.json index 1eb546a80..fd794d062 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "index.js", "scripts": { "start": "ts-node-dev -- src/server/index.ts", + "debug": "ts-node-dev --inspect -- src/server/index.ts", "build": "webpack --env production", "test": "mocha -r ts-node/register test/**/*.ts", "tsc": "tsc" @@ -171,4 +172,4 @@ "uuid": "^3.3.2", "xoauth2": "^1.2.0" } -} +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index b4252c2a1..f90724152 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -286,41 +286,31 @@ function GetRefField([id, callback]: [string, (result?: Transferable) => void]) Database.Instance.getDocument(id, callback, "newDocuments"); } +const suffixMap: { [type: string]: string } = { + "number": "_n", + "string": "_t" +}; function UpdateField(socket: Socket, diff: Diff) { Database.Instance.update(diff.id, diff.diff, () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); - //if (diff.diff === Types.Text) { - //Search.Instance.updateDocument({ name: "john", burns: "true" }); - //Search.Instance.updateDocument({ id: diff.id, data: diff.diff.data }); - //console.log("set field"); - //} - const docid = { id: diff.id }; - var docfield = diff.diff; - docfield = JSON.parse(JSON.stringify(docfield).split("fields.").join("")); + const docfield = diff.diff; + const update: any = { id: diff.id }; console.log("FIELD: ", docfield); - var dynfield = false; - for (var key in docfield) { + let dynfield = false; + for (let key in docfield) { + if (!key.startsWith("fields.")) continue; const val = docfield[key]; - if (typeof val === 'number') { - const new_key: string = key + "_n"; - docfield = JSON.parse(JSON.stringify(docfield).split(key).join(new_key)); - //docfield[new_key] = { 'set': val }; - dynfield = true; - } - else if (typeof val === 'string') { - const new_key: string = key + "_t"; - docfield = JSON.parse(JSON.stringify(docfield).split(key).join(new_key)); - docfield[new_key] = { 'set': val }; + const suffix = suffixMap[typeof val]; + if (suffix !== undefined) { + key = key.substring(7); + Object.values(suffixMap).forEach(suf => update[key + suf] = null); + update[key + suffix] = { set: val }; dynfield = true; } } - var merged = {}; - _.extend(merged, docid, docfield); - console.log("MERGED: ", merged); - console.log("DOC_FIELD: ", docfield); if (dynfield) { console.log("dynamic field detected!"); - Search.Instance.updateDocument(merged); + Search.Instance.updateDocument(update); } } -- cgit v1.2.3-70-g09d2 From e19fdbba4cf672aee5bfb59b91b6162431d146d3 Mon Sep 17 00:00:00 2001 From: ab Date: Sat, 4 May 2019 18:48:01 -0400 Subject: queries semi work --- package.json | 3 ++- src/debug/Test.tsx | 19 ++++++++++++++++++- src/server/Search.ts | 25 +++++++++++++++++++++++++ src/server/index.ts | 3 +++ 4 files changed, 48 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/package.json b/package.json index fd794d062..3393eacc6 100644 --- a/package.json +++ b/package.json @@ -167,9 +167,10 @@ "serializr": "^1.5.1", "socket.io": "^2.2.0", "socket.io-client": "^2.2.0", + "solr-node": "^1.1.3", "typescript-collections": "^1.3.2", "url-loader": "^1.1.2", "uuid": "^3.3.2", "xoauth2": "^1.2.0" } -} \ No newline at end of file +} diff --git a/src/debug/Test.tsx b/src/debug/Test.tsx index 47cfac2c1..7d72a1ba0 100644 --- a/src/debug/Test.tsx +++ b/src/debug/Test.tsx @@ -3,6 +3,9 @@ import * as ReactDOM from 'react-dom'; import { serialize, deserialize, map } from 'serializr'; import { URLField, Doc, createSchema, makeInterface, makeStrictInterface, List, ListSpec } from '../fields/NewDoc'; import { SerializationHelper } from '../client/util/SerializationHelper'; +import { Search } from '../server/Search'; +import { restProperty } from 'babel-types'; +import * as rp from 'request-promise'; const schema1 = createSchema({ hello: "number", @@ -76,8 +79,22 @@ class Test extends React.Component { // console.log(SerializationHelper.Serialize(l)); } + onEnter = async (e: any) => { + var key = e.keyCode || e.which; + if (key === 13) { + var query = e.target.value; + await rp.get('http://localhost:1050/search', { + qs: { + query + } + }); + } + } + render() { - return ; + return
+ +
; } } diff --git a/src/server/Search.ts b/src/server/Search.ts index 9e462f0ae..4911edd1d 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -5,6 +5,31 @@ import { thisExpression } from 'babel-types'; export class Search { public static Instance = new Search(); private url = 'http://localhost:8983/solr/'; + private client: any; + + constructor() { + console.log("Search Instantiated!"); + var SolrNode = require('solr-node'); + this.client = new SolrNode({ + host: 'localhost', + port: '8983', + core: 'dash', + protocol: 'http' + }); + var strQuery = this.client.query().q('text:test'); + + console.log(strQuery); + + // Search documents using strQuery + // client.search(strQuery, (err: any, result: any) => { + // if (err) { + // console.log(err); + // return; + // } + // console.log('Response:', result.response); + // }); + } + public async updateDocument(document: any) { console.log("UPDATE: ", JSON.stringify(document)); diff --git a/src/server/index.ts b/src/server/index.ts index f90724152..f2bcd3f00 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -122,11 +122,14 @@ app.get("/pull", (req, res) => res.redirect("/"); })); +// SEARCH + // GETTERS app.get("/search", async (req, res) => { let query = req.query.query || "hello"; let results = await Search.Instance.search(query); + console.log(results); res.send(results); }); -- cgit v1.2.3-70-g09d2 From 6683c5450eb25da291090091421e791bf0498aba Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 7 May 2019 16:56:41 -0400 Subject: Fix after merge --- src/server/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index 2381f9840..6b92e8e8e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -303,7 +303,10 @@ const suffixMap: { [type: string]: string } = { function UpdateField(socket: Socket, diff: Diff) { Database.Instance.update(diff.id, diff.diff, () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); - const docfield = diff.diff; + const docfield = diff.diff.$set; + if (!docfield) { + return; + } const update: any = { id: diff.id }; console.log("FIELD: ", docfield); let dynfield = false; -- cgit v1.2.3-70-g09d2 From 152fadbad5d3c4e9c452bb6a1ade543bd84c6416 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 7 May 2019 19:26:36 -0400 Subject: Added copy fields to search to enable easier searching --- solr/conf/schema.xml | 13 ++++++++++--- src/server/Search.ts | 26 -------------------------- src/server/index.ts | 3 --- 3 files changed, 10 insertions(+), 32 deletions(-) (limited to 'src/server') diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml index 5e80b17d9..99087a1db 100644 --- a/solr/conf/schema.xml +++ b/solr/conf/schema.xml @@ -40,10 +40,17 @@ - + - - + + + + + + + + + diff --git a/src/server/Search.ts b/src/server/Search.ts index 4911edd1d..59bdd4803 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -5,34 +5,8 @@ import { thisExpression } from 'babel-types'; export class Search { public static Instance = new Search(); private url = 'http://localhost:8983/solr/'; - private client: any; - - constructor() { - console.log("Search Instantiated!"); - var SolrNode = require('solr-node'); - this.client = new SolrNode({ - host: 'localhost', - port: '8983', - core: 'dash', - protocol: 'http' - }); - var strQuery = this.client.query().q('text:test'); - - console.log(strQuery); - - // Search documents using strQuery - // client.search(strQuery, (err: any, result: any) => { - // if (err) { - // console.log(err); - // return; - // } - // console.log('Response:', result.response); - // }); - } - public async updateDocument(document: any) { - console.log("UPDATE: ", JSON.stringify(document)); return rp.post(this.url + "dash/update", { headers: { 'content-type': 'application/json' }, body: JSON.stringify([document]) diff --git a/src/server/index.ts b/src/server/index.ts index 6b92e8e8e..44251de3d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -308,7 +308,6 @@ function UpdateField(socket: Socket, diff: Diff) { return; } const update: any = { id: diff.id }; - console.log("FIELD: ", docfield); let dynfield = false; for (let key in docfield) { if (!key.startsWith("fields.")) continue; @@ -322,14 +321,12 @@ function UpdateField(socket: Socket, diff: Diff) { } } if (dynfield) { - console.log("dynamic field detected!"); Search.Instance.updateDocument(update); } } function CreateField(newValue: any) { Database.Instance.insert(newValue, "newDocuments"); - console.log("created field"); } server.listen(serverPort); -- cgit v1.2.3-70-g09d2 From 85d8e29c15b45cd83d258f185d3d55ff400a145e Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 7 May 2019 21:57:48 -0400 Subject: Added date support --- solr/conf/schema.xml | 1 + solr/conf/solrconfig.xml | 2 +- src/server/index.ts | 24 ++++++++++++++++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) (limited to 'src/server') diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml index 99087a1db..3d1ccb698 100644 --- a/solr/conf/schema.xml +++ b/solr/conf/schema.xml @@ -46,6 +46,7 @@ + diff --git a/solr/conf/solrconfig.xml b/solr/conf/solrconfig.xml index 981500022..90eff5363 100644 --- a/solr/conf/solrconfig.xml +++ b/solr/conf/solrconfig.xml @@ -696,7 +696,7 @@ explicit 10 - data + text diff --git a/src/server/index.ts b/src/server/index.ts index 44251de3d..5023bf717 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -296,10 +296,17 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v } -const suffixMap: { [type: string]: string } = { +const suffixMap: { [type: string]: string | [string, string] | [string, string, (json: any) => any] } = { "number": "_n", - "string": "_t" + "string": "_t", + "image": ["_t", "url"], + "video": ["_t", "url"], + "pdf": ["_t", "url"], + "audio": ["_t", "url"], + "web": ["_t", "url"], + "date": ["_d", "date", millis => new Date(millis).toISOString()], }; + function UpdateField(socket: Socket, diff: Diff) { Database.Instance.update(diff.id, diff.diff, () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); @@ -311,9 +318,18 @@ function UpdateField(socket: Socket, diff: Diff) { let dynfield = false; for (let key in docfield) { if (!key.startsWith("fields.")) continue; - const val = docfield[key]; - const suffix = suffixMap[typeof val]; + let val = docfield[key]; + const type = val.__type || typeof val; + let suffix = suffixMap[type]; if (suffix !== undefined) { + if (Array.isArray(suffix)) { + val = val[suffix[1]]; + const func = suffix[2]; + if (func) { + val = func(val); + } + suffix = suffix[0]; + } key = key.substring(7); Object.values(suffixMap).forEach(suf => update[key + suf] = null); update[key + suffix] = { set: val }; -- cgit v1.2.3-70-g09d2 From 823b04d8084f14e298a408615eccf712dd76e2b9 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 7 May 2019 22:27:15 -0400 Subject: Removed console.logs --- src/server/Search.ts | 2 -- src/server/index.ts | 1 - 2 files changed, 3 deletions(-) (limited to 'src/server') diff --git a/src/server/Search.ts b/src/server/Search.ts index 7a670e21b..59bdd4803 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -14,8 +14,6 @@ export class Search { } public async search(query: string) { - console.log("____________________________"); - console.log(query); const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { qs: { q: query diff --git a/src/server/index.ts b/src/server/index.ts index 5023bf717..5c54babb2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -128,7 +128,6 @@ app.get("/pull", (req, res) => app.get("/search", async (req, res) => { let query = req.query.query || "hello"; let results = await Search.Instance.search(query); - console.log(results); res.send(results); }); -- cgit v1.2.3-70-g09d2 From c7b2ccddb6d75283a7255b612693c5e809b68f5f Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Thu, 9 May 2019 00:26:09 -0400 Subject: Added lists and documents to search --- solr/conf/schema.xml | 12 +++++---- src/client/documents/Documents.ts | 1 + src/server/index.ts | 51 ++++++++++++++++++++++++++++----------- 3 files changed, 45 insertions(+), 19 deletions(-) (limited to 'src/server') diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml index 92ba9c6ff..a568db14c 100644 --- a/solr/conf/schema.xml +++ b/solr/conf/schema.xml @@ -35,19 +35,21 @@ - + - - - + + + - + + + diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index a770ccc93..00233a989 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -175,6 +175,7 @@ export namespace Docs { if (!("creationDate" in protoProps)) { protoProps.creationDate = new DateField; } + protoProps.isPrototype = true; return SetDelegateOptions(SetInstanceOptions(proto, protoProps, data), delegateProps); } diff --git a/src/server/index.ts b/src/server/index.ts index 5c54babb2..93e4cafbf 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -295,7 +295,7 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v } -const suffixMap: { [type: string]: string | [string, string] | [string, string, (json: any) => any] } = { +const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", "image": ["_t", "url"], @@ -303,9 +303,40 @@ const suffixMap: { [type: string]: string | [string, string] | [string, string, "pdf": ["_t", "url"], "audio": ["_t", "url"], "web": ["_t", "url"], - "date": ["_d", "date", millis => new Date(millis).toISOString()], + "date": ["_d", value => new Date(value.date).toISOString()], + "proxy": ["_i", "fieldId"], + "list": ["_l", list => { + const results = []; + for (const value of list.fields) { + const term = ToSearchTerm(value); + if (term) { + results.push(term.value); + } + } + return results.length ? results : null; + }] }; +function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { + const type = val.__type || typeof val; + let suffix = suffixMap[type]; + if (!suffix) { + return; + } + + if (Array.isArray(suffix)) { + const accessor = suffix[1]; + if (typeof accessor === "function") { + val = accessor(val); + } else { + val = val[accessor]; + } + suffix = suffix[0]; + } + + return { suffix, value: val } +} + function UpdateField(socket: Socket, diff: Diff) { Database.Instance.update(diff.id, diff.diff, () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); @@ -318,20 +349,12 @@ function UpdateField(socket: Socket, diff: Diff) { for (let key in docfield) { if (!key.startsWith("fields.")) continue; let val = docfield[key]; - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (suffix !== undefined) { - if (Array.isArray(suffix)) { - val = val[suffix[1]]; - const func = suffix[2]; - if (func) { - val = func(val); - } - suffix = suffix[0]; - } + let term = ToSearchTerm(val); + if (term !== undefined) { + let { suffix, value } = term; key = key.substring(7); Object.values(suffixMap).forEach(suf => update[key + suf] = null); - update[key + suffix] = { set: val }; + update[key + suffix] = { set: value }; dynfield = true; } } -- cgit v1.2.3-70-g09d2 From b18aec5d4b07bbc859a5b1b2d22ca3bd92ca53cd Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Thu, 9 May 2019 20:30:40 -0400 Subject: Added try catches to search to not throw errors on the server --- src/server/Search.ts | 48 ++++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) (limited to 'src/server') diff --git a/src/server/Search.ts b/src/server/Search.ts index 59bdd4803..1d8cfdcf0 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -7,31 +7,39 @@ export class Search { private url = 'http://localhost:8983/solr/'; public async updateDocument(document: any) { - return rp.post(this.url + "dash/update", { - headers: { 'content-type': 'application/json' }, - body: JSON.stringify([document]) - }); + try { + return rp.post(this.url + "dash/update", { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify([document]) + }); + } catch { } } public async search(query: string) { - const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { - qs: { - q: query - } - })); - const fields = searchResults.response.docs; - const ids = fields.map((field: any) => field.id); - return ids; + try { + const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { + qs: { + q: query + } + })); + const fields = searchResults.response.docs; + const ids = fields.map((field: any) => field.id); + return ids; + } catch { + return []; + } } public async clear() { - return rp.post(this.url + "dash/update", { - body: { - delete: { - query: "*:*" - } - }, - json: true - }); + try { + return rp.post(this.url + "dash/update", { + body: { + delete: { + query: "*:*" + } + }, + json: true + }); + } catch { } } } \ No newline at end of file -- cgit v1.2.3-70-g09d2