aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/server/ActionUtilities.ts27
-rw-r--r--src/server/ApiManagers/SearchManager.ts2
-rw-r--r--src/server/GarbageCollector.ts4
-rw-r--r--src/server/Search.ts28
-rw-r--r--src/server/Websocket/Websocket.ts15
-rw-r--r--src/server/authentication/models/user_model.ts5
-rw-r--r--src/server/remapUrl.ts2
-rw-r--r--src/server/updateSearch.ts114
8 files changed, 160 insertions, 37 deletions
diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts
index 94008e171..4667254d8 100644
--- a/src/server/ActionUtilities.ts
+++ b/src/server/ActionUtilities.ts
@@ -3,7 +3,7 @@ import { ExecOptions } from 'shelljs';
import { exec } from 'child_process';
import * as path from 'path';
import * as rimraf from "rimraf";
-import { yellow } from 'colors';
+import { yellow, Color } from 'colors';
export const command_line = (command: string, fromDirectory?: string) => {
return new Promise<string>((resolve, reject) => {
@@ -29,18 +29,29 @@ export const write_text_file = (relativePath: string, contents: any) => {
});
};
-export interface LogData {
+export interface LogData<T> {
startMessage: string;
endMessage: string;
- action: () => void | Promise<void>;
+ action: () => T | Promise<T>;
+ color?: Color;
}
let current = Math.ceil(Math.random() * 20);
-export async function log_execution({ startMessage, endMessage, action }: LogData) {
- const color = `\x1b[${31 + current++ % 6}m%s\x1b[0m`;
- console.log(color, `${startMessage}...`);
- await action();
- console.log(color, endMessage);
+export async function log_execution<T>({ startMessage, endMessage, action, color }: LogData<T>): Promise<T> {
+ let result: T;
+ const formattedStart = `${startMessage}...`;
+ const formattedEnd = `${endMessage}.`;
+ if (color) {
+ console.log(color(formattedStart));
+ result = await action();
+ console.log(color(formattedEnd));
+ } else {
+ const color = `\x1b[${31 + current++ % 6}m%s\x1b[0m`;
+ console.log(color, formattedStart);
+ result = await action();
+ console.log(color, formattedEnd);
+ }
+ return result;
}
export function logPort(listener: string, port: number) {
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
index ccd0896bd..ccfd570b8 100644
--- a/src/server/ApiManagers/SearchManager.ts
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -57,7 +57,7 @@ export class SearchManager extends ApiManager {
res.send([]);
return;
}
- const results = await Search.Instance.search(solrQuery);
+ const results = await Search.search(solrQuery);
res.send(results);
}
});
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
index 09b52eadf..5729c3ee5 100644
--- a/src/server/GarbageCollector.ts
+++ b/src/server/GarbageCollector.ts
@@ -100,7 +100,7 @@ async function GarbageCollect(full: boolean = true) {
if (!full) {
await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } });
await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } });
- console.log(await Search.Instance.updateDocuments(
+ console.log(await Search.updateDocuments(
notToDelete.map<any>(id => ({
id, deleted: { set: null }
}))
@@ -122,7 +122,7 @@ async function GarbageCollect(full: boolean = true) {
// const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
console.log(`${deleted} documents deleted`);
- await Search.Instance.deleteDocuments(toDelete);
+ await Search.deleteDocuments(toDelete);
console.log("Cleared search documents");
const folder = "./src/server/public/files/";
diff --git a/src/server/Search.ts b/src/server/Search.ts
index 723dc101b..2b59c14b1 100644
--- a/src/server/Search.ts
+++ b/src/server/Search.ts
@@ -1,14 +1,12 @@
import * as rp from 'request-promise';
-import { Database } from './database';
-import { thisExpression } from 'babel-types';
-export class Search {
- public static Instance = new Search();
- private url = 'http://localhost:8983/solr/';
+const pathTo = (relative: string) => `http://localhost:8983/solr/dash/${relative}`;
- public async updateDocument(document: any) {
+export namespace Search {
+
+ export async function updateDocument(document: any) {
try {
- const res = await rp.post(this.url + "dash/update", {
+ const res = await rp.post(pathTo("update"), {
headers: { 'content-type': 'application/json' },
body: JSON.stringify([document])
});
@@ -18,9 +16,9 @@ export class Search {
}
}
- public async updateDocuments(documents: any[]) {
+ export async function updateDocuments(documents: any[]) {
try {
- const res = await rp.post(this.url + "dash/update", {
+ const res = await rp.post(pathTo("update"), {
headers: { 'content-type': 'application/json' },
body: JSON.stringify(documents)
});
@@ -30,9 +28,9 @@ export class Search {
}
}
- public async search(query: any) {
+ export async function search(query: any) {
try {
- const searchResults = JSON.parse(await rp.get(this.url + "dash/select", {
+ const searchResults = JSON.parse(await rp.get(pathTo("select"), {
qs: query
}));
const { docs, numFound } = searchResults.response;
@@ -43,9 +41,9 @@ export class Search {
}
}
- public async clear() {
+ export async function clear() {
try {
- return await rp.post(this.url + "dash/update", {
+ return rp.post(pathTo("update"), {
body: {
delete: {
query: "*:*"
@@ -56,7 +54,7 @@ export class Search {
} catch { }
}
- public deleteDocuments(docs: string[]) {
+ export async function deleteDocuments(docs: string[]) {
const promises: rp.RequestPromise[] = [];
const nToDelete = 1000;
let index = 0;
@@ -64,7 +62,7 @@ export class Search {
const count = Math.min(docs.length - index, nToDelete);
const deleteIds = docs.slice(index, index + count);
index += count;
- promises.push(rp.post(this.url + "dash/update", {
+ promises.push(rp.post(pathTo("update"), {
body: {
delete: {
query: deleteIds.map(id => `id:"${id}"`).join(" ")
diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts
index 5c0bb508b..76e02122b 100644
--- a/src/server/Websocket/Websocket.ts
+++ b/src/server/Websocket/Websocket.ts
@@ -10,7 +10,6 @@ import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader";
import { logPort } from "../ActionUtilities";
import { timeMap } from "../ApiManagers/UserManager";
import { green } from "colors";
-import { SolrManager } from "../ApiManagers/SearchManager";
export namespace WebSocket {
@@ -80,7 +79,7 @@ export namespace WebSocket {
export async function deleteFields() {
await Database.Instance.deleteAll();
- await Search.Instance.clear();
+ await Search.clear();
await Database.Instance.deleteAll('newDocuments');
}
@@ -89,7 +88,7 @@ export namespace WebSocket {
await Database.Instance.deleteAll('newDocuments');
await Database.Instance.deleteAll('sessions');
await Database.Instance.deleteAll('users');
- await Search.Instance.clear();
+ await Search.clear();
}
function barReceived(socket: SocketIO.Socket, userEmail: string) {
@@ -111,7 +110,7 @@ export namespace WebSocket {
Database.Instance.update(newValue.id, newValue, () =>
socket.broadcast.emit(MessageStore.SetField.Message, newValue));
if (newValue.type === Types.Text) {
- Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data });
+ Search.updateDocument({ id: newValue.id, data: (newValue as any).data });
console.log("set field");
console.log("checking in");
}
@@ -197,7 +196,7 @@ export namespace WebSocket {
}
}
if (dynfield) {
- Search.Instance.updateDocument(update);
+ Search.updateDocument(update);
}
}
@@ -206,16 +205,14 @@ export namespace WebSocket {
socket.broadcast.emit(MessageStore.DeleteField.Message, id);
});
- Search.Instance.deleteDocuments([id]);
+ Search.deleteDocuments([id]);
}
function DeleteFields(socket: Socket, ids: string[]) {
Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => {
socket.broadcast.emit(MessageStore.DeleteFields.Message, ids);
});
-
- Search.Instance.deleteDocuments(ids);
-
+ Search.deleteDocuments(ids);
}
function CreateField(newValue: any) {
diff --git a/src/server/authentication/models/user_model.ts b/src/server/authentication/models/user_model.ts
index 6b71397dc..78e39dbc1 100644
--- a/src/server/authentication/models/user_model.ts
+++ b/src/server/authentication/models/user_model.ts
@@ -73,8 +73,11 @@ userSchema.pre("save", function save(next) {
});
const comparePassword: comparePasswordFunction = function (this: DashUserModel, candidatePassword, cb) {
+ // Choose one of the following bodies for authentication logic.
+ // secure
bcrypt.compare(candidatePassword, this.password, cb);
- // return cb(undefined, true); // use this to bypass passwords
+ // bypass password
+ // cb(undefined, true);
};
userSchema.methods.comparePassword = comparePassword;
diff --git a/src/server/remapUrl.ts b/src/server/remapUrl.ts
index 5218a239a..45d2fdd33 100644
--- a/src/server/remapUrl.ts
+++ b/src/server/remapUrl.ts
@@ -54,7 +54,7 @@ async function update() {
}));
console.log("Done");
// await Promise.all(updates.map(update => {
- // return limit(() => Search.Instance.updateDocument(update));
+ // return limit(() => Search.updateDocument(update));
// }));
cursor.close();
}
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
new file mode 100644
index 000000000..69f32f945
--- /dev/null
+++ b/src/server/updateSearch.ts
@@ -0,0 +1,114 @@
+import { Database } from "./database";
+import { Search } from "./Search";
+import { log_execution } from "./ActionUtilities";
+import { cyan, green, yellow, red } from "colors";
+
+const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
+ "number": "_n",
+ "string": "_t",
+ "boolean": "_b",
+ "image": ["_t", "url"],
+ "video": ["_t", "url"],
+ "pdf": ["_t", "url"],
+ "audio": ["_t", "url"],
+ "web": ["_t", "url"],
+ "date": ["_d", value => new Date(value.date).toISOString()],
+ "proxy": ["_i", "fieldId"],
+ "list": ["_l", list => {
+ const results = [];
+ for (const value of list.fields) {
+ const term = ToSearchTerm(value);
+ if (term) {
+ results.push(term.value);
+ }
+ }
+ return results.length ? results : null;
+ }]
+};
+
+function ToSearchTerm(val: any): { suffix: string, value: any } | undefined {
+ if (val === null || val === undefined) {
+ return;
+ }
+ const type = val.__type || typeof val;
+ let suffix = suffixMap[type];
+ if (!suffix) {
+ return;
+ }
+
+ if (Array.isArray(suffix)) {
+ const accessor = suffix[1];
+ if (typeof accessor === "function") {
+ val = accessor(val);
+ } else {
+ val = val[accessor];
+ }
+ suffix = suffix[0];
+ }
+
+ return { suffix, value: val };
+}
+
+async function update() {
+ console.log(green("Beginning update..."));
+ await log_execution<void>({
+ startMessage: "Clearing existing Solr information",
+ endMessage: "Solr information successfully cleared",
+ action: Search.clear,
+ color: cyan
+ });
+ const cursor = await log_execution({
+ startMessage: "Connecting to and querying for all documents from database",
+ endMessage: "Connection successful and query complete",
+ action: () => Database.Instance.query({}),
+ color: yellow
+ });
+ const updates: any[] = [];
+ let numDocs = 0;
+ function updateDoc(doc: any) {
+ numDocs++;
+ if ((numDocs % 50) === 0) {
+ console.log(`Batch of 50 complete, total of ${numDocs}`);
+ }
+ if (doc.__type !== "Doc") {
+ return;
+ }
+ const fields = doc.fields;
+ if (!fields) {
+ return;
+ }
+ const update: any = { id: doc._id };
+ let dynfield = false;
+ for (const key in fields) {
+ const value = fields[key];
+ const term = ToSearchTerm(value);
+ if (term !== undefined) {
+ const { suffix, value } = term;
+ update[key + suffix] = value;
+ dynfield = true;
+ }
+ }
+ if (dynfield) {
+ updates.push(update);
+ }
+ }
+ await cursor.forEach(updateDoc);
+ const result = await log_execution({
+ startMessage: `Dispatching updates for ${updates.length} documents`,
+ endMessage: "Dispatched updates complete",
+ action: () => Search.updateDocuments(updates),
+ color: cyan
+ });
+ try {
+ const { status } = JSON.parse(result).responseHeader;
+ console.log(status ? red(`Failed with status code (${status})`) : green("Success!"));
+ } catch {
+ console.log(red("Error:"));
+ console.log(result);
+ console.log("\n");
+ }
+ await cursor.close();
+ process.exit(0);
+}
+
+update(); \ No newline at end of file
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 23:01:35 +0000