aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/server/ApiManagers/SearchManager.ts154
-rw-r--r--src/server/updateSearch.ts125
2 files changed, 137 insertions, 142 deletions
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
index 4bbf31ac5..be17c3105 100644
--- a/src/server/ApiManagers/SearchManager.ts
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -4,11 +4,15 @@ import { Search } from "../Search";
const findInFiles = require('find-in-files');
import * as path from 'path';
import { pathToDirectory, Directory } from "./UploadManager";
-import { red, cyan, yellow } from "colors";
+import { red, cyan, yellow, green } from "colors";
import RouteSubscriber from "../RouteSubscriber";
import { exec, execSync } from "child_process";
import { onWindows } from "..";
import { get } from "request-promise";
+import { log_execution } from "../ActionUtilities";
+import { Database } from "../database";
+import rimraf = require("rimraf");
+import { mkdirSync, chmod, chmodSync } from "fs";
export class SearchManager extends ApiManager {
@@ -19,14 +23,17 @@ export class SearchManager extends ApiManager {
subscription: new RouteSubscriber("solr").add("action"),
secureHandler: async ({ req, res }) => {
const { action } = req.params;
- if (["start", "stop"].includes(action)) {
- const status = req.params.action === "start";
- const success = await SolrManager.SetRunning(status);
- console.log(success ? `Successfully ${status ? "started" : "stopped"} Solr!` : `Uh oh! Check the console for the error that occurred while ${status ? "starting" : "stopping"} Solr`);
- } else if (action === "update") {
- execSync("npx ts-node updateSearch.ts", { cwd: path.resolve(__dirname, "../"), stdio: "inherit" });
- } else {
- console.log(yellow(`${action} is an unknown solr operation.`));
+ switch (action) {
+ case "start":
+ case "stop":
+ const status = req.params.action === "start";
+ SolrManager.SetRunning(status);
+ break;
+ case "update":
+ await SolrManager.update();
+ break;
+ default:
+ console.log(yellow(`${action} is an unknown solr operation.`));
}
res.redirect("/home");
}
@@ -73,12 +80,10 @@ export class SearchManager extends ApiManager {
export namespace SolrManager {
- const command = onWindows ? "solr.cmd" : "solr";
-
- export async function SetRunning(status: boolean): Promise<boolean> {
+ export function SetRunning(status: boolean) {
const args = status ? "start" : "stop -p 8983";
console.log(`solr management: trying to ${args}`);
- exec(`${command} ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => {
+ exec(`solr ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => {
if (error) {
console.log(red(`solr management error: unable to ${args} server`));
console.log(red(error.message));
@@ -86,12 +91,127 @@ export namespace SolrManager {
console.log(cyan(stdout));
console.log(yellow(stderr));
});
+ if (status) {
+ console.log(cyan("Start script is executing: please allow 15 seconds for solr to start on port 8983."));
+ }
+ }
+
+ export async function update() {
+ console.log(green("Beginning update..."));
+ await log_execution<void>({
+ startMessage: "Clearing existing Solr information...",
+ endMessage: "Solr information successfully cleared",
+ action: Search.clear,
+ color: cyan
+ });
+ const cursor = await log_execution({
+ startMessage: "Connecting to and querying for all documents from database...",
+ endMessage: ({ result, error }) => {
+ const success = error === null && result !== undefined;
+ if (!success) {
+ console.log(red("Unable to connect to the database."));
+ process.exit(0);
+ }
+ return "Connection successful and query complete";
+ },
+ action: () => Database.Instance.query({}),
+ color: yellow
+ });
+ const updates: any[] = [];
+ let numDocs = 0;
+ function updateDoc(doc: any) {
+ numDocs++;
+ if ((numDocs % 50) === 0) {
+ console.log(`Batch of 50 complete, total of ${numDocs}`);
+ }
+ if (doc.__type !== "Doc") {
+ return;
+ }
+ const fields = doc.fields;
+ if (!fields) {
+ return;
+ }
+ const update: any = { id: doc._id };
+ let dynfield = false;
+ for (const key in fields) {
+ const value = fields[key];
+ const term = ToSearchTerm(value);
+ if (term !== undefined) {
+ const { suffix, value } = term;
+ update[key + suffix] = value;
+ dynfield = true;
+ }
+ }
+ if (dynfield) {
+ updates.push(update);
+ }
+ }
+ await cursor?.forEach(updateDoc);
+ const result = await log_execution({
+ startMessage: `Dispatching updates for ${updates.length} documents`,
+ endMessage: "Dispatched updates complete",
+ action: () => Search.updateDocuments(updates),
+ color: cyan
+ });
try {
- await get("http://localhost:8983");
- return true;
- } catch {
- return false;
+ if (result) {
+ const { status } = JSON.parse(result).responseHeader;
+ console.log(status ? red(`Failed with status code (${status})`) : green("Success!"));
+ } else {
+ console.log(red("Solr is likely not running!"));
+ }
+ } catch (e) {
+ console.log(red("Error:"));
+ console.log(e);
+ console.log("\n");
}
+ await cursor?.close();
+ }
+
+ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
+ "number": "_n",
+ "string": "_t",
+ "boolean": "_b",
+ "image": ["_t", "url"],
+ "video": ["_t", "url"],
+ "pdf": ["_t", "url"],
+ "audio": ["_t", "url"],
+ "web": ["_t", "url"],
+ "date": ["_d", value => new Date(value.date).toISOString()],
+ "proxy": ["_i", "fieldId"],
+ "list": ["_l", list => {
+ const results = [];
+ for (const value of list.fields) {
+ const term = ToSearchTerm(value);
+ if (term) {
+ results.push(term.value);
+ }
+ }
+ return results.length ? results : null;
+ }]
+ };
+
+ function ToSearchTerm(val: any): { suffix: string, value: any } | undefined {
+ if (val === null || val === undefined) {
+ return;
+ }
+ const type = val.__type || typeof val;
+ let suffix = suffixMap[type];
+ if (!suffix) {
+ return;
+ }
+
+ if (Array.isArray(suffix)) {
+ const accessor = suffix[1];
+ if (typeof accessor === "function") {
+ val = accessor(val);
+ } else {
+ val = val[accessor];
+ }
+ suffix = suffix[0];
+ }
+
+ return { suffix, value: val };
}
} \ No newline at end of file
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
deleted file mode 100644
index dd2067c87..000000000
--- a/src/server/updateSearch.ts
+++ /dev/null
@@ -1,125 +0,0 @@
-import { Database } from "./database";
-import { Search } from "./Search";
-import { log_execution } from "./ActionUtilities";
-import { cyan, green, yellow, red } from "colors";
-
-const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
- "number": "_n",
- "string": "_t",
- "boolean": "_b",
- "image": ["_t", "url"],
- "video": ["_t", "url"],
- "pdf": ["_t", "url"],
- "audio": ["_t", "url"],
- "web": ["_t", "url"],
- "date": ["_d", value => new Date(value.date).toISOString()],
- "proxy": ["_i", "fieldId"],
- "list": ["_l", list => {
- const results = [];
- for (const value of list.fields) {
- const term = ToSearchTerm(value);
- if (term) {
- results.push(term.value);
- }
- }
- return results.length ? results : null;
- }]
-};
-
-function ToSearchTerm(val: any): { suffix: string, value: any } | undefined {
- if (val === null || val === undefined) {
- return;
- }
- const type = val.__type || typeof val;
- let suffix = suffixMap[type];
- if (!suffix) {
- return;
- }
-
- if (Array.isArray(suffix)) {
- const accessor = suffix[1];
- if (typeof accessor === "function") {
- val = accessor(val);
- } else {
- val = val[accessor];
- }
- suffix = suffix[0];
- }
-
- return { suffix, value: val };
-}
-
-async function update() {
- console.log(green("Beginning update..."));
- await log_execution<void>({
- startMessage: "Clearing existing Solr information...",
- endMessage: "Solr information successfully cleared",
- action: Search.clear,
- color: cyan
- });
- const cursor = await log_execution({
- startMessage: "Connecting to and querying for all documents from database...",
- endMessage: ({ result, error }) => {
- const success = error === null && result !== undefined;
- if (!success) {
- console.log(red("Unable to connect to the database."));
- process.exit(0);
- }
- return "Connection successful and query complete";
- },
- action: () => Database.Instance.query({}),
- color: yellow
- });
- const updates: any[] = [];
- let numDocs = 0;
- function updateDoc(doc: any) {
- numDocs++;
- if ((numDocs % 50) === 0) {
- console.log(`Batch of 50 complete, total of ${numDocs}`);
- }
- if (doc.__type !== "Doc") {
- return;
- }
- const fields = doc.fields;
- if (!fields) {
- return;
- }
- const update: any = { id: doc._id };
- let dynfield = false;
- for (const key in fields) {
- const value = fields[key];
- const term = ToSearchTerm(value);
- if (term !== undefined) {
- const { suffix, value } = term;
- update[key + suffix] = value;
- dynfield = true;
- }
- }
- if (dynfield) {
- updates.push(update);
- }
- }
- await cursor?.forEach(updateDoc);
- const result = await log_execution({
- startMessage: `Dispatching updates for ${updates.length} documents`,
- endMessage: "Dispatched updates complete",
- action: () => Search.updateDocuments(updates),
- color: cyan
- });
- try {
- if (result) {
- const { status } = JSON.parse(result).responseHeader;
- console.log(status ? red(`Failed with status code (${status})`) : green("Success!"));
- } else {
- console.log(red("Solr is likely not running!"));
- }
- } catch (e) {
- console.log(red("Error:"));
- console.log(e);
- console.log("\n");
- }
- await cursor?.close();
- process.exit(0);
-}
-
-update(); \ No newline at end of file
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-07 18:52:03 +0000