aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/client/views/collections/CollectionSubView.tsx7
-rw-r--r--src/scraping/buxton/scraper.py14
-rw-r--r--src/server/ApiManagers/DownloadManager.ts6
-rw-r--r--src/server/ApiManagers/GooglePhotosManager.ts9
-rw-r--r--src/server/ApiManagers/UploadManager.ts3
-rw-r--r--src/server/ApiManagers/UtilManager.ts7
-rw-r--r--src/server/DashUploadUtils.ts26
7 files changed, 54 insertions, 18 deletions
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index 0eeb1c83d..9cdd48089 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -254,7 +254,12 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) {
const img = tags[0].startsWith("img") ? tags[0] : tags.length > 1 && tags[1].startsWith("img") ? tags[1] : "";
if (img) {
const split = img.split("src=\"")[1].split("\"")[0];
- const doc = Docs.Create.ImageDocument(split, { ...options, _width: 300 });
+ let source = split;
+ if (split.startsWith("data:image") && split.includes("base64")) {
+ const [{ clientAccessPath }] = await Networking.PostToServer("/uploadRemoteImage", { sources: [split] });
+ source = Utils.prepend(clientAccessPath);
+ }
+ const doc = Docs.Create.ImageDocument(source, { ...options, _width: 300 });
ImageUtils.ExtractExif(doc);
this.props.addDocument(doc);
return;
diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py
index f7a38112d..c502ac30c 100644
--- a/src/scraping/buxton/scraper.py
+++ b/src/scraping/buxton/scraper.py
@@ -115,8 +115,8 @@ def write_collection(parse_results, display_fields, storage_key, viewType):
target_collection.insert_one(view_doc)
data_doc_guid = data_doc["_id"]
- print(f"inserted view document ({view_doc_guid})")
- print(f"inserted data document ({data_doc_guid})\n")
+ # print(f"inserted view document ({view_doc_guid})")
+ # print(f"inserted data document ({data_doc_guid})\n")
return view_doc_guid
@@ -233,7 +233,7 @@ def parse_document(file_name: str):
result = {}
dir_path = image_dist + "/" + pure_name
- print(dir_path)
+ # print(dir_path)
mkdir_if_absent(dir_path)
raw = str(docx2txt.process(source + "/" + file_name, dir_path))
@@ -252,7 +252,7 @@ def parse_document(file_name: str):
medium = dir_path + "/" + image.replace(".", "_m.", 1)
copyfile(resolved, original)
copyfile(resolved, medium)
- print(f"extracted {count} images...")
+ # print(f"extracted {count} images...")
def sanitize(line): return re.sub("[\n\t]+", "", line).replace(u"\u00A0", " ").replace(
u"\u2013", "-").replace(u"\u201c", '''"''').replace(u"\u201d", '''"''').strip()
@@ -360,7 +360,7 @@ def parse_document(file_name: str):
if len(notes) > 0:
result["notes"] = listify(notes)
- print("writing child schema...")
+ # print("writing child schema...")
return {
"schema": {
@@ -392,7 +392,7 @@ def write_common_proto():
if os.path.exists(image_dist):
- shutil.rmtree(image_dist)
+ shutil.rmtree(image_dist, True)
while os.path.exists(image_dist):
pass
os.mkdir(image_dist)
@@ -415,7 +415,7 @@ parent_guid = write_collection({
"__type": "Doc"
},
"child_guids": schema_guids
-}, ["title", "short_description", "original_price"], "data", 2)
+}, ["title", "short_description", "original_price"], "data", 4)
print("appending parent schema to main workspace...\n")
target_collection.update_one(
diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts
index 1bb84f374..fad5e6789 100644
--- a/src/server/ApiManagers/DownloadManager.ts
+++ b/src/server/ApiManagers/DownloadManager.ts
@@ -254,11 +254,13 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera
// and dropped in the browser and thus hosted remotely) so we upload it
// to our server and point the zip file to it, so it can bundle up the bytes
const information = await DashUploadUtils.UploadImage(result);
- path = information.serverAccessPaths[SizeSuffix.Original];
+ path = information instanceof Error ? "" : information.serverAccessPaths[SizeSuffix.Original];
}
// write the file specified by the path to the directory in the
// zip file given by the prefix.
- file.file(path, { name: documentTitle, prefix });
+ if (path) {
+ file.file(path, { name: documentTitle, prefix });
+ }
} else {
// we've hit a collection, so we have to recurse
await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`);
diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts
index 107542ce2..1727cc5a6 100644
--- a/src/server/ApiManagers/GooglePhotosManager.ts
+++ b/src/server/ApiManagers/GooglePhotosManager.ts
@@ -88,8 +88,13 @@ export default class GooglePhotosManager extends ApiManager {
if (contents) {
const completed: Opt<DashUploadUtils.ImageUploadInformation>[] = [];
for (const item of contents.mediaItems) {
- const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl);
- const found: Opt<DashUploadUtils.ImageUploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!);
+ const results = await DashUploadUtils.InspectImage(item.baseUrl);
+ if (results instanceof Error) {
+ failed++;
+ continue;
+ }
+ const { contentSize, ...attributes } = results;
+ const found: Opt<DashUploadUtils.ImageUploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize);
if (!found) {
const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error));
if (upload) {
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index a92b613b7..4d09528f4 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -65,7 +65,8 @@ export default class UploadManager extends ApiManager {
secureHandler: async ({ req, res }) => {
const { sources } = req.body;
if (Array.isArray(sources)) {
- return res.send(await Promise.all(sources.map(url => DashUploadUtils.UploadImage(url))));
+ const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source)));
+ return res.send(results);
}
res.send();
}
diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts
index a0d0d0f4b..d7b085a30 100644
--- a/src/server/ApiManagers/UtilManager.ts
+++ b/src/server/ApiManagers/UtilManager.ts
@@ -47,7 +47,12 @@ export default class UtilManager extends ApiManager {
const onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); };
const onRejected = (err: any) => { console.error(err.message); res.send(err); };
- const tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected);
+ const tryPython3 = (reason: any) => {
+ console.log("Initial scraper failed for the following reason:");
+ console.log(red(reason.Error));
+ console.log("Falling back to python3...");
+ command_line('python3 scraper.py', cwd).then(onResolved, onRejected);
+ };
return command_line('python scraper.py', cwd).then(onResolved, tryPython3);
},
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index cb7104757..27c4bf854 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -1,4 +1,4 @@
-import { unlinkSync, createWriteStream, readFileSync, rename } from 'fs';
+import { unlinkSync, createWriteStream, readFileSync, rename, writeFile } from 'fs';
import { Utils } from '../Utils';
import * as path from 'path';
import * as sharp from 'sharp';
@@ -127,9 +127,12 @@ export namespace DashUploadUtils {
* 3) the size of the image, in bytes (4432130)
* 4) the content type of the image, i.e. image/(jpeg | png | ...)
*/
- export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise<ImageUploadInformation> => {
+ export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise<ImageUploadInformation | Error> => {
const metadata = await InspectImage(source);
- return UploadInspectedImage(metadata, filename, format, prefix);
+ if (metadata instanceof Error) {
+ return metadata;
+ }
+ return UploadInspectedImage(metadata, filename || metadata.filename, format, prefix);
};
export interface InspectionResults {
@@ -140,6 +143,7 @@ export namespace DashUploadUtils {
contentType: string;
nativeWidth: number;
nativeHeight: number;
+ filename?: string;
}
export interface EnrichedExifData {
@@ -164,7 +168,20 @@ export namespace DashUploadUtils {
*
* @param source is the path or url to the image in question
*/
- export const InspectImage = async (source: string): Promise<InspectionResults> => {
+ export const InspectImage = async (source: string): Promise<InspectionResults | Error> => {
+ let rawMatches: RegExpExecArray | null;
+ let filename: string | undefined;
+ if ((rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source)) !== null) {
+ const [ext, data] = rawMatches.slice(1, 3);
+ const resolved = filename = `upload_${Utils.GenerateGuid()}.${ext}`;
+ const error = await new Promise<Error | null>(resolve => {
+ writeFile(serverPathToFile(Directory.images, resolved), data, "base64", resolve);
+ });
+ if (error !== null) {
+ return error;
+ }
+ source = `http://localhost:1050${clientPathToFile(Directory.images, resolved)}`;
+ }
let resolvedUrl: string;
const matches = isLocal().exec(source);
if (matches === null) {
@@ -187,6 +204,7 @@ export namespace DashUploadUtils {
contentType: headers[type],
nativeWidth,
nativeHeight,
+ filename,
...results
};
};
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 16:26:33 +0000