aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/client/northstar/utils/Extensions.ts27
-rw-r--r--src/client/util/Import & Export/DirectoryImportBox.tsx33
-rw-r--r--src/client/util/UtilExtensions.ts39
-rw-r--r--src/client/views/Main.tsx8
-rw-r--r--src/server/apis/google/GooglePhotosUploadUtils.ts58
-rw-r--r--src/server/credentials/google_docs_token.json2
-rw-r--r--src/server/index.ts53
7 files changed, 155 insertions, 65 deletions
diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts
index ab9384f1f..720f4a062 100644
--- a/src/client/northstar/utils/Extensions.ts
+++ b/src/client/northstar/utils/Extensions.ts
@@ -5,6 +5,8 @@ interface String {
hasNewline(): boolean;
}
+const extensions = require(".././/.//../util/UtilExtensions");
+
String.prototype.ReplaceAll = function (toReplace: string, replacement: string): string {
var target = this;
return target.split(toReplace).join(replacement);
@@ -18,6 +20,31 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri
return target;
};
+interface Action<T> {
+ handler: (batch: T[]) => any;
+ interval?: number;
+}
+
+interface BatchParameters<T> {
+ size: number;
+ action?: Action<T>;
+}
+
+interface Array<T> {
+ batch(parameters: BatchParameters<T>): Promise<T[][]>;
+ lastElement(): T;
+}
+
+Array.prototype.batch = extensions.Batch;
+
+Array.prototype.lastElement = function <T>() {
+ if (!this.length) {
+ return undefined;
+ }
+ const last: T = this[this.length - 1];
+ return last;
+};
+
interface Math {
log10(val: number): number;
}
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index 260c6a629..5915f3412 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -92,29 +92,28 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
this.completed = 0;
});
- let sizes = [];
- let modifiedDates = [];
+ let sizes: number[] = [];
+ let modifiedDates: number[] = [];
- let i = 0;
const uploads: FileResponse[] = [];
- const batchSize = 15;
- while (i < validated.length) {
- const cap = Math.min(validated.length, i + batchSize);
- let formData = new FormData();
- const batch = validated.slice(i, cap);
+ await validated.batch({
+ size: 15,
+ action: {
+ handler: async (batch: File[]) => {
+ sizes.push(...batch.map(file => file.size));
+ modifiedDates.push(...batch.map(file => file.lastModified));
- sizes.push(...batch.map(file => file.size));
- modifiedDates.push(...batch.map(file => file.lastModified));
+ let formData = new FormData();
+ batch.forEach(file => formData.append(Utils.GenerateGuid(), file));
+ const parameters = { method: 'POST', body: formData };
- batch.forEach(file => formData.append(Utils.GenerateGuid(), file));
- const parameters = { method: 'POST', body: formData };
- uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json()));
-
- runInAction(() => this.completed += batch.length);
- i = cap;
- }
+ uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json()));
+ runInAction(() => this.completed += batch.length);
+ }
+ }
+ });
await Promise.all(uploads.map(async upload => {
const type = upload.type;
diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts
new file mode 100644
index 000000000..1e277b242
--- /dev/null
+++ b/src/client/util/UtilExtensions.ts
@@ -0,0 +1,39 @@
+module.exports.Batch = async function <T>(parameters: BatchParameters<T>) {
+ const { size, action } = parameters;
+ const batches: T[][] = [];
+ let i = 0;
+ while (i < this.length) {
+ const cap = Math.min(i + size, this.length);
+ batches.push(this.slice(i, cap));
+ i = cap;
+ }
+ console.log(`Beginning action on ${this.length} elements, split into ${batches.length} groups => ${batches.map(batch => batch.length).join(", ")}`);
+ if (action) {
+ const { handler, interval } = action;
+ if (!interval || batches.length === 1) {
+ for (let batch of batches) {
+ await handler(batch);
+ }
+ } else {
+ return new Promise<T[][]>(resolve => {
+ const iterator = batches[Symbol.iterator]();
+ const quota = batches.length;
+ let completed = 0;
+ const tag = setInterval(async () => {
+ const next = iterator.next();
+ if (next.done) {
+ clearInterval(tag);
+ return;
+ }
+ const batch = next.value;
+ console.log(`Handling next batch with ${batch.length} elements`);
+ await handler(batch);
+ if (++completed === quota) {
+ resolve(batches);
+ }
+ }, interval);
+ });
+ }
+ }
+ return batches;
+}; \ No newline at end of file
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index b623cab4e..aa002cee9 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -20,14 +20,6 @@ String.prototype.hasNewline = function () {
return this.endsWith("\n");
};
-(Array.prototype as any).lastElement = function (this: any[]) {
- if (!this.length) {
- return undefined;
- }
- return this[this.length - 1];
-};
-
-
let swapDocs = async () => {
let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc);
// Docs.Prototypes.MainLinkDocument().allLinks = new List<Doc>();
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index e91f8352b..3989590c6 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -6,6 +6,7 @@ import * as path from 'path';
import { Opt } from '../../../new_fields/Doc';
import * as sharp from 'sharp';
import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes';
+import { NewMediaItem } from '../..';
const uploadDirectory = path.join(__dirname, "../../public/files/");
@@ -39,7 +40,7 @@ export namespace GooglePhotosUploadUtils {
};
export const DispatchGooglePhotosUpload = async (url: string) => {
- const body = await request(url, { encoding: null });
+ const body = await request(url, { encoding: null }).catch(error => console.log("Error in streaming body!", error));
const parameters = {
method: 'POST',
headers: {
@@ -56,36 +57,37 @@ export namespace GooglePhotosUploadUtils {
return reject(error);
}
resolve(body);
- }));
+ }).catch(error => console.log("Error in literal uploading process to Google's servers!", error))).catch(error => console.log("Error in literal uploading process to Google's servers!", error));
};
- export const CreateMediaItems = async (newMediaItems: any[], album?: { id: string }): Promise<MediaItemCreationResult> => {
- const quota = newMediaItems.length;
- let handled = 0;
+ export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => {
const newMediaItemResults: NewMediaItemResult[] = [];
- while (handled < quota) {
- const cap = Math.min(newMediaItems.length, handled + 50);
- const batch = newMediaItems.slice(handled, cap);
- console.log(batch.length);
- const parameters = {
- method: 'POST',
- headers: headers('json'),
- uri: prepend('mediaItems:batchCreate'),
- body: { newMediaItems: batch } as any,
- json: true
- };
- album && (parameters.body.albumId = album.id);
- newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => {
- request(parameters, (error, _response, body) => {
- if (error) {
- reject(error);
- } else {
- resolve(body);
- }
- });
- })).newMediaItemResults);
- handled = cap;
- }
+ await newMediaItems.batch({
+ size: 50,
+ action: {
+ handler: async (batch: NewMediaItem[]) => {
+ console.log(batch.length);
+ const parameters = {
+ method: 'POST',
+ headers: headers('json'),
+ uri: prepend('mediaItems:batchCreate'),
+ body: { newMediaItems: batch } as any,
+ json: true
+ };
+ album && (parameters.body.albumId = album.id);
+ newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => {
+ request(parameters, (error, _response, body) => {
+ if (error) {
+ reject(error);
+ } else {
+ resolve(body);
+ }
+ });
+ })).newMediaItemResults);
+ },
+ interval: 1000
+ }
+ });
return { newMediaItemResults };
};
diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json
index bdeca837b..d8e0eae21 100644
--- a/src/server/credentials/google_docs_token.json
+++ b/src/server/credentials/google_docs_token.json
@@ -1 +1 @@
-{"access_token":"ya29.ImCCBwLh8M4qd5ApvvhgMeCvbQidOUehUNU2fj3RH6Zx8D3rnCooiVgxoWbJ2ddS3a0_PGAQvCA7-GAeS70wUny80VKgCLjNbTlZkuxaRqpAd5yFGuWzcRljXrEIuA7EVu0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568394019509} \ No newline at end of file
+{"access_token":"ya29.GlyDB0wsV3-oS6q5TFuJSmH1YP_SPf_X6RHaJVmfqj0NTCtaPLFonZRxdT52kUkiHJgAoRizxZvlSIGptXKfnmG4BFouhgyo9ZKP0QtOH-kPR9b9x5WhGCd5NWqz0A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568412547334} \ No newline at end of file
diff --git a/src/server/index.ts b/src/server/index.ts
index fdcc79b4d..542a4ea65 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -46,6 +46,7 @@ const MongoStore = require('connect-mongo')(session);
const mongoose = require('mongoose');
const probe = require("probe-image-size");
import * as qs from 'query-string';
+const extensions = require("../client/util/UtilExtensions");
const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
let youtubeApiKey: string;
@@ -827,20 +828,50 @@ app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.R
const tokenError = "Unable to successfully upload bytes for all images!";
const mediaError = "Unable to convert all uploaded bytes to media items!";
+export interface NewMediaItem {
+ description: string;
+ simpleMediaItem: {
+ uploadToken: string;
+ };
+}
+
+Array.prototype.batch = extensions.Batch;
+
app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
- const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
+ const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath });
- const newMediaItems = await Promise.all(media.map(async element => {
- const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url).catch(error => {
- console.log("Dispatching upload error!");
- console.log(error);
+
+ const newMediaItems: NewMediaItem[] = [];
+ let failed = 0;
+ const size = 25;
+
+ try {
+ await mediaInput.batch({
+ size,
+ action: {
+ handler: async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
+ await Promise.all(batch.map(async element => {
+ console.log(`Uploading ${element.url} to Google's servers...`);
+ const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
+ if (uploadToken) {
+ newMediaItems.push({
+ description: element.description,
+ simpleMediaItem: { uploadToken }
+ });
+ } else {
+ console.log("FAIL!", element.url, element.description);
+ failed++;
+ }
+ }));
+ },
+ interval: 3000
+ }
});
- return !uploadToken ? undefined : {
- description: element.description,
- simpleMediaItem: { uploadToken }
- };
- }));
- if (!newMediaItems.every(item => item)) {
+ } catch (e) {
+ console.log("WHAT HAPPENED?");
+ console.log(e);
+ }
+ if (failed) {
return _error(res, tokenError);
}
GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then(
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-07 04:54:47 +0000