diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/apis/google/GooglePhotosUploadUtils.ts | 58 | ||||
| -rw-r--r-- | src/server/credentials/google_docs_token.json | 2 | ||||
| -rw-r--r-- | src/server/index.ts | 53 |
3 files changed, 73 insertions, 40 deletions
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index e91f8352b..3989590c6 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -6,6 +6,7 @@ import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; +import { NewMediaItem } from '../..'; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -39,7 +40,7 @@ export namespace GooglePhotosUploadUtils { }; export const DispatchGooglePhotosUpload = async (url: string) => { - const body = await request(url, { encoding: null }); + const body = await request(url, { encoding: null }).catch(error => console.log("Error in streaming body!", error)); const parameters = { method: 'POST', headers: { @@ -56,36 +57,37 @@ export namespace GooglePhotosUploadUtils { return reject(error); } resolve(body); - })); + }).catch(error => console.log("Error in literal uploading process to Google's servers!", error))).catch(error => console.log("Error in literal uploading process to Google's servers!", error)); }; - export const CreateMediaItems = async (newMediaItems: any[], album?: { id: string }): Promise<MediaItemCreationResult> => { - const quota = newMediaItems.length; - let handled = 0; + export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => { const newMediaItemResults: NewMediaItemResult[] = []; - while (handled < quota) { - const cap = Math.min(newMediaItems.length, handled + 50); - const batch = newMediaItems.slice(handled, cap); - console.log(batch.length); - const parameters = { - method: 'POST', - headers: headers('json'), - uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems: batch } as any, - json: true - }; - album && (parameters.body.albumId = album.id); - newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => { - request(parameters, (error, _response, body) => { - if (error) { - reject(error); - } else { - resolve(body); - } - }); - })).newMediaItemResults); - handled = cap; - } + await newMediaItems.batch({ + size: 50, + action: { + handler: async (batch: NewMediaItem[]) => { + console.log(batch.length); + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems: batch } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => { + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + })).newMediaItemResults); + }, + interval: 1000 + } + }); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index bdeca837b..d8e0eae21 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCCBwLh8M4qd5ApvvhgMeCvbQidOUehUNU2fj3RH6Zx8D3rnCooiVgxoWbJ2ddS3a0_PGAQvCA7-GAeS70wUny80VKgCLjNbTlZkuxaRqpAd5yFGuWzcRljXrEIuA7EVu0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568394019509}
\ No newline at end of file +{"access_token":"ya29.GlyDB0wsV3-oS6q5TFuJSmH1YP_SPf_X6RHaJVmfqj0NTCtaPLFonZRxdT52kUkiHJgAoRizxZvlSIGptXKfnmG4BFouhgyo9ZKP0QtOH-kPR9b9x5WhGCd5NWqz0A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568412547334}
\ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index fdcc79b4d..542a4ea65 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -46,6 +46,7 @@ const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; +const extensions = require("../client/util/UtilExtensions"); const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -827,20 +828,50 @@ app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.R const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; +export interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; +} + +Array.prototype.batch = extensions.Batch; + app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { - const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; + const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); - const newMediaItems = await Promise.all(media.map(async element => { - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url).catch(error => { - console.log("Dispatching upload error!"); - console.log(error); + + const newMediaItems: NewMediaItem[] = []; + let failed = 0; + const size = 25; + + try { + await mediaInput.batch({ + size, + action: { + handler: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + await Promise.all(batch.map(async element => { + console.log(`Uploading ${element.url} to Google's servers...`); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (uploadToken) { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); + } else { + console.log("FAIL!", element.url, element.description); + failed++; + } + })); + }, + interval: 3000 + } }); - return !uploadToken ? undefined : { - description: element.description, - simpleMediaItem: { uploadToken } - }; - })); - if (!newMediaItems.every(item => item)) { + } catch (e) { + console.log("WHAT HAPPENED?"); + console.log(e); + } + if (failed) { return _error(res, tokenError); } GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( |