diff options
| -rw-r--r-- | package-lock.json | 181 | ||||
| -rw-r--r-- | package.json | 1 | ||||
| -rw-r--r-- | src/client/documents/Documents.ts | 3 | ||||
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 2 | ||||
| -rw-r--r-- | src/fields/URLField.ts | 2 | ||||
| -rw-r--r-- | src/server/ApiManagers/AzureManager.ts | 74 | ||||
| -rw-r--r-- | src/server/DashUploadUtils.ts | 25 |
7 files changed, 274 insertions, 14 deletions
diff --git a/package-lock.json b/package-lock.json index 4da0600b0..c44ebd4a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,121 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@azure/abort-controller": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", + "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", + "requires": { + "tslib": "^2.2.0" + } + }, + "@azure/core-auth": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.4.0.tgz", + "integrity": "sha512-HFrcTgmuSuukRf/EdPmqBrc5l6Q5Uu+2TbuhaKbgaCpP2TfAeiNaQPAadxO+CYBRHGUzIDteMAjFspFLDLnKVQ==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "tslib": "^2.2.0" + } + }, + "@azure/core-http": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.2.tgz", + "integrity": "sha512-o1wR9JrmoM0xEAa0Ue7Sp8j+uJvmqYaGoHOCT5qaVYmvgmnZDC0OvQimPA/JR3u77Sz6D1y3Xmk1y69cDU9q9A==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/core-util": "^1.1.1", + "@azure/logger": "^1.0.0", + "@types/node-fetch": "^2.5.0", + "@types/tunnel": "^0.0.3", + "form-data": "^4.0.0", + "node-fetch": "^2.6.7", + "process": "^0.11.10", + "tslib": "^2.2.0", + "tunnel": "^0.0.6", + "uuid": "^8.3.0", + "xml2js": "^0.5.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, + "@azure/core-lro": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.3.tgz", + "integrity": "sha512-ubkOf2YCnVtq7KqEJQqAI8dDD5rH1M6OP5kW0KO/JQyTaxLA0N0pjFWvvaysCj9eHMNBcuuoZXhhl0ypjod2DA==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.2.0" + } + }, + "@azure/core-paging": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz", + "integrity": "sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==", + "requires": { + "tslib": "^2.2.0" + } + }, + "@azure/core-tracing": { + "version": "1.0.0-preview.13", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz", + "integrity": "sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ==", + "requires": { + "@opentelemetry/api": "^1.0.1", + "tslib": "^2.2.0" + } + }, + "@azure/core-util": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.3.2.tgz", + "integrity": "sha512-2bECOUh88RvL1pMZTcc6OzfobBeWDBf5oBbhjIhT1MV9otMVWCzpOJkkiKtrnO88y5GGBelgY8At73KGAdbkeQ==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "tslib": "^2.2.0" + } + }, + "@azure/logger": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz", + "integrity": "sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg==", + "requires": { + "tslib": "^2.2.0" + } + }, + "@azure/storage-blob": { + "version": "12.14.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.14.0.tgz", + "integrity": "sha512-g8GNUDpMisGXzBeD+sKphhH5yLwesB4JkHr1U6be/X3F+cAMcyGLPD1P89g2M7wbEtUJWoikry1rlr83nNRBzg==", + "requires": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-http": "^3.0.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/logger": "^1.0.0", + "events": "^3.0.0", + "tslib": "^2.2.0" + } + }, "@babel/code-frame": { "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", @@ -1472,6 +1587,11 @@ "@octokit/openapi-types": "^16.0.0" } }, + "@opentelemetry/api": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", + "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==" + }, "@popperjs/core": { "version": "2.11.7", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.7.tgz", @@ -2320,8 +2440,28 @@ "@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", - "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==", - "dev": true + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" + }, + "@types/node-fetch": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz", + "integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==", + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + }, + "dependencies": { + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } }, "@types/nodemailer": { "version": "4.6.8", @@ -2852,6 +2992,14 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" }, + "@types/tunnel": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", + "integrity": "sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==", + "requires": { + "@types/node": "*" + } + }, "@types/typescript": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/typescript/-/typescript-2.0.0.tgz", @@ -8723,8 +8871,7 @@ "events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "dev": true + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" }, "eventsource": { "version": "2.0.2", @@ -20922,6 +21069,11 @@ } } }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "saxes": { "version": "3.1.11", "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz", @@ -23038,6 +23190,11 @@ } } }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -24595,6 +24752,22 @@ "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", "dev": true }, + "xml2js": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", + "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "dependencies": { + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + } + } + }, "xmlbuilder": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.0.0.tgz", diff --git a/package.json b/package.json index 6cd271c96..3a3d17517 100644 --- a/package.json +++ b/package.json @@ -130,6 +130,7 @@ "webpack-hot-middleware": "^2.25.1" }, "dependencies": { + "@azure/storage-blob": "^12.14.0", "@emotion/react": "^11.11.0", "@emotion/styled": "^11.11.0", "@ffmpeg/core": "0.10.0", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index ffde9fe1b..ac6c48619 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -1739,7 +1739,8 @@ export namespace DocUtils { return; } const full = { ...options, _width: 400, title: name }; - const pathname = Utils.prepend(result.accessPaths.agnostic.client); + // const pathname = Utils.prepend(result.accessPaths.agnostic.client); + const pathname = result.accessPaths.azure.client; const doc = await DocUtils.DocumentFromType(type, pathname, full, overwriteDoc); if (doc) { const proto = Doc.GetProto(doc); diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 9acbee1e7..de1365adc 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -295,7 +295,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp choosePath(url: URL) { const lower = url.href.toLowerCase(); if (url.protocol === 'data') return url.href; - if (url.href.indexOf(window.location.origin) === -1) return Utils.CorsProxy(url.href); + if (url.href.indexOf(window.location.origin) === -1 && url.href.indexOf("dashblobstore") === -1) return Utils.CorsProxy(url.href); if (!/\.(png|jpg|jpeg|gif|webp)$/.test(lower)) return `/assets/unknown-file-icon-hi.png`; const ext = extname(url.href); diff --git a/src/fields/URLField.ts b/src/fields/URLField.ts index 8ac20b1e5..8db4d6003 100644 --- a/src/fields/URLField.ts +++ b/src/fields/URLField.ts @@ -25,7 +25,7 @@ export abstract class URLField extends ObjectField { constructor(url: URL | string) { super(); if (typeof url === 'string') { - url = url.startsWith('http') ? new URL(url) : new URL(url, window.location.origin); + url = (url.startsWith('http') || url.startsWith('https')) ? new URL(url) : new URL(url, window.location.origin); } this.url = url; } diff --git a/src/server/ApiManagers/AzureManager.ts b/src/server/ApiManagers/AzureManager.ts new file mode 100644 index 000000000..e105f5d80 --- /dev/null +++ b/src/server/ApiManagers/AzureManager.ts @@ -0,0 +1,74 @@ +import { ContainerClient, BlobServiceClient } from "@azure/storage-blob"; +// import * as dotenv from 'dotenv'; +import * as fs from "fs"; +import { Readable, Stream } from "stream"; +// dotenv.config(); +const AZURE_STORAGE_CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=dashblobstore;AccountKey=3i+E5XkCz3TJ0m5QOatiEnbRACz9V1qCW72L6ldiYGH1tLdfJWa2eQoRfYmPA68lx1a6YAcfYJfWHadIxQvhGQ==;EndpointSuffix=core.windows.net"; + +export class AzureManager { + private _containerClient: ContainerClient; + private _blobServiceClient: BlobServiceClient; + private static _instance: AzureManager | undefined; + + public static CONTAINER_NAME = "dashmedia"; + public static STORAGE_ACCOUNT_NAME = "dashblobstore"; + + constructor() { + if (!AZURE_STORAGE_CONNECTION_STRING) { + throw Error("Azure Storage Connection String Not Found"); + } + // this._blobServiceClient = BlobServiceClient.fromConnectionString(process.env.AZURE_STORAGE_CONNECTION_STRING); + this._blobServiceClient = BlobServiceClient.fromConnectionString(AZURE_STORAGE_CONNECTION_STRING); + this._containerClient = this.BlobServiceClient.getContainerClient(AzureManager.CONTAINER_NAME); + } + + public static get Instance() { + return this._instance = this._instance ?? new AzureManager(); + } + + public get BlobServiceClient() { + return this._blobServiceClient; + } + + public get ContainerClient() { + return this._containerClient; + } + + public static UploadBlob(filename: string, filepath: string, filetype: string) { + console.log("Upload Blob File"); + const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); + const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }}; + const stream = fs.createReadStream(filepath); + return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions); + } + + public static UploadBlobStream(stream: Readable, filename: string, filetype: string) { + console.log("Upload Blob Stream: %s, %s", filename, filetype); + const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); + const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }}; + return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions); + } + + public static DeleteBlob(filename: string) { + console.log("Delete Blob from filename"); + const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); + return blockBlobClient.deleteIfExists(); + } + + public static async GetBlobs() { + console.log("Get Blobs"); + const foundBlobs = []; + for await (const blob of this.Instance.ContainerClient.listBlobsFlat()) { + console.log(`${blob.name}`); + + const blobItem = { + url : `https://${AzureManager.STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AzureManager.CONTAINER_NAME}/${blob.name}`, + name : blob.name + } + + foundBlobs.push(blobItem); + } + + return foundBlobs; + } +}
\ No newline at end of file diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index eaaac4e6d..53f2d6501 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -6,7 +6,7 @@ import { createReadStream, createWriteStream, existsSync, readFileSync, rename, import * as path from 'path'; import { basename } from 'path'; import * as sharp from 'sharp'; -import { Stream } from 'stream'; +import { Readable, Stream } from 'stream'; import { filesDirectory, publicDirectory } from '.'; import { Opt } from '../fields/Doc'; import { ParsedPDF } from '../server/PdfTypes'; @@ -17,6 +17,7 @@ import { resolvedServerUrl } from './server_Initialization'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; import request = require('request-promise'); import formidable = require('formidable'); +import { AzureManager } from './ApiManagers/AzureManager'; const spawn = require('child_process').spawn; const { exec } = require('child_process'); const parse = require('pdf-parse'); @@ -193,7 +194,7 @@ export namespace DashUploadUtils { switch (category) { case 'image': if (imageFormats.includes(format)) { - const result = await UploadImage(path, basename(path)); + const result = await UploadImage(path, basename(path), name); return { source: file, result }; } fs.unlink(path, () => {}); @@ -323,12 +324,13 @@ export namespace DashUploadUtils { * 3) the size of the image, in bytes (4432130) * 4) the content type of the image, i.e. image/(jpeg | png | ...) */ - export const UploadImage = async (source: string, filename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => { + export const UploadImage = async (source: string, filename?: string, originalFilename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => { const metadata = await InspectImage(source); if (metadata instanceof Error) { return { name: metadata.name, message: metadata.message }; } - return UploadInspectedImage(metadata, filename || metadata.filename, prefix); + console.log(originalFilename); + return UploadInspectedImage(metadata, filename || metadata.filename, originalFilename, prefix); }; export async function buildFileDirectories() { @@ -478,17 +480,21 @@ export namespace DashUploadUtils { }; } - export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = '', cleanUp = true): Promise<Upload.ImageInformation> => { + export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, originalFilename?: string, prefix = '', cleanUp = true): Promise<Upload.ImageInformation> => { const { requestable, source, ...remaining } = metadata; const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split('/')[1].toLowerCase()}`; const { images } = Directory; const information: Upload.ImageInformation = { accessPaths: { agnostic: getAccessPaths(images, resolved), + azure: { + client: `https://dashblobstore.blob.core.windows.net/dashmedia/${originalFilename}`, + server: `https://dashblobstore.blob.core.windows.net/dashmedia/${originalFilename}` + } }, ...metadata, }; - const writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images)); + const writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images), originalFilename, metadata.contentType); for (const suffix of Object.keys(writtenFiles)) { information.accessPaths[suffix] = getAccessPaths(images, writtenFiles[suffix]); } @@ -533,16 +539,21 @@ export namespace DashUploadUtils { force: true, }; - export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string) { + export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string, originalFilename?: string, contentType?: string) { + console.log("resized original filename: ", originalFilename); const writtenFiles: { [suffix: string]: string } = {}; for (const { resizer, suffix } of resizers(path.extname(outputFileName))) { const outputPath = path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix))); + console.log(`https://dashblobstore.blob.core.windows.net/dashmedia/${InjectSize(originalFilename!, suffix)}`); await new Promise<void>(async (resolve, reject) => { const source = streamProvider(); let readStream: Stream = source instanceof Promise ? await source : source; if (resizer) { readStream = readStream.pipe(resizer.withMetadata()); } + if(originalFilename && contentType) { + AzureManager.UploadBlobStream(readStream as Readable, InjectSize(originalFilename, suffix), contentType); + } readStream.pipe(createWriteStream(outputPath)).on('close', resolve).on('error', reject); }); } |