From b285803c4e8c37302f6e02624a6127667d628305 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 25 Jun 2019 19:49:54 -0400 Subject: Youtube Api Exploration --- src/server/Message.ts | 1 + src/server/index.ts | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) (limited to 'src/server') diff --git a/src/server/Message.ts b/src/server/Message.ts index e9a8b0f0c..ee9142222 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -45,4 +45,5 @@ export namespace MessageStore { export const GetRefFields = new Message("Get Ref Fields"); export const UpdateField = new Message("Update Ref Field"); export const CreateField = new Message("Create Ref Field"); + export const YoutubeApiKey = new Message("Youtube Api Key"); } diff --git a/src/server/index.ts b/src/server/index.ts index fd66c90b4..2629519a7 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -39,6 +39,11 @@ import { debug } from 'util'; import _ = require('lodash'); const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); +//let fs = require('fs'); +let readline = require('readline'); +let { google } = require('googleapis'); +let OAuth2 = google.auth.OAuth2; + const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); @@ -310,6 +315,7 @@ server.on("connection", function (socket: Socket) { Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); + Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiKey, GetYoutubeApiKey); Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); @@ -360,6 +366,17 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v Database.Instance.getDocuments(ids, callback, "newDocuments"); } +function GetYoutubeApiKey(callback: (result?: string) => void) { + // Load client secrets from a local file. + fs.readFile('client_secret.json', function processClientSecrets(err: any, content: any) { + if (err) { + console.log('Error loading client secret file: ' + err); + return; + } + callback(content); + }); +} + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", @@ -442,4 +459,5 @@ function CreateField(newValue: any) { } server.listen(serverPort); -console.log(`listening on port ${serverPort}`); \ No newline at end of file +console.log(`listening on port ${serverPort}`); + -- cgit v1.2.3-70-g09d2 From b86050edd2da3acca258f117e8350aa8d53272d9 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Wed, 26 Jun 2019 15:14:46 -0400 Subject: Sample working onserver side --- client_secret.json | 1 + client_secret.json.json | 1 - src/client/DocServer.ts | 6 +- src/client/apis/youtube/YoutubeBox.tsx | 9 +- src/client/apis/youtube/youtubeApiSample.d.ts | 3 - src/client/apis/youtube/youtubeApiSample.js | 122 ----------------------- src/server/Message.ts | 6 +- src/server/Search.ts | 2 +- src/server/index.ts | 29 +++--- src/server/youtubeApi/youtubeApiSample.d.ts | 2 + src/server/youtubeApi/youtubeApiSample.js | 135 ++++++++++++++++++++++++++ 11 files changed, 162 insertions(+), 154 deletions(-) create mode 100644 client_secret.json delete mode 100644 client_secret.json.json delete mode 100644 src/client/apis/youtube/youtubeApiSample.d.ts delete mode 100644 src/client/apis/youtube/youtubeApiSample.js create mode 100644 src/server/youtubeApi/youtubeApiSample.d.ts create mode 100644 src/server/youtubeApi/youtubeApiSample.js (limited to 'src/server') diff --git a/client_secret.json b/client_secret.json new file mode 100644 index 000000000..a9c698421 --- /dev/null +++ b/client_secret.json @@ -0,0 +1 @@ +{"installed":{"client_id":"1005546247619-kqpnvh42mpa803tem8556b87umi4j9r0.apps.googleusercontent.com","project_id":"brown-dash","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"WshLb5TH9SdFVGGbQcnYj7IU","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file diff --git a/client_secret.json.json b/client_secret.json.json deleted file mode 100644 index 99dfbdf7b..000000000 --- a/client_secret.json.json +++ /dev/null @@ -1 +0,0 @@ -{"installed":{"client_id":"605555062960-ebbjrsv6ugkigpdoh6cdb3g6gmhdalnn.apps.googleusercontent.com","project_id":"dulcet-name-244817","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"3e8A9w5OJacQCGYIrHTu9ef2","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index c9cbce78e..9dae54d74 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -1,5 +1,5 @@ import * as OpenSocket from 'socket.io-client'; -import { MessageStore } from "./../server/Message"; +import { MessageStore, YoutubeQueryTypes } from "./../server/Message"; import { Opt } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; import { SerializationHelper } from './util/SerializationHelper'; @@ -47,8 +47,8 @@ export namespace DocServer { } } - export async function getYoutubeApiKey() { - let apiKey = await Utils.EmitCallback(_socket, MessageStore.YoutubeApiKey, undefined); + export async function getYoutubeChannels() { + let apiKey = await Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, YoutubeQueryTypes.Channels); return apiKey; } diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index ee190750f..b029c51ec 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -7,7 +7,6 @@ import { observer } from "mobx-react"; import { computed, reaction, IReactionDisposer } from 'mobx'; import { DocumentDecorations } from "../../views/DocumentDecorations"; import { InkingControl } from "../../views/InkingControl"; -import * as YoutubeApi from "./youtubeApiSample"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; @@ -15,14 +14,11 @@ import { DocServer } from "../../DocServer"; @observer export class YoutubeBox extends React.Component { - private youtubeApiKey: string = ""; public static LayoutString() { return FieldView.LayoutString(YoutubeBox); } - async componentWillMount() { - let apiKey = await DocServer.getYoutubeApiKey(); - this.youtubeApiKey = apiKey; - YoutubeApi.authorizedGetChannel(this.youtubeApiKey); + componentWillMount() { + DocServer.getYoutubeChannels(); } _ignore = 0; @@ -45,7 +41,6 @@ export class YoutubeBox extends React.Component { render() { let field = this.props.Document[this.props.fieldKey]; let view; - YoutubeApi.readFsFile(); if (field instanceof HtmlField) { view = ; } else if (field instanceof WebField) { diff --git a/src/client/apis/youtube/youtubeApiSample.d.ts b/src/client/apis/youtube/youtubeApiSample.d.ts deleted file mode 100644 index 87a669e36..000000000 --- a/src/client/apis/youtube/youtubeApiSample.d.ts +++ /dev/null @@ -1,3 +0,0 @@ - -declare const YoutubeApi: any; -export = YoutubeApi; \ No newline at end of file diff --git a/src/client/apis/youtube/youtubeApiSample.js b/src/client/apis/youtube/youtubeApiSample.js deleted file mode 100644 index 7f14f2d3e..000000000 --- a/src/client/apis/youtube/youtubeApiSample.js +++ /dev/null @@ -1,122 +0,0 @@ -import { Utils } from "tslint"; - - -// If modifying these scopes, delete your previously saved credentials -// at ~/.credentials/youtube-nodejs-quickstart.json -let SCOPES = ['https://www.googleapis.com/auth/youtube.readonly']; -let TOKEN_DIR = (process.env.HOME || process.env.HOMEPATH || - process.env.USERPROFILE) + '/.credentials/'; -let TOKEN_PATH = TOKEN_DIR + 'youtube-nodejs-quickstart.json'; - - - -function authorizedGetChannel(apiKey) { - // Authorize a client with the loaded credentials, then call the YouTube API. - authorize(JSON.parse(apiKey), getChannel); -} - - -/** - * Create an OAuth2 client with the given credentials, and then execute the - * given callback function. - * - * @param {Object} credentials The authorization client credentials. - * @param {function} callback The callback to call with the authorized client. - */ -function authorize(credentials, callback) { - let clientSecret = credentials.installed.client_secret; - let clientId = credentials.installed.client_id; - let redirectUrl = credentials.installed.redirect_uris[0]; - let oauth2Client = new OAuth2(clientId, clientSecret, redirectUrl); - - // Check if we have previously stored a token. - fs.readFile(TOKEN_PATH, function (err, token) { - if (err) { - getNewToken(oauth2Client, callback); - } else { - oauth2Client.credentials = JSON.parse(token); - callback(oauth2Client); - } - }); -} - -/** - * Get and store new token after prompting for user authorization, and then - * execute the given callback with the authorized OAuth2 client. - * - * @param {google.auth.OAuth2} oauth2Client The OAuth2 client to get token for. - * @param {getEventsCallback} callback The callback to call with the authorized - * client. - */ -function getNewToken(oauth2Client, callback) { - var authUrl = oauth2Client.generateAuthUrl({ - access_type: 'offline', - scope: SCOPES - }); - console.log('Authorize this app by visiting this url: ', authUrl); - var rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }); - rl.question('Enter the code from that page here: ', function (code) { - rl.close(); - oauth2Client.getToken(code, function (err, token) { - if (err) { - console.log('Error while trying to retrieve access token', err); - return; - } - oauth2Client.credentials = token; - storeToken(token); - callback(oauth2Client); - }); - }); -} - -/** - * Store token to disk be used in later program executions. - * - * @param {Object} token The token to store to disk. - */ -function storeToken(token) { - try { - fs.mkdirSync(TOKEN_DIR); - } catch (err) { - if (err.code != 'EEXIST') { - throw err; - } - } - fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => { - if (err) throw err; - console.log('Token stored to ' + TOKEN_PATH); - }); - console.log('Token stored to ' + TOKEN_PATH); -} - -/** - * Lists the names and IDs of up to 10 files. - * - * @param {google.auth.OAuth2} auth An authorized OAuth2 client. - */ -function getChannel(auth) { - var service = google.youtube('v3'); - service.channels.list({ - auth: auth, - part: 'snippet,contentDetails,statistics', - forUsername: 'GoogleDevelopers' - }, function (err, response) { - if (err) { - console.log('The API returned an error: ' + err); - return; - } - var channels = response.data.items; - if (channels.length == 0) { - console.log('No channel found.'); - } else { - console.log('This channel\'s ID is %s. Its title is \'%s\', and ' + - 'it has %s views.', - channels[0].id, - channels[0].snippet.title, - channels[0].statistics.viewCount); - } - }); -} \ No newline at end of file diff --git a/src/server/Message.ts b/src/server/Message.ts index ee9142222..0fc5dd4b5 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -24,6 +24,10 @@ export interface Transferable { readonly data?: any; } +export enum YoutubeQueryTypes { + Channels +} + export interface Reference { readonly id: string; } @@ -45,5 +49,5 @@ export namespace MessageStore { export const GetRefFields = new Message("Get Ref Fields"); export const UpdateField = new Message("Update Ref Field"); export const CreateField = new Message("Create Ref Field"); - export const YoutubeApiKey = new Message("Youtube Api Key"); + export const YoutubeApiQuery = new Message("Youtube Api Query"); } diff --git a/src/server/Search.ts b/src/server/Search.ts index 5ca5578a7..d776480c6 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -14,7 +14,7 @@ export class Search { }); return res; } catch (e) { - console.warn("Search error: " + e + document); + // console.warn("Search error: " + e + document); } } diff --git a/src/server/index.ts b/src/server/index.ts index 2629519a7..b36cb0bf2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,7 +24,7 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; -import { MessageStore, Transferable, Types, Diff } from "./Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType } from "./Message"; import { RouteStore } from './RouteStore'; const app = express(); const config = require('../../webpack.config'); @@ -37,15 +37,16 @@ import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); +import * as YoutubeApi from './youtubeApi/youtubeApiSample.js'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); -//let fs = require('fs'); -let readline = require('readline'); -let { google } = require('googleapis'); -let OAuth2 = google.auth.OAuth2; +// let { google } = require('googleapis'); +// let OAuth2 = google.auth.OAuth2; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); +let youtubeApiKey: string; +YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); const mongoUrl = 'mongodb://localhost:27017/Dash'; mongoose.connect(mongoUrl); @@ -315,7 +316,7 @@ server.on("connection", function (socket: Socket) { Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); - Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiKey, GetYoutubeApiKey); + Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); @@ -366,18 +367,14 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v Database.Instance.getDocuments(ids, callback, "newDocuments"); } -function GetYoutubeApiKey(callback: (result?: string) => void) { - // Load client secrets from a local file. - fs.readFile('client_secret.json', function processClientSecrets(err: any, content: any) { - if (err) { - console.log('Error loading client secret file: ' + err); - return; - } - callback(content); - }); +function HandleYoutubeQuery([type, callback]: [YoutubeQueryType, (result?: string) => void]) { + switch (type) { + case YoutubeQueryType.Channels: + YoutubeApi.authorizedGetChannel(youtubeApiKey); + break; + } } - const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", diff --git a/src/server/youtubeApi/youtubeApiSample.d.ts b/src/server/youtubeApi/youtubeApiSample.d.ts new file mode 100644 index 000000000..427f54608 --- /dev/null +++ b/src/server/youtubeApi/youtubeApiSample.d.ts @@ -0,0 +1,2 @@ +declare const YoutubeApi: any; +export = YoutubeApi; \ No newline at end of file diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js new file mode 100644 index 000000000..7d5c936f5 --- /dev/null +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -0,0 +1,135 @@ +const fs = require('fs'); +const readline = require('readline'); +const { google } = require('googleapis'); +const OAuth2 = google.auth.OAuth2; + + +// If modifying these scopes, delete your previously saved credentials +// at ~/.credentials/youtube-nodejs-quickstart.json +let SCOPES = ['https://www.googleapis.com/auth/youtube.readonly']; +let TOKEN_DIR = (process.env.HOME || process.env.HOMEPATH || + process.env.USERPROFILE) + '/.credentials/'; +let TOKEN_PATH = TOKEN_DIR + 'youtube-nodejs-quickstart.json'; + +module.exports.readApiKey = (callback) => { + fs.readFile('client_secret.json', function processClientSecrets(err, content) { + if (err) { + console.log('Error loading client secret file: ' + err); + return; + } + callback(content); + }); +} + +module.exports.authorizedGetChannel = (apiKey) => { + //this didnt get called + console.log("I get called"); + // Authorize a client with the loaded credentials, then call the YouTube API. + authorize(JSON.parse(apiKey), getChannel); +} + + +/** + * Create an OAuth2 client with the given credentials, and then execute the + * given callback function. + * + * @param {Object} credentials The authorization client credentials. + * @param {function} callback The callback to call with the authorized client. + */ +function authorize(credentials, callback) { + let clientSecret = credentials.installed.client_secret; + let clientId = credentials.installed.client_id; + let redirectUrl = credentials.installed.redirect_uris[0]; + let oauth2Client = new OAuth2(clientId, clientSecret, redirectUrl); + + // Check if we have previously stored a token. + fs.readFile(TOKEN_PATH, function (err, token) { + if (err) { + getNewToken(oauth2Client, callback); + } else { + oauth2Client.credentials = JSON.parse(token); + callback(oauth2Client); + } + }); +} + +/** + * Get and store new token after prompting for user authorization, and then + * execute the given callback with the authorized OAuth2 client. + * + * @param {google.auth.OAuth2} oauth2Client The OAuth2 client to get token for. + * @param {getEventsCallback} callback The callback to call with the authorized + * client. + */ +function getNewToken(oauth2Client, callback) { + var authUrl = oauth2Client.generateAuthUrl({ + access_type: 'offline', + scope: SCOPES + }); + console.log('Authorize this app by visiting this url: ', authUrl); + var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + rl.question('Enter the code from that page here: ', function (code) { + rl.close(); + oauth2Client.getToken(code, function (err, token) { + if (err) { + console.log('Error while trying to retrieve access token', err); + return; + } + oauth2Client.credentials = token; + storeToken(token); + callback(oauth2Client); + }); + }); +} + +/** + * Store token to disk be used in later program executions. + * + * @param {Object} token The token to store to disk. + */ +function storeToken(token) { + try { + fs.mkdirSync(TOKEN_DIR); + } catch (err) { + if (err.code != 'EEXIST') { + throw err; + } + } + fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => { + if (err) throw err; + console.log('Token stored to ' + TOKEN_PATH); + }); + console.log('Token stored to ' + TOKEN_PATH); +} + +/** + * Lists the names and IDs of up to 10 files. + * + * @param {google.auth.OAuth2} auth An authorized OAuth2 client. + */ +function getChannel(auth) { + var service = google.youtube('v3'); + service.channels.list({ + auth: auth, + part: 'snippet,contentDetails,statistics', + forUsername: 'GoogleDevelopers' + }, function (err, response) { + if (err) { + console.log('The API returned an error: ' + err); + return; + } + var channels = response.data.items; + if (channels.length == 0) { + console.log('No channel found.'); + } else { + console.log('This channel\'s ID is %s. Its title is \'%s\', and ' + + 'it has %s views.', + channels[0].id, + channels[0].snippet.title, + channels[0].statistics.viewCount); + } + }); +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 530f38e60d6289a221a463ba36af2ed22c15d8d2 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Wed, 26 Jun 2019 17:54:16 -0400 Subject: Video Search Sample --- src/client/DocServer.ts | 4 ++++ src/client/apis/youtube/YoutubeBox.tsx | 1 + src/server/Message.ts | 2 +- src/server/index.ts | 2 ++ src/server/youtubeApi/youtubeApiSample.js | 25 ++++++++++++++++++++++++- 5 files changed, 32 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 9dae54d74..b0060bfdc 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -52,6 +52,10 @@ export namespace DocServer { return apiKey; } + export function getYoutubeVideos() { + Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, YoutubeQueryTypes.SearchVideo); + } + export async function GetRefFields(ids: string[]): Promise<{ [id: string]: Opt }> { const requestedIds: string[] = []; diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index b029c51ec..dce891a07 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -19,6 +19,7 @@ export class YoutubeBox extends React.Component { componentWillMount() { DocServer.getYoutubeChannels(); + DocServer.getYoutubeVideos(); } _ignore = 0; diff --git a/src/server/Message.ts b/src/server/Message.ts index 0fc5dd4b5..87bb9d0fc 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -25,7 +25,7 @@ export interface Transferable { } export enum YoutubeQueryTypes { - Channels + Channels, SearchVideo } export interface Reference { diff --git a/src/server/index.ts b/src/server/index.ts index b36cb0bf2..3d4f59ee4 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -372,6 +372,8 @@ function HandleYoutubeQuery([type, callback]: [YoutubeQueryType, (result?: strin case YoutubeQueryType.Channels: YoutubeApi.authorizedGetChannel(youtubeApiKey); break; + case YoutubeQueryType.SearchVideo: + YoutubeApi.authorizedGetVideos(youtubeApiKey); } } diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index 7d5c936f5..35d74c62f 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -23,11 +23,16 @@ module.exports.readApiKey = (callback) => { module.exports.authorizedGetChannel = (apiKey) => { //this didnt get called - console.log("I get called"); + console.log("I get called ", apiKey); + console.log(TOKEN_PATH); // Authorize a client with the loaded credentials, then call the YouTube API. authorize(JSON.parse(apiKey), getChannel); } +module.exports.authorizedGetVideos = (apiKey) => { + authorize(JSON.parse(apiKey), getSampleVideos); +} + /** * Create an OAuth2 client with the given credentials, and then execute the @@ -132,4 +137,22 @@ function getChannel(auth) { channels[0].statistics.viewCount); } }); +} + +function getSampleVideos(auth) { + let service = google.youtube('v3'); + service.search.list({ + auth: auth, + part: 'id, snippet', + type: 'video', + q: 'istanbul', + maxResults: 3 + }, function (err, response) { + if (err) { + console.log('The API returned an error: ' + err); + return; + } + let videos = response.data.items; + console.log('Videos found: ' + videos[0].id.videoId, " ", videos[0].snippet.title); + }); } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 21bc14319013e4757ca24f56a685b7d75eaa259a Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Wed, 26 Jun 2019 19:15:50 -0400 Subject: Searching through document for a youtube video done --- src/client/DocServer.ts | 6 +++--- src/client/apis/youtube/YoutubeBox.tsx | 26 +++++++++++++++----------- src/server/Message.ts | 7 ++++++- src/server/index.ts | 8 ++++---- src/server/youtubeApi/youtubeApiSample.js | 12 ++++++------ 5 files changed, 34 insertions(+), 25 deletions(-) (limited to 'src/server') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index b0060bfdc..65d662ebc 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -48,12 +48,12 @@ export namespace DocServer { } export async function getYoutubeChannels() { - let apiKey = await Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, YoutubeQueryTypes.Channels); + let apiKey = await Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.Channels }); return apiKey; } - export function getYoutubeVideos() { - Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, YoutubeQueryTypes.SearchVideo); + export function getYoutubeVideos(videoTitle: string) { + Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.SearchVideo, userInput: videoTitle }); } diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index dce891a07..b044e2a05 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -4,7 +4,7 @@ import { FieldViewProps, FieldView } from "../../views/nodes/FieldView"; import { HtmlField } from "../../../new_fields/HtmlField"; import { WebField } from "../../../new_fields/URLField"; import { observer } from "mobx-react"; -import { computed, reaction, IReactionDisposer } from 'mobx'; +import { computed, reaction, IReactionDisposer, observable } from 'mobx'; import { DocumentDecorations } from "../../views/DocumentDecorations"; import { InkingControl } from "../../views/InkingControl"; import { Utils } from "../../../Utils"; @@ -14,12 +14,12 @@ import { DocServer } from "../../DocServer"; @observer export class YoutubeBox extends React.Component { + @observable YoutubeSearchElement: HTMLInputElement | undefined; public static LayoutString() { return FieldView.LayoutString(YoutubeBox); } componentWillMount() { DocServer.getYoutubeChannels(); - DocServer.getYoutubeVideos(); } _ignore = 0; @@ -39,19 +39,23 @@ export class YoutubeBox extends React.Component { e.stopPropagation(); } } + + onEnterKeyDown = (e: React.KeyboardEvent) => { + if (e.keyCode === 13) { + let submittedTitle = this.YoutubeSearchElement!.value; + console.log(submittedTitle); + this.YoutubeSearchElement!.value = ""; + this.YoutubeSearchElement!.blur(); + DocServer.getYoutubeVideos(submittedTitle); + + } + } + render() { let field = this.props.Document[this.props.fieldKey]; - let view; - if (field instanceof HtmlField) { - view = ; - } else if (field instanceof WebField) { - view = ; + // } else if (this.videoClicked) { + // return ; + // } } else { return (null); } } @action - embedVideoOnClick = (videoId: string) => { + embedVideoOnClick = (videoId: string, filteredTitle: string) => { let embeddedUrl = "https://www.youtube.com/embed/" + videoId; this.selectedVideoUrl = embeddedUrl; - this.searchResultsFound = false; + let addFunction = this.props.addDocument!; + let newVideoX = NumCast(this.props.Document.x) + NumCast(this.props.Document.width); + let newVideoY = NumCast(this.props.Document.y) + NumCast(this.props.Document.height); + + addFunction(Docs.Create.VideoDocument(embeddedUrl, { title: filteredTitle, width: 400, height: 315, x: newVideoX, y: newVideoY })); + + //this.props.addDocument(Docs.Create.VideoDocument(embeddedUrl, { title: embeddedUrl, width: 400, height: 315 })); + //this.searchResultsFound = false; this.videoClicked = true; } diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index e95f99015..f875812d5 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -153,7 +153,7 @@ function getSampleVideos(auth, args) { return; } let videos = response.data.items; - console.log('Videos found: ' + videos[0].id.videoId, " ", videos[0].snippet.title); + console.log('Videos found: ' + videos[0].id.videoId, " ", unescape(videos[0].snippet.title)); args.callBack(videos); }); } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 4446a3a52c4cf4b03c201ab2d6a9179647686e40 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Mon, 22 Jul 2019 18:47:14 -0400 Subject: Pulled Duration and ViewCount details, Need to csss duration --- src/client/DocServer.ts | 4 ++ src/client/apis/youtube/YoutubeBox.scss | 37 ++++++++++++++++++ src/client/apis/youtube/YoutubeBox.tsx | 64 ++++++++++++++++++++++++++++--- src/server/Message.ts | 3 +- src/server/index.ts | 2 + src/server/youtubeApi/youtubeApiSample.js | 21 ++++++++++ 6 files changed, 125 insertions(+), 6 deletions(-) (limited to 'src/server') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index bc5819061..8a9abb514 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -168,6 +168,10 @@ export namespace DocServer { Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.SearchVideo, userInput: videoTitle }, callBack); } + export function getYoutubeVideoDetails(videoIds: string, callBack: (videoDetails: any[]) => void) { + Utils.EmitCallback(_socket, MessageStore.YoutubeApiQuery, { type: YoutubeQueryTypes.VideoDetails, videoIds: videoIds }, callBack); + } + /** * Given a list of Doc GUIDs, this utility function will asynchronously attempt to each id's associated diff --git a/src/client/apis/youtube/YoutubeBox.scss b/src/client/apis/youtube/YoutubeBox.scss index 5b539b463..00979f945 100644 --- a/src/client/apis/youtube/YoutubeBox.scss +++ b/src/client/apis/youtube/YoutubeBox.scss @@ -19,6 +19,27 @@ li:hover { display: inline-flex; height: 175px; + .video_duration { + margin: 0; + padding: 0; + border: 0; + background: transparent; + display: inline-block; + position: absolute; + bottom: 0; + right: 0; + margin: 4px; + color: #FFFFFF; + background-color: rgba(0, 0, 0, 0.80); + padding: 2px 4px; + border-radius: 2px; + letter-spacing: .5px; + font-size: 1.2rem; + font-weight: 500; + line-height: 1.2rem; + + } + .textual_info { font-family: Arial, Helvetica, sans-serif; @@ -80,6 +101,22 @@ li:hover { font-weight: 400; text-transform: none; } + + .viewCount { + + margin-left: 8px; + padding: 0; + border: 0; + background: transparent; + color: #606060; + max-width: 100%; + line-height: 1.8rem; + max-height: 3.6rem; + overflow: hidden; + font-size: 1.3rem; + font-weight: 400; + text-transform: none; + } diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index 7ac8d06f6..824a0251d 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -26,12 +26,15 @@ export class YoutubeBox extends React.Component { @observable videoClicked: boolean = false; @observable selectedVideoUrl: string = ""; @observable lisOfBackUp: JSX.Element[] = []; + @observable videoIds: string | undefined; + @observable videoDetails: any[] = []; public static LayoutString() { return FieldView.LayoutString(YoutubeBox); } async componentWillMount() { //DocServer.getYoutubeChannels(); + //DocServer.getYoutubeVideoDetails("Ks-_Mh1QhMc, 1NmvhSmN2uM", (results: any[]) => console.log("Details results: ", results)); let castedBackUpDocs = Cast(this.props.Document.cachedSearch, listSpec(Doc)); if (!castedBackUpDocs) { this.props.Document.cachedSearch = castedBackUpDocs = new List(); @@ -95,6 +98,15 @@ export class YoutubeBox extends React.Component { this.searchResults = videos; if (this.searchResults.length > 0) { this.searchResultsFound = true; + this.videoIds = ""; + videos.forEach((video) => { + if (this.videoIds === "") { + this.videoIds = video.id.videoId; + } else { + this.videoIds = this.videoIds! + ", " + video.id.videoId; + } + }); + DocServer.getYoutubeVideoDetails(this.videoIds, this.processVideoDetails); this.backUpSearchResults(videos); if (this.videoClicked) { this.videoClicked = false; @@ -102,6 +114,12 @@ export class YoutubeBox extends React.Component { } } + @action + processVideoDetails = (videoDetails: any[]) => { + this.videoDetails = videoDetails; + console.log("Detail Res: ", this.videoDetails); + } + backUpSearchResults = (videos: any[]) => { let newCachedList = new List(); this.props.Document.cachedSearch = newCachedList; @@ -190,28 +208,64 @@ export class YoutubeBox extends React.Component { } } + convertIsoTimeToDuration = (isoDur: string) => { + + let convertedTime = isoDur.replace(/D|H|M/g, ":").replace(/P|T|S/g, "").split(":"); + + if (1 === convertedTime.length) { + 2 !== convertedTime[0].length && (convertedTime[0] = "0" + convertedTime[0]), convertedTime[0] = "0:" + convertedTime[0]; + } else { + for (var r = 1, l = convertedTime.length - 1; l >= r; r++) { + 2 !== convertedTime[r].length && (convertedTime[r] = "0" + convertedTime[r]); + } + } + + return convertedTime.join(":"); + } + + abbreviateViewCount = (viewCount: number) => { + if (viewCount < 1000) { + return viewCount.toString(); + } else if (viewCount >= 1000 && viewCount < 1000000) { + return (Math.trunc(viewCount / 1000)) + "K"; + } else if (viewCount >= 1000000 && viewCount < 1000000000) { + return (Math.trunc(viewCount / 1000000)) + "M"; + } else if (viewCount >= 1000000000) { + return (Math.trunc(viewCount / 1000000000)) + "B"; + } + } + renderSearchResultsOrVideo = () => { if (this.searchResultsFound) { if (this.searchResults.length !== 0) { return
    - {this.searchResults.map((video) => { + {this.searchResults.map((video, index) => { let filteredTitle = this.filterYoutubeTitleResult(video.snippet.title); let channelTitle = video.snippet.channelTitle; let videoDescription = video.snippet.description; let pusblishDate = this.roundPublishTime2(video.snippet.publishedAt); - // let duration = video.contentDetails.duration; - //let viewCount = video.statistics.viewCount; + let duration; + let viewCount; + if (this.videoDetails.length !== 0) { + duration = this.convertIsoTimeToDuration(this.videoDetails[index].contentDetails.duration); + viewCount = this.abbreviateViewCount(this.videoDetails[index].statistics.viewCount); + } //this.roundPublishTime(pusblishDate); //this.roundPublishTime2(video.snippet.publishedAt); + return
  • this.embedVideoOnClick(video.id.videoId, filteredTitle)} key={Utils.GenerateGuid()}>
    - +
    + + {duration} +
    {filteredTitle} {channelTitle} + {viewCount} {pusblishDate} - {/*
    {viewCount}
    */}

    {videoDescription}

    +
    • ; diff --git a/src/server/Message.ts b/src/server/Message.ts index 1e29aef0b..aaee143e8 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -25,12 +25,13 @@ export interface Transferable { } export enum YoutubeQueryTypes { - Channels, SearchVideo + Channels, SearchVideo, VideoDetails } export interface YoutubeQueryInput { readonly type: YoutubeQueryTypes; readonly userInput?: string; + readonly videoIds?: string; } export interface Reference { diff --git a/src/server/index.ts b/src/server/index.ts index 60e34de8c..dfbc1a468 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -537,6 +537,8 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any break; case YoutubeQueryType.SearchVideo: YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); + case YoutubeQueryType.VideoDetails: + YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); } } diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index f875812d5..cf41a33e7 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -33,6 +33,10 @@ module.exports.authorizedGetVideos = (apiKey, userInput, callBack) => { authorize(JSON.parse(apiKey), getSampleVideos, { userInput: userInput, callBack: callBack }); } +module.exports.authorizedGetVideoDetails = (apiKey, videoIds, callBack) => { + authorize(JSON.parse(apiKey), getVideoDetails, { videoIds: videoIds, callBack: callBack }); +} + /** * Create an OAuth2 client with the given credentials, and then execute the @@ -156,4 +160,21 @@ function getSampleVideos(auth, args) { console.log('Videos found: ' + videos[0].id.videoId, " ", unescape(videos[0].snippet.title)); args.callBack(videos); }); +} + +function getVideoDetails(auth, args) { + let service = google.youtube('v3'); + service.videos.list({ + auth: auth, + part: 'contentDetails, statistics', + id: args.videoIds + }, function (err, response) { + if (err) { + console.log('The API returned an error: ' + err); + return; + } + let videoDetails = response.data.items; + console.log('Video Details founds: ', videoDetails); + args.callBack(videoDetails); + }); } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 0591b8f1e60d1285fd9aac3e61160824948a166b Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 23 Jul 2019 17:33:31 -0400 Subject: Everything related to search stored --- src/client/apis/youtube/YoutubeBox.tsx | 123 +++++++++++++++++++----------- src/client/documents/Documents.ts | 2 +- src/server/youtubeApi/youtubeApiSample.js | 4 - 3 files changed, 81 insertions(+), 48 deletions(-) (limited to 'src/server') diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index 373eee5c4..e630c11ae 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -16,6 +16,16 @@ import { Doc, DocListCastAsync } from "../../../new_fields/Doc"; import { listSpec } from "../../../new_fields/Schema"; import { List } from "../../../new_fields/List"; +interface VideoTemplate { + thumbnailUrl: string; + videoTitle: string; + videoId: string; + duration: string; + channelTitle: string; + viewCount: string; + publishDate: string; + videoDescription: string; +} @observer export class YoutubeBox extends React.Component { @@ -28,58 +38,71 @@ export class YoutubeBox extends React.Component { @observable lisOfBackUp: JSX.Element[] = []; @observable videoIds: string | undefined; @observable videoDetails: any[] = []; + @observable curVideoTemplates: VideoTemplate[] = []; public static LayoutString() { return FieldView.LayoutString(YoutubeBox); } async componentWillMount() { //DocServer.getYoutubeChannels(); - //DocServer.getYoutubeVideoDetails("Ks-_Mh1QhMc, 1NmvhSmN2uM", (results: any[]) => console.log("Details results: ", results)); - let castedBackUpDocs = Cast(this.props.Document.cachedSearch, listSpec(Doc)); let castedSearchBackUp = Cast(this.props.Document.cachedSearchResults, Doc); let awaitedBackUp = await castedSearchBackUp; + let castedDetailBackUp = Cast(this.props.Document.cachedDetails, Doc); + let awaitedDetails = await castedDetailBackUp; - console.log("Backup results: ", awaitedBackUp); - console.log("Original Backup results: ", castedBackUpDocs); - - let json = Cast(awaitedBackUp!.json, Doc); - let jsonList = await DocListCastAsync(json); - console.log("Fucked up list: ", jsonList); - for (let video of jsonList!) { - let videoId = await Cast(video.id, Doc); - let id = StrCast(videoId!.videoId); - console.log("ID: ", id); - } + let jsonList = await DocListCastAsync(awaitedBackUp!.json); + let jsonDetailList = await DocListCastAsync(awaitedDetails!.json); - if (!castedBackUpDocs) { - this.props.Document.cachedSearch = castedBackUpDocs = new List(); - } - if (castedBackUpDocs.length !== 0) { - + if (jsonList!.length !== 0) { runInAction(() => this.searchResultsFound = true); + let index = 0; + for (let video of jsonList!) { + + let videoId = await Cast(video.id, Doc); + let id = StrCast(videoId!.videoId); + let snippet = await Cast(video.snippet, Doc); + let videoTitle = this.filterYoutubeTitleResult(StrCast(snippet!.title)); + let thumbnail = await Cast(snippet!.thumbnails, Doc); + let thumbnailMedium = await Cast(thumbnail!.medium, Doc); + let thumbnailUrl = StrCast(thumbnailMedium!.url); + let videoDescription = StrCast(snippet!.description); + let pusblishDate = (this.roundPublishTime2(StrCast(snippet!.publishedAt)))!; + let channelTitle = StrCast(snippet!.channelTitle); + let duration: string; + let viewCount: string; + if (jsonDetailList!.length !== 0) { + let contentDetails = await Cast(jsonDetailList![index].contentDetails, Doc); + let statistics = await Cast(jsonDetailList![index].statistics, Doc); + duration = this.convertIsoTimeToDuration(StrCast(contentDetails!.duration)); + viewCount = this.abbreviateViewCount(NumCast(statistics!.viewCount))!; + } + index = index + 1; + let newTemplate: VideoTemplate = { videoId: id, videoTitle: videoTitle, thumbnailUrl: thumbnailUrl, publishDate: pusblishDate, channelTitle: channelTitle, videoDescription: videoDescription, duration: duration!, viewCount: viewCount! }; + runInAction(() => this.curVideoTemplates.push(newTemplate)); + + // runInAction(() => this.lisOfBackUp.push(( + //
  • this.embedVideoOnClick(id, videoTitle)} key={Utils.GenerateGuid() + id}> + //
    + //
    + // + // {duration} + //
    + //
    + // {videoTitle} + // {channelTitle} + // {viewCount} + // {pusblishDate} + //

    {videoDescription}

    + + //
    + //
    + //
    • ) + // )); - for (let videoBackUp of castedBackUpDocs) { - let curBackUp = await videoBackUp; - let videoId = StrCast(curBackUp.videoId); - let videoTitle = StrCast(curBackUp.videoTitle); - let thumbnailUrl = StrCast(curBackUp.thumbnailUrl); - runInAction(() => this.lisOfBackUp.push(( -
  • this.embedVideoOnClick(videoId, videoTitle)} - key={Utils.GenerateGuid()} - > - - {videoTitle} -
    • ) - )); } - - } - - } _ignore = 0; @@ -134,11 +157,10 @@ export class YoutubeBox extends React.Component { @action processVideoDetails = (videoDetails: any[]) => { this.videoDetails = videoDetails; - console.log("Detail Res: ", this.videoDetails); + this.props.Document.cachedDetails = Docs.Get.DocumentHierarchyFromJson(videoDetails, "detailBackUp"); } backUpSearchResults = (videos: any[]) => { - console.log("Res: ", videos); this.props.Document.cachedSearchResults = Docs.Get.DocumentHierarchyFromJson(videos, "videosBackUp"); let newCachedList = new List(); this.props.Document.cachedSearch = newCachedList; @@ -164,7 +186,6 @@ export class YoutubeBox extends React.Component { let videoYearDif = curDate.getFullYear() - date.getFullYear(); let videoMonthDif = curDate.getMonth() - date.getMonth(); let videoDayDif = curDate.getDay() - date.getDay(); - console.log("video day dif: ", videoDayDif, " first day: ", curDate.getDay(), " second day: ", date.getDay()); let videoHoursDif = curDate.getHours() - date.getHours(); let videoMinutesDif = curDate.getMinutes() - date.getMinutes(); let videoSecondsDif = curDate.getSeconds() - date.getSeconds(); @@ -182,7 +203,6 @@ export class YoutubeBox extends React.Component { return videoSecondsDif + " seconds ago"; } - console.log("Date : ", date); } roundPublishTime2 = (publishTime: string) => { @@ -290,8 +310,26 @@ export class YoutubeBox extends React.Component { ; })}
; - } else if (this.lisOfBackUp.length !== 0) { - return
    {this.lisOfBackUp}
; + } else if (this.curVideoTemplates.length !== 0) { + return
    + {this.curVideoTemplates.map((video: VideoTemplate) => { + return
  • this.embedVideoOnClick(video.videoId, video.videoTitle)} key={Utils.GenerateGuid()}> +
    +
    + + {video.duration} +
    +
    + {video.videoTitle} + {video.channelTitle} + {video.viewCount} + {video.publishDate} +

    {video.videoDescription}

    +
    +
    +
    • ; + })} +
; } } else { return (null); @@ -301,7 +339,6 @@ export class YoutubeBox extends React.Component { @action embedVideoOnClick = (videoId: string, filteredTitle: string) => { let embeddedUrl = "https://www.youtube.com/embed/" + videoId; - console.log("EmbeddedUrl: ", embeddedUrl); this.selectedVideoUrl = embeddedUrl; let addFunction = this.props.addDocument!; let newVideoX = NumCast(this.props.Document.x); diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 191be9b7d..333e9859b 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -498,7 +498,7 @@ export namespace Docs { const convertObject = (object: any, title?: string): Doc => { let target = new Doc(), result: Opt; Object.keys(object).map(key => (result = toField(object[key], key)) && (target[key] = result)); - title && (target.title = title); + title && !target.title && (target.title = title); return target; }; diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index cf41a33e7..4fede08aa 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -23,8 +23,6 @@ module.exports.readApiKey = (callback) => { module.exports.authorizedGetChannel = (apiKey) => { //this didnt get called - console.log("I get called ", apiKey); - console.log(TOKEN_PATH); // Authorize a client with the loaded credentials, then call the YouTube API. authorize(JSON.parse(apiKey), getChannel); } @@ -157,7 +155,6 @@ function getSampleVideos(auth, args) { return; } let videos = response.data.items; - console.log('Videos found: ' + videos[0].id.videoId, " ", unescape(videos[0].snippet.title)); args.callBack(videos); }); } @@ -174,7 +171,6 @@ function getVideoDetails(auth, args) { return; } let videoDetails = response.data.items; - console.log('Video Details founds: ', videoDetails); args.callBack(videoDetails); }); } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 778286579008b57a76fbf82235348b613f5c1a5b Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 23 Jul 2019 18:56:23 -0400 Subject: Fixed document --- src/client/apis/youtube/YoutubeBox.tsx | 24 ++---------------------- src/server/youtubeApi/youtubeApiSample.js | 2 +- 2 files changed, 3 insertions(+), 23 deletions(-) (limited to 'src/server') diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index e630c11ae..d2f5112c2 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -81,26 +81,6 @@ export class YoutubeBox extends React.Component { index = index + 1; let newTemplate: VideoTemplate = { videoId: id, videoTitle: videoTitle, thumbnailUrl: thumbnailUrl, publishDate: pusblishDate, channelTitle: channelTitle, videoDescription: videoDescription, duration: duration!, viewCount: viewCount! }; runInAction(() => this.curVideoTemplates.push(newTemplate)); - - // runInAction(() => this.lisOfBackUp.push(( - //
  • this.embedVideoOnClick(id, videoTitle)} key={Utils.GenerateGuid() + id}> - //
    - //
    - // - // {duration} - //
    - //
    - // {videoTitle} - // {channelTitle} - // {viewCount} - // {pusblishDate} - //

    {videoDescription}

    - - //
    - //
    - //
    • ) - // )); - } } } @@ -146,6 +126,7 @@ export class YoutubeBox extends React.Component { this.videoIds = this.videoIds! + ", " + video.id.videoId; } }); + console.log("Video Ids: ", this.videoIds); DocServer.getYoutubeVideoDetails(this.videoIds, this.processVideoDetails); this.backUpSearchResults(videos); if (this.videoClicked) { @@ -289,8 +270,7 @@ export class YoutubeBox extends React.Component { duration = this.convertIsoTimeToDuration(this.videoDetails[index].contentDetails.duration); viewCount = this.abbreviateViewCount(this.videoDetails[index].statistics.viewCount); } - //this.roundPublishTime(pusblishDate); - //this.roundPublishTime2(video.snippet.publishedAt); + return
  • this.embedVideoOnClick(video.id.videoId, filteredTitle)} key={Utils.GenerateGuid()}>
    diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index 4fede08aa..f81f0dfb5 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -167,7 +167,7 @@ function getVideoDetails(auth, args) { id: args.videoIds }, function (err, response) { if (err) { - console.log('The API returned an error: ' + err); + console.log('The API returned an error from details: ' + err); return; } let videoDetails = response.data.items; -- cgit v1.2.3-70-g09d2 From 6cdebe507b777f60a0e30a1d7a75300304fbce09 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 23 Jul 2019 19:17:03 -0400 Subject: Refactor and flag --- src/client/apis/youtube/YoutubeBox.tsx | 63 ++++++++++++++++--------------- src/server/youtubeApi/youtubeApiSample.js | 4 +- 2 files changed, 35 insertions(+), 32 deletions(-) (limited to 'src/server') diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index 414abcc15..8d6334c6e 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -58,37 +58,40 @@ export class YoutubeBox extends React.Component { let awaitedDetails = await castedDetailBackUp; - - let jsonList = await DocListCastAsync(awaitedBackUp!.json); - let jsonDetailList = await DocListCastAsync(awaitedDetails!.json); - - if (jsonList!.length !== 0) { - runInAction(() => this.searchResultsFound = true); - let index = 0; - //getting the necessary information from backUps and building templates that will be used to map in render - for (let video of jsonList!) { - - let videoId = await Cast(video.id, Doc); - let id = StrCast(videoId!.videoId); - let snippet = await Cast(video.snippet, Doc); - let videoTitle = this.filterYoutubeTitleResult(StrCast(snippet!.title)); - let thumbnail = await Cast(snippet!.thumbnails, Doc); - let thumbnailMedium = await Cast(thumbnail!.medium, Doc); - let thumbnailUrl = StrCast(thumbnailMedium!.url); - let videoDescription = StrCast(snippet!.description); - let pusblishDate = (this.roundPublishTime(StrCast(snippet!.publishedAt)))!; - let channelTitle = StrCast(snippet!.channelTitle); - let duration: string; - let viewCount: string; - if (jsonDetailList!.length !== 0) { - let contentDetails = await Cast(jsonDetailList![index].contentDetails, Doc); - let statistics = await Cast(jsonDetailList![index].statistics, Doc); - duration = this.convertIsoTimeToDuration(StrCast(contentDetails!.duration)); - viewCount = this.abbreviateViewCount(NumCast(statistics!.viewCount))!; + if (awaitedBackUp) { + + + let jsonList = await DocListCastAsync(awaitedBackUp!.json); + let jsonDetailList = await DocListCastAsync(awaitedDetails!.json); + + if (jsonList!.length !== 0) { + runInAction(() => this.searchResultsFound = true); + let index = 0; + //getting the necessary information from backUps and building templates that will be used to map in render + for (let video of jsonList!) { + + let videoId = await Cast(video.id, Doc); + let id = StrCast(videoId!.videoId); + let snippet = await Cast(video.snippet, Doc); + let videoTitle = this.filterYoutubeTitleResult(StrCast(snippet!.title)); + let thumbnail = await Cast(snippet!.thumbnails, Doc); + let thumbnailMedium = await Cast(thumbnail!.medium, Doc); + let thumbnailUrl = StrCast(thumbnailMedium!.url); + let videoDescription = StrCast(snippet!.description); + let pusblishDate = (this.roundPublishTime(StrCast(snippet!.publishedAt)))!; + let channelTitle = StrCast(snippet!.channelTitle); + let duration: string; + let viewCount: string; + if (jsonDetailList!.length !== 0) { + let contentDetails = await Cast(jsonDetailList![index].contentDetails, Doc); + let statistics = await Cast(jsonDetailList![index].statistics, Doc); + duration = this.convertIsoTimeToDuration(StrCast(contentDetails!.duration)); + viewCount = this.abbreviateViewCount(NumCast(statistics!.viewCount))!; + } + index = index + 1; + let newTemplate: VideoTemplate = { videoId: id, videoTitle: videoTitle, thumbnailUrl: thumbnailUrl, publishDate: pusblishDate, channelTitle: channelTitle, videoDescription: videoDescription, duration: duration!, viewCount: viewCount! }; + runInAction(() => this.curVideoTemplates.push(newTemplate)); } - index = index + 1; - let newTemplate: VideoTemplate = { videoId: id, videoTitle: videoTitle, thumbnailUrl: thumbnailUrl, publishDate: pusblishDate, channelTitle: channelTitle, videoDescription: videoDescription, duration: duration!, viewCount: viewCount! }; - runInAction(() => this.curVideoTemplates.push(newTemplate)); } } } diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index f81f0dfb5..9853241b6 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -28,7 +28,7 @@ module.exports.authorizedGetChannel = (apiKey) => { } module.exports.authorizedGetVideos = (apiKey, userInput, callBack) => { - authorize(JSON.parse(apiKey), getSampleVideos, { userInput: userInput, callBack: callBack }); + authorize(JSON.parse(apiKey), getVideos, { userInput: userInput, callBack: callBack }); } module.exports.authorizedGetVideoDetails = (apiKey, videoIds, callBack) => { @@ -141,7 +141,7 @@ function getChannel(auth) { }); } -function getSampleVideos(auth, args) { +function getVideos(auth, args) { let service = google.youtube('v3'); service.search.list({ auth: auth, -- cgit v1.2.3-70-g09d2 From 2edcfb8a755b7fec7f937f135c794cbadbe0c94e Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 23 Jul 2019 19:40:47 -0400 Subject: Error on api extra call fixed. Don't know why would it get called twice tho --- src/server/youtubeApi/youtubeApiSample.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/server') diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index 9853241b6..ec532c78e 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -160,6 +160,9 @@ function getVideos(auth, args) { } function getVideoDetails(auth, args) { + if (args.videoIds === undefined) { + return; + } let service = google.youtube('v3'); service.videos.list({ auth: auth, -- cgit v1.2.3-70-g09d2 From bb607ecb02a7ecb8fe782d60e10ebf5bdedd790b Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 29 Jul 2019 14:08:46 -0400 Subject: Fixed bad headers crashing cors proxy on server --- src/server/index.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index 40c0e7981..adf218be6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -437,8 +437,22 @@ app.post(RouteStore.forgot, postForgot); app.get(RouteStore.reset, getReset); app.post(RouteStore.reset, postReset); -app.use(RouteStore.corsProxy, (req, res) => - req.pipe(request(decodeURIComponent(req.url.substring(1)))).pipe(res)); +const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; +app.use(RouteStore.corsProxy, (req, res) => { + req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { + const headers = Object.keys(res.headers); + headers.forEach(headerName => { + const header = res.headers[headerName]; + if (Array.isArray(header)) { + res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); + } else if (header) { + if (headerCharRegex.test(header as any)) { + delete res.headers[headerName]; + } + } + }); + }).pipe(res); +}); app.get(RouteStore.delete, (req, res) => { if (release) { -- cgit v1.2.3-70-g09d2 From 14455e7be675ed3cb04fa645df79845b2aecd0d4 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 00:09:01 -0400 Subject: Started implementing downloading "exported" version of documents --- src/server/database.ts | 23 +++++++++++++++++ src/server/index.ts | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) (limited to 'src/server') diff --git a/src/server/database.ts b/src/server/database.ts index 7f5331998..acb6ce751 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -126,6 +126,29 @@ export class Database { } } + public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments") { + if (this.db) { + const visited = new Set(); + while (ids.length) { + const count = Math.min(ids.length, 1000); + const index = ids.length - count; + const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); + if (!fetchIds.length) { + continue; + } + const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments")); + for (const doc of docs) { + const id = doc.id; + visited.add(id); + ids.push(...fn(doc)); + } + } + + } else { + this.onConnect.push(() => this.visit(ids, fn, collectionName)); + } + } + public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise { if (this.db) { let cursor = this.db.collection(collectionName).find(query); diff --git a/src/server/index.ts b/src/server/index.ts index adf218be6..230c574cf 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -177,6 +177,75 @@ function msToTime(duration: number) { return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } +app.get("/serializeDoc/:docId", async (req, res) => { + const files: { [name: string]: string[] } = {}; + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const ids: string[] = []; + for (const key in doc) { + if (!doc.hasOwnProperty(key)) { + continue; + } + const field = doc[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "list") { + ids.push(...fn(field.fields)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + const ext = path.extname(pathname); + const fileName = path.basename(pathname, ext); + let exts = files[fileName]; + if (!exts) { + files[fileName] = exts = []; + } + exts.push(ext); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + const ext = path.extname(pathname); + const fileName = path.basename(pathname, ext); + let exts = files[fileName]; + if (!exts) { + files[fileName] = exts = []; + } + exts.push(ext); + } + } + + docs[doc.id] = doc; + return ids; + }; + Database.Instance.visit([req.params.docId], fn); +}); + +app.get("/downloadId/:docId", (req, res) => { + res.download(`/serializeDoc/${req.params.docId}`, `DocumentExport.zip`); +}); + app.get("/whosOnline", (req, res) => { let users: any = { active: {}, inactive: {} }; const now = Date.now(); -- cgit v1.2.3-70-g09d2 From b8ace05da169802bc18d138b83576dbab98053e9 Mon Sep 17 00:00:00 2001 From: bob Date: Tue, 30 Jul 2019 12:50:18 -0400 Subject: added youtube as button snapshots. changed video default to 10. --- src/client/views/nodes/VideoBox.tsx | 49 +++++++++++++++++++++---------- src/server/youtubeApi/youtubeApiSample.js | 2 +- 2 files changed, 35 insertions(+), 16 deletions(-) (limited to 'src/server') diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index 34cb47b20..d2657227a 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -8,7 +8,6 @@ import { Cast, FieldValue, NumCast } from "../../../new_fields/Types"; import { VideoField } from "../../../new_fields/URLField"; import { RouteStore } from "../../../server/RouteStore"; import { Utils } from "../../../Utils"; -import { DocServer } from "../../DocServer"; import { Docs, DocUtils } from "../../documents/Documents"; import { ContextMenu } from "../ContextMenu"; import { ContextMenuProps } from "../ContextMenuItem"; @@ -21,6 +20,9 @@ import { pageSchema } from "./ImageBox"; import "./VideoBox.scss"; import { library } from "@fortawesome/fontawesome-svg-core"; import { faVideo } from "@fortawesome/free-solid-svg-icons"; +import { CompileScript } from "../../util/Scripting"; +import { Doc } from "../../../new_fields/Doc"; +import { ScriptField } from "../../../new_fields/ScriptField"; type VideoDocument = makeInterface<[typeof positionSchema, typeof pageSchema]>; const VideoDocument = makeInterface(positionSchema, pageSchema); @@ -165,21 +167,38 @@ export class VideoBox extends DocComponent(VideoD this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height); } - //convert to desired file format - var dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' - // if you want to preview the captured image, - let filename = encodeURIComponent("snapshot" + this.props.Document.title + "_" + this.props.Document.curPage).replace(/\./g, ""); - VideoBox.convertDataUri(dataUrl, filename).then(returnedFilename => { - if (returnedFilename) { - let url = Utils.prepend(returnedFilename); - let imageSummary = Docs.Create.ImageDocument(url, { - x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y), - width: 150, height: height / width * 150, title: "--snapshot" + NumCast(this.props.Document.curPage) + " image-" - }); - this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(imageSummary, false); - DocUtils.MakeLink(imageSummary, this.props.Document); + if (!this._videoRef) { // can't find a way to take snapshots of videos + let b = Docs.Create.ButtonDocument({ + x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y), width: 150, height: 50, title: NumCast(this.props.Document.curPage).toString() + }); + const script = CompileScript(`this.props.Document.curPage = ${NumCast(this.props.Document.curPage)}`, { + params: { this: Doc.name }, + typecheck: false, + editable: true, + }); + if (!script.compiled) { + console.log(script.errors.map(error => error.messageText).join("\n")); + return; } - }); + b.onClick = new ScriptField(script); + this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(b, false); + } else { + //convert to desired file format + var dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' + // if you want to preview the captured image, + let filename = encodeURIComponent("snapshot" + this.props.Document.title + "_" + this.props.Document.curPage).replace(/\./g, ""); + VideoBox.convertDataUri(dataUrl, filename).then(returnedFilename => { + if (returnedFilename) { + let url = Utils.prepend(returnedFilename); + let imageSummary = Docs.Create.ImageDocument(url, { + x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y), + width: 150, height: height / width * 150, title: "--snapshot" + NumCast(this.props.Document.curPage) + " image-" + }); + this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(imageSummary, false); + DocUtils.MakeLink(imageSummary, this.props.Document); + } + }); + } }, icon: "expand-arrows-alt" }); diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js index ec532c78e..50b3c7b38 100644 --- a/src/server/youtubeApi/youtubeApiSample.js +++ b/src/server/youtubeApi/youtubeApiSample.js @@ -148,7 +148,7 @@ function getVideos(auth, args) { part: 'id, snippet', type: 'video', q: args.userInput, - maxResults: 3 + maxResults: 10 }, function (err, response) { if (err) { console.log('The API returned an error: ' + err); -- cgit v1.2.3-70-g09d2 From 26086ee95a9a16486d637aa43c96638b6154379f Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 16:02:42 -0400 Subject: Added document export and import --- package.json | 4 + .../collectionFreeForm/CollectionFreeFormView.tsx | 39 +++++- src/client/views/nodes/DocumentView.tsx | 10 ++ src/scraping/buxton/scraper.py | 2 + src/server/database.ts | 37 +++++- src/server/index.ts | 142 +++++++++++++++++---- 6 files changed, 202 insertions(+), 32 deletions(-) (limited to 'src/server') diff --git a/package.json b/package.json index 37052fde3..b29355738 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,9 @@ "@hig/theme-context": "^2.1.3", "@hig/theme-data": "^2.3.3", "@trendmicro/react-dropdown": "^1.3.0", + "@types/adm-zip": "^0.4.32", "@types/animejs": "^2.0.2", + "@types/archiver": "^3.0.0", "@types/async": "^2.4.1", "@types/bcrypt-nodejs": "0.0.30", "@types/bluebird": "^3.5.25", @@ -105,6 +107,8 @@ "@types/uuid": "^3.4.4", "@types/webpack": "^4.4.25", "@types/youtube": "0.0.38", + "adm-zip": "^0.4.13", + "archiver": "^3.0.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 8dac785e1..cbab14976 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols"; import { InkField, StrokeData } from "../../../../new_fields/InkField"; import { createSchema, makeInterface } from "../../../../new_fields/Schema"; import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types"; -import { emptyFunction, returnOne } from "../../../../Utils"; +import { emptyFunction, returnOne, Utils } from "../../../../Utils"; import { DocumentManager } from "../../../util/DocumentManager"; import { DragManager } from "../../../util/DragManager"; import { HistoryUtil } from "../../../util/History"; @@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; import { library } from "@fortawesome/fontawesome-svg-core"; import { faEye } from "@fortawesome/free-regular-svg-icons"; -import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons"; +import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons"; import { undo } from "prosemirror-history"; import { number } from "prop-types"; import { ContextMenu } from "../../ContextMenu"; +import { RouteStore } from "../../../../server/RouteStore"; +import { DocServer } from "../../../DocServer"; -library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass); +library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload); export const panZoomSchema = createSchema({ panX: "number", @@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } - onContextMenu = () => { + onContextMenu = (e: React.MouseEvent) => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, @@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); }, icon: "paint-brush" }); + ContextMenu.Instance.addItem({ + description: "Import document", icon: "upload", event: () => { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".zip"; + input.onchange = async _e => { + const files = input.files; + if (!files) return; + const file = files[0]; + let formData = new FormData(); + formData.append('file', file); + formData.append('remap', "true"); + const upload = Utils.prepend("/uploadDoc"); + const response = await fetch(upload, { method: "POST", body: formData }); + const json = await response.json(); + if (json === "error") { + return; + } + const doc = await DocServer.GetRefField(json); + if (!doc || !(doc instanceof Doc)) { + return; + } + const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY); + doc.x = x, doc.y = y; + this.addDocument(doc, false); + }; + input.click(); + } + }); } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4b5cf3a43..58e2443c2 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -45,6 +45,7 @@ const JsxParser = require('react-jsx-parser').default; //TODO Why does this need library.add(fa.faTrash); library.add(fa.faShare); +library.add(fa.faDownload); library.add(fa.faExpandArrowsAlt); library.add(fa.faCompressArrowsAlt); library.add(fa.faLayerGroup); @@ -597,6 +598,15 @@ export class DocumentView extends DocComponent(Docu copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" }); cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" }); } + cm.addItem({ + description: "Download document", icon: "download", event: () => { + const a = document.createElement("a"); + const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); + a.href = url; + a.download = `DocExport-${this.props.Document[Id]}.zip`; + a.click(); + } + }); cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" }); type User = { email: string, userDocumentId: string }; let usersMenu: ContextMenuProps[] = []; diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 1ff0e3b31..8ff7cb223 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -236,6 +236,8 @@ def parse_document(file_name: str): view_guids.append(write_image(pure_name, image)) copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) + copyfile(dir_path + "/" + image, dir_path + + "/" + image) os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/server/database.ts b/src/server/database.ts index acb6ce751..a7254fb0c 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -17,7 +17,7 @@ export class Database { }); } - public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) { + public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { if (this.db) { let collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; @@ -30,7 +30,7 @@ export class Database { delete this.currentWrites[id]; } resolve(); - callback(); + callback(err, res); }); }); }; @@ -41,6 +41,30 @@ export class Database { } } + public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { + if (this.db) { + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.replaceOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); + } + } + public delete(query: any, collectionName?: string): Promise; public delete(id: string, collectionName?: string): Promise; public delete(id: any, collectionName = Database.DocumentsCollection) { @@ -126,7 +150,7 @@ export class Database { } } - public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments") { + public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { if (this.db) { const visited = new Set(); while (ids.length) { @@ -145,7 +169,12 @@ export class Database { } } else { - this.onConnect.push(() => this.visit(ids, fn, collectionName)); + return new Promise(res => { + this.onConnect.push(() => { + this.visit(ids, fn, collectionName); + res(); + }); + }); } } diff --git a/src/server/index.ts b/src/server/index.ts index 230c574cf..2fa5132d0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -27,6 +27,7 @@ import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, Message } from "./Message"; import { RouteStore } from './RouteStore'; +import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); import { createCanvas, loadImage, Canvas } from "canvas"; @@ -39,7 +40,10 @@ import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); +import * as Archiver from 'archiver'; +import * as AdmZip from 'adm-zip'; import { Response } from 'express-serve-static-core'; +import { DocComponent } from '../client/views/DocComponent'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -177,16 +181,21 @@ function msToTime(duration: number) { return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } -app.get("/serializeDoc/:docId", async (req, res) => { - const files: { [name: string]: string[] } = {}; +async function getDocs(id: string) { + const files = new Set(); const docs: { [id: string]: any } = {}; const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } const ids: string[] = []; - for (const key in doc) { - if (!doc.hasOwnProperty(key)) { + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { continue; } - const field = doc[key]; + const field = doc.fields[key]; if (field === undefined || field === null) { continue; } @@ -194,7 +203,7 @@ app.get("/serializeDoc/:docId", async (req, res) => { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { ids.push(field.fieldId); } else if (field.__type === "list") { - ids.push(...fn(field.fields)); + ids.push(...fn(field)); } else if (typeof field === "string") { const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; let match: string[] | null; @@ -215,35 +224,120 @@ app.get("/serializeDoc/:docId", async (req, res) => { while ((match = re2.exec(field.Data)) !== null) { const urlString = match[1]; const pathname = new URL(urlString).pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { const url = new URL(field.url); const pathname = url.pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } - docs[doc.id] = doc; + if (doc.id) { + docs[doc.id] = doc; + } return ids; }; - Database.Instance.visit([req.params.docId], fn); + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} +app.get("/serializeDoc/:docId", async (req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); }); -app.get("/downloadId/:docId", (req, res) => { - res.download(`/serializeDoc/${req.params.docId}`, `DocumentExport.zip`); +app.get("/downloadId/:docId", async (req, res) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); +}); + +app.post("/uploadDoc", (req, res) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path = files[name].path; + const zip = new AdmZip(path); + zip.getEntries().forEach(entry => { + if (!entry.name.startsWith("files/")) return; + zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false); + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); }); app.get("/whosOnline", (req, res) => { -- cgit v1.2.3-70-g09d2 From 8ca17d379ce7d3cc751408553b6819223d31a3e0 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 16:22:54 -0400 Subject: Fixed import thing --- src/scraping/buxton/scraper.py | 2 -- src/server/index.ts | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 8ff7cb223..f0f45d8f9 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -237,8 +237,6 @@ def parse_document(file_name: str): copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) copyfile(dir_path + "/" + image, dir_path + - "/" + image) - os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/server/index.ts b/src/server/index.ts index 281fc1eb2..1912cf5c1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -205,6 +205,10 @@ async function getDocs(id: string) { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } } else if (field.__type === "list") { ids.push(...fn(field)); } else if (typeof field === "string") { @@ -293,6 +297,10 @@ app.post("/uploadDoc", (req, res) => { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } } else if (field.__type === "list") { mapFn(field); } else if (typeof field === "string") { -- cgit v1.2.3-70-g09d2