aboutsummaryrefslogtreecommitdiff
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/Message.ts11
-rw-r--r--src/server/database.ts56
-rw-r--r--src/server/index.ts209
-rw-r--r--src/server/youtubeApi/youtubeApiSample.d.ts2
-rw-r--r--src/server/youtubeApi/youtubeApiSample.js179
5 files changed, 451 insertions, 6 deletions
diff --git a/src/server/Message.ts b/src/server/Message.ts
index 19e0a48aa..aaee143e8 100644
--- a/src/server/Message.ts
+++ b/src/server/Message.ts
@@ -24,6 +24,16 @@ export interface Transferable {
readonly data?: any;
}
+export enum YoutubeQueryTypes {
+ Channels, SearchVideo, VideoDetails
+}
+
+export interface YoutubeQueryInput {
+ readonly type: YoutubeQueryTypes;
+ readonly userInput?: string;
+ readonly videoIds?: string;
+}
+
export interface Reference {
readonly id: string;
}
@@ -45,6 +55,7 @@ export namespace MessageStore {
export const GetRefFields = new Message<string[]>("Get Ref Fields");
export const UpdateField = new Message<Diff>("Update Ref Field");
export const CreateField = new Message<Reference>("Create Ref Field");
+ export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query");
export const DeleteField = new Message<string>("Delete field");
export const DeleteFields = new Message<string[]>("Delete fields");
}
diff --git a/src/server/database.ts b/src/server/database.ts
index 7f5331998..a7254fb0c 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -17,7 +17,7 @@ export class Database {
});
}
- public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) {
+ public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
if (this.db) {
let collection = this.db.collection(collectionName);
const prom = this.currentWrites[id];
@@ -30,7 +30,7 @@ export class Database {
delete this.currentWrites[id];
}
resolve();
- callback();
+ callback(err, res);
});
});
};
@@ -41,6 +41,30 @@ export class Database {
}
}
+ public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
+ if (this.db) {
+ let collection = this.db.collection(collectionName);
+ const prom = this.currentWrites[id];
+ let newProm: Promise<void>;
+ const run = (): Promise<void> => {
+ return new Promise<void>(resolve => {
+ collection.replaceOne({ _id: id }, value, { upsert }
+ , (err, res) => {
+ if (this.currentWrites[id] === newProm) {
+ delete this.currentWrites[id];
+ }
+ resolve();
+ callback(err, res);
+ });
+ });
+ };
+ newProm = prom ? prom.then(run) : run();
+ this.currentWrites[id] = newProm;
+ } else {
+ this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName));
+ }
+ }
+
public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
public delete(id: any, collectionName = Database.DocumentsCollection) {
@@ -126,6 +150,34 @@ export class Database {
}
}
+ public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise<void> {
+ if (this.db) {
+ const visited = new Set<string>();
+ while (ids.length) {
+ const count = Math.min(ids.length, 1000);
+ const index = ids.length - count;
+ const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
+ if (!fetchIds.length) {
+ continue;
+ }
+ const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments"));
+ for (const doc of docs) {
+ const id = doc.id;
+ visited.add(id);
+ ids.push(...fn(doc));
+ }
+ }
+
+ } else {
+ return new Promise(res => {
+ this.onConnect.push(() => {
+ this.visit(ids, fn, collectionName);
+ res();
+ });
+ });
+ }
+ }
+
public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
if (this.db) {
let cursor = this.db.collection(collectionName).find(query);
diff --git a/src/server/index.ts b/src/server/index.ts
index 40c0e7981..1912cf5c1 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -25,8 +25,9 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo
import { DashUserModel } from './authentication/models/user_model';
import { Client } from './Client';
import { Database } from './database';
-import { MessageStore, Transferable, Types, Diff, Message } from "./Message";
+import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message";
import { RouteStore } from './RouteStore';
+import v4 = require('uuid/v4');
const app = express();
const config = require('../../webpack.config');
import { createCanvas, loadImage, Canvas } from "canvas";
@@ -39,12 +40,18 @@ import c = require("crypto");
import { Search } from './Search';
import { debug } from 'util';
import _ = require('lodash');
+import * as Archiver from 'archiver';
+import * as AdmZip from 'adm-zip';
+import * as YoutubeApi from './youtubeApi/youtubeApiSample.js';
import { Response } from 'express-serve-static-core';
+import { DocComponent } from '../client/views/DocComponent';
const MongoStore = require('connect-mongo')(session);
const mongoose = require('mongoose');
const probe = require("probe-image-size");
const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
+let youtubeApiKey: string;
+YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey);
const release = process.env.RELEASE === "true";
if (process.env.RELEASE === "true") {
@@ -177,6 +184,173 @@ function msToTime(duration: number) {
return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
}
+async function getDocs(id: string) {
+ const files = new Set<string>();
+ const docs: { [id: string]: any } = {};
+ const fn = (doc: any): string[] => {
+ const id = doc.id;
+ if (typeof id === "string" && id.endsWith("Proto")) {
+ //Skip protos
+ return [];
+ }
+ const ids: string[] = [];
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc.fields[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+
+ if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+ ids.push(field.fieldId);
+ } else if (field.__type === "script" || field.__type === "computed") {
+ if (field.captures) {
+ ids.push(field.captures.fieldId);
+ }
+ } else if (field.__type === "list") {
+ ids.push(...fn(field));
+ } else if (typeof field === "string") {
+ const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field)) !== null) {
+ ids.push(match[1]);
+ }
+ } else if (field.__type === "RichTextField") {
+ const re = /"href"\s*:\s*"(.*?)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const split = new URL(urlString).pathname.split("doc/");
+ if (split.length > 1) {
+ ids.push(split[split.length - 1]);
+ }
+ }
+ const re2 = /"src"\s*:\s*"(.*?)"/g;
+ while ((match = re2.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const pathname = new URL(urlString).pathname;
+ files.add(pathname);
+ }
+ } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+ const url = new URL(field.url);
+ const pathname = url.pathname;
+ files.add(pathname);
+ }
+ }
+
+ if (doc.id) {
+ docs[doc.id] = doc;
+ }
+ return ids;
+ };
+ await Database.Instance.visit([id], fn);
+ return { id, docs, files };
+}
+app.get("/serializeDoc/:docId", async (req, res) => {
+ const { docs, files } = await getDocs(req.params.docId);
+ res.send({ docs, files: Array.from(files) });
+});
+
+app.get("/downloadId/:docId", async (req, res) => {
+ res.set('Content-disposition', `attachment;`);
+ res.set('Content-Type', "application/zip");
+ const { id, docs, files } = await getDocs(req.params.docId);
+ const docString = JSON.stringify({ id, docs });
+ const zip = Archiver('zip');
+ zip.pipe(res);
+ zip.append(docString, { name: "doc.json" });
+ files.forEach(val => {
+ zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) });
+ });
+ zip.finalize();
+});
+
+app.post("/uploadDoc", (req, res) => {
+ let form = new formidable.IncomingForm();
+ form.keepExtensions = true;
+ // let path = req.body.path;
+ const ids: { [id: string]: string } = {};
+ let remap = true;
+ const getId = (id: string): string => {
+ if (!remap) return id;
+ if (id.endsWith("Proto")) return id;
+ if (id in ids) {
+ return ids[id];
+ } else {
+ return ids[id] = v4();
+ }
+ };
+ const mapFn = (doc: any) => {
+ if (doc.id) {
+ doc.id = getId(doc.id);
+ }
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc.fields[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+
+ if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+ field.fieldId = getId(field.fieldId);
+ } else if (field.__type === "script" || field.__type === "computed") {
+ if (field.captures) {
+ field.captures.fieldId = getId(field.captures.fieldId);
+ }
+ } else if (field.__type === "list") {
+ mapFn(field);
+ } else if (typeof field === "string") {
+ const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+ doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ } else if (field.__type === "RichTextField") {
+ const re = /("href"\s*:\s*")(.*?)"/g;
+ field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ }
+ }
+ };
+ form.parse(req, async (err, fields, files) => {
+ remap = fields.remap !== "false";
+ let id: string = "";
+ try {
+ for (const name in files) {
+ const path = files[name].path;
+ const zip = new AdmZip(path);
+ zip.getEntries().forEach(entry => {
+ if (!entry.name.startsWith("files/")) return;
+ zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false);
+ });
+ const json = zip.getEntry("doc.json");
+ let docs: any;
+ try {
+ let data = JSON.parse(json.getData().toString("utf8"));
+ docs = data.docs;
+ id = data.id;
+ docs = Object.keys(docs).map(key => docs[key]);
+ docs.forEach(mapFn);
+ await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => {
+ err && console.log(err);
+ res();
+ }, true, "newDocuments"))));
+ } catch (e) { console.log(e); }
+ fs.unlink(path, () => { });
+ }
+ if (id) {
+ res.send(JSON.stringify(getId(id)));
+ } else {
+ res.send(JSON.stringify("error"));
+ }
+ } catch (e) { console.log(e); }
+ });
+});
+
app.get("/whosOnline", (req, res) => {
let users: any = { active: {}, inactive: {} };
const now = Date.now();
@@ -437,8 +611,22 @@ app.post(RouteStore.forgot, postForgot);
app.get(RouteStore.reset, getReset);
app.post(RouteStore.reset, postReset);
-app.use(RouteStore.corsProxy, (req, res) =>
- req.pipe(request(decodeURIComponent(req.url.substring(1)))).pipe(res));
+const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/;
+app.use(RouteStore.corsProxy, (req, res) => {
+ req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => {
+ const headers = Object.keys(res.headers);
+ headers.forEach(headerName => {
+ const header = res.headers[headerName];
+ if (Array.isArray(header)) {
+ res.headers[headerName] = header.filter(h => !headerCharRegex.test(h));
+ } else if (header) {
+ if (headerCharRegex.test(header as any)) {
+ delete res.headers[headerName];
+ }
+ }
+ });
+ }).pipe(res);
+});
app.get(RouteStore.delete, (req, res) => {
if (release) {
@@ -493,6 +681,7 @@ server.on("connection", function (socket: Socket) {
}
Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField);
+ Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery);
Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff));
Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id));
Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids));
@@ -547,6 +736,17 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v
Database.Instance.getDocuments(ids, callback, "newDocuments");
}
+function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) {
+ switch (query.type) {
+ case YoutubeQueryType.Channels:
+ YoutubeApi.authorizedGetChannel(youtubeApiKey);
+ break;
+ case YoutubeQueryType.SearchVideo:
+ YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback);
+ case YoutubeQueryType.VideoDetails:
+ YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback);
+ }
+}
const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
"number": "_n",
@@ -646,4 +846,5 @@ function CreateField(newValue: any) {
}
server.listen(serverPort);
-console.log(`listening on port ${serverPort}`); \ No newline at end of file
+console.log(`listening on port ${serverPort}`);
+
diff --git a/src/server/youtubeApi/youtubeApiSample.d.ts b/src/server/youtubeApi/youtubeApiSample.d.ts
new file mode 100644
index 000000000..427f54608
--- /dev/null
+++ b/src/server/youtubeApi/youtubeApiSample.d.ts
@@ -0,0 +1,2 @@
+declare const YoutubeApi: any;
+export = YoutubeApi; \ No newline at end of file
diff --git a/src/server/youtubeApi/youtubeApiSample.js b/src/server/youtubeApi/youtubeApiSample.js
new file mode 100644
index 000000000..50b3c7b38
--- /dev/null
+++ b/src/server/youtubeApi/youtubeApiSample.js
@@ -0,0 +1,179 @@
+const fs = require('fs');
+const readline = require('readline');
+const { google } = require('googleapis');
+const OAuth2 = google.auth.OAuth2;
+
+
+// If modifying these scopes, delete your previously saved credentials
+// at ~/.credentials/youtube-nodejs-quickstart.json
+let SCOPES = ['https://www.googleapis.com/auth/youtube.readonly'];
+let TOKEN_DIR = (process.env.HOME || process.env.HOMEPATH ||
+ process.env.USERPROFILE) + '/.credentials/';
+let TOKEN_PATH = TOKEN_DIR + 'youtube-nodejs-quickstart.json';
+
+module.exports.readApiKey = (callback) => {
+ fs.readFile('client_secret.json', function processClientSecrets(err, content) {
+ if (err) {
+ console.log('Error loading client secret file: ' + err);
+ return;
+ }
+ callback(content);
+ });
+}
+
+module.exports.authorizedGetChannel = (apiKey) => {
+ //this didnt get called
+ // Authorize a client with the loaded credentials, then call the YouTube API.
+ authorize(JSON.parse(apiKey), getChannel);
+}
+
+module.exports.authorizedGetVideos = (apiKey, userInput, callBack) => {
+ authorize(JSON.parse(apiKey), getVideos, { userInput: userInput, callBack: callBack });
+}
+
+module.exports.authorizedGetVideoDetails = (apiKey, videoIds, callBack) => {
+ authorize(JSON.parse(apiKey), getVideoDetails, { videoIds: videoIds, callBack: callBack });
+}
+
+
+/**
+ * Create an OAuth2 client with the given credentials, and then execute the
+ * given callback function.
+ *
+ * @param {Object} credentials The authorization client credentials.
+ * @param {function} callback The callback to call with the authorized client.
+ */
+function authorize(credentials, callback, args = {}) {
+ let clientSecret = credentials.installed.client_secret;
+ let clientId = credentials.installed.client_id;
+ let redirectUrl = credentials.installed.redirect_uris[0];
+ let oauth2Client = new OAuth2(clientId, clientSecret, redirectUrl);
+
+ // Check if we have previously stored a token.
+ fs.readFile(TOKEN_PATH, function (err, token) {
+ if (err) {
+ getNewToken(oauth2Client, callback);
+ } else {
+ oauth2Client.credentials = JSON.parse(token);
+ callback(oauth2Client, args);
+ }
+ });
+}
+
+/**
+ * Get and store new token after prompting for user authorization, and then
+ * execute the given callback with the authorized OAuth2 client.
+ *
+ * @param {google.auth.OAuth2} oauth2Client The OAuth2 client to get token for.
+ * @param {getEventsCallback} callback The callback to call with the authorized
+ * client.
+ */
+function getNewToken(oauth2Client, callback) {
+ var authUrl = oauth2Client.generateAuthUrl({
+ access_type: 'offline',
+ scope: SCOPES
+ });
+ console.log('Authorize this app by visiting this url: ', authUrl);
+ var rl = readline.createInterface({
+ input: process.stdin,
+ output: process.stdout
+ });
+ rl.question('Enter the code from that page here: ', function (code) {
+ rl.close();
+ oauth2Client.getToken(code, function (err, token) {
+ if (err) {
+ console.log('Error while trying to retrieve access token', err);
+ return;
+ }
+ oauth2Client.credentials = token;
+ storeToken(token);
+ callback(oauth2Client);
+ });
+ });
+}
+
+/**
+ * Store token to disk be used in later program executions.
+ *
+ * @param {Object} token The token to store to disk.
+ */
+function storeToken(token) {
+ try {
+ fs.mkdirSync(TOKEN_DIR);
+ } catch (err) {
+ if (err.code != 'EEXIST') {
+ throw err;
+ }
+ }
+ fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => {
+ if (err) throw err;
+ console.log('Token stored to ' + TOKEN_PATH);
+ });
+ console.log('Token stored to ' + TOKEN_PATH);
+}
+
+/**
+ * Lists the names and IDs of up to 10 files.
+ *
+ * @param {google.auth.OAuth2} auth An authorized OAuth2 client.
+ */
+function getChannel(auth) {
+ var service = google.youtube('v3');
+ service.channels.list({
+ auth: auth,
+ part: 'snippet,contentDetails,statistics',
+ forUsername: 'GoogleDevelopers'
+ }, function (err, response) {
+ if (err) {
+ console.log('The API returned an error: ' + err);
+ return;
+ }
+ var channels = response.data.items;
+ if (channels.length == 0) {
+ console.log('No channel found.');
+ } else {
+ console.log('This channel\'s ID is %s. Its title is \'%s\', and ' +
+ 'it has %s views.',
+ channels[0].id,
+ channels[0].snippet.title,
+ channels[0].statistics.viewCount);
+ }
+ });
+}
+
+function getVideos(auth, args) {
+ let service = google.youtube('v3');
+ service.search.list({
+ auth: auth,
+ part: 'id, snippet',
+ type: 'video',
+ q: args.userInput,
+ maxResults: 10
+ }, function (err, response) {
+ if (err) {
+ console.log('The API returned an error: ' + err);
+ return;
+ }
+ let videos = response.data.items;
+ args.callBack(videos);
+ });
+}
+
+function getVideoDetails(auth, args) {
+ if (args.videoIds === undefined) {
+ return;
+ }
+ let service = google.youtube('v3');
+ service.videos.list({
+ auth: auth,
+ part: 'contentDetails, statistics',
+ id: args.videoIds
+ }, function (err, response) {
+ if (err) {
+ console.log('The API returned an error from details: ' + err);
+ return;
+ }
+ let videoDetails = response.data.items;
+ args.callBack(videoDetails);
+ });
+} \ No newline at end of file
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-10 03:28:27 +0000