From 79e1323acd0d0f95d08a09cefce908e35d0e7558 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 8 Feb 2020 17:32:33 -0500 Subject: initial commit, pending png read error bug fix --- src/client/documents/Documents.ts | 45 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/scraping/buxton/final/BuxtonImporter.ts | 316 ++++++++++++++ src/scraping/buxton/json/buxton.json | 116 ----- src/scraping/buxton/json/incomplete.json | 468 --------------------- src/scraping/buxton/node_scraper.ts | 256 ----------- src/server/ApiManagers/UtilManager.ts | 38 +- .../authentication/models/current_user_utils.ts | 2 +- 8 files changed, 373 insertions(+), 870 deletions(-) create mode 100644 src/scraping/buxton/final/BuxtonImporter.ts delete mode 100644 src/scraping/buxton/json/buxton.json delete mode 100644 src/scraping/buxton/json/incomplete.json delete mode 100644 src/scraping/buxton/node_scraper.ts (limited to 'src') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index d647b34e6..f05bb3736 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -54,6 +54,7 @@ import { InkField } from "../../new_fields/InkField"; import { InkingControl } from "../views/InkingControl"; import { RichTextField } from "../../new_fields/RichTextField"; import { Networking } from "../Network"; +import { extname } from "path"; const requestImageSize = require('../util/request-image-size'); const path = require('path'); @@ -341,8 +342,34 @@ export namespace Docs { */ export namespace Create { - export async function Buxton() { - console.log(await Networking.FetchFromServer("/newBuxton")); + export function Buxton() { + const loading = new Doc; + loading.title = "Please wait for the import script..."; + const parent = TreeDocument([loading], { + title: "The Buxton Collection", + _width: 400, + _height: 400 + }); + Networking.FetchFromServer("/buxton").then(response => { + parent.data = new List(); + const devices = JSON.parse(response); + if (!Array.isArray(devices)) { + alert("Improper Buxton import formatting!"); + return; + } + devices.forEach(device => { + const { __images } = device; + delete device.__images; + const { ImageDocument, StackingDocument } = Docs.Create; + if (Array.isArray(__images)) { + const deviceImages = __images.map((url, i) => ImageDocument(url, { title: `image${i}.${extname(url)}` })); + const doc = StackingDocument(deviceImages, { title: device.title }); + Docs.Get.DocumentHierarchyFromJson(device, undefined, doc); + Doc.AddDocToList(parent, "data", doc); + } + }); + }); + return parent; } Scripting.addGlobal(Buxton); @@ -628,7 +655,7 @@ export namespace Docs { * or the result of any JSON.parse() call. * @param title an optional title to give to the highest parent document in the hierarchy */ - export function DocumentHierarchyFromJson(input: any, title?: string): Opt { + export function DocumentHierarchyFromJson(input: any, title?: string, appendToTarget?: Doc): Opt { if (input === undefined || input === null || ![...primitives, "object"].includes(typeof input)) { return undefined; } @@ -638,7 +665,7 @@ export namespace Docs { } let converted: Doc; if (typeof parsed === "object" && !(parsed instanceof Array)) { - converted = convertObject(parsed, title); + converted = convertObject(parsed, title, appendToTarget); } else { (converted = new Doc).json = toField(parsed); } @@ -653,12 +680,12 @@ export namespace Docs { * @returns the object mapped from JSON to field values, where each mapping * might involve arbitrary recursion (since toField might itself call convertObject) */ - const convertObject = (object: any, title?: string): Doc => { - const target = new Doc(); + const convertObject = (object: any, title?: string, target?: Doc): Doc => { + const resolved = target ?? new Doc; let result: Opt; - Object.keys(object).map(key => (result = toField(object[key], key)) && (target[key] = result)); - title && !target.title && (target.title = title); - return target; + Object.keys(object).map(key => (result = toField(object[key], key)) && (resolved[key] = result)); + title && !resolved.title && (resolved.title = title); + return resolved; }; /** diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 2518a4a55..66e4ef1b0 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -807,7 +807,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { doFreeformLayout(poolData: Map) { const layoutDocs = this.childLayoutPairs.map(pair => pair.layout); const initResult = this.Document.arrangeInit && this.Document.arrangeInit.script.run({ docs: layoutDocs, collection: this.Document }, console.log); - let state = initResult && initResult.success ? initResult.result.scriptState : undefined; + const state = initResult && initResult.success ? initResult.result.scriptState : undefined; const elements = initResult && initResult.success ? this.viewDefsToJSX(initResult.result.views) : []; this.childLayoutPairs.filter(pair => this.isCurrent(pair.layout)).map((pair, i) => { diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts new file mode 100644 index 000000000..804d65d74 --- /dev/null +++ b/src/scraping/buxton/final/BuxtonImporter.ts @@ -0,0 +1,316 @@ +import { readdirSync, writeFile, mkdirSync, createWriteStream, createReadStream, unlinkSync } from "fs"; +import * as path from "path"; +import { red, cyan, yellow, green } from "colors"; +import { Utils } from "../../../Utils"; +import rimraf = require("rimraf"); +const StreamZip = require('node-stream-zip'); +import * as sharp from 'sharp'; +import { SizeSuffix, DashUploadUtils, InjectSize } from "../../../server/DashUploadUtils"; +import { AcceptibleMedia } from "../../../server/SharedMediaTypes"; + +export interface DeviceDocument { + title: string; + shortDescription: string; + longDescription: string; + company: string; + year: number; + originalPrice: number; + degreesOfFreedom: number; + dimensions: string; + primaryKey: string; + secondaryKey: string; +} + +interface DocumentContents { + body: string; + images: string[]; +} + +interface AnalysisResult { + device?: DeviceDocument; + errors?: any; +} + +type Converter = (raw: string) => { transformed?: T, error?: string }; + +interface Processor { + exp: RegExp; + matchIndex?: number; + transformer?: Converter; +} + +namespace Utilities { + + export function numberValue(raw: string) { + const transformed = Number(raw); + if (isNaN(transformed)) { + return { error: `${transformed} cannot be parsed to a numeric value.` }; + } + return { transformed }; + } + + export function collectUniqueTokens(raw: string) { + return { transformed: Array.from(new Set(raw.replace(/,|\s+and\s+/g, " ").split(/\s+/).map(token => token.toLowerCase().trim()))).map(capitalize).sort() }; + } + + export function correctSentences(raw: string) { + raw = raw.replace(/\./g, ". ").replace(/\:/g, ": ").replace(/\,/g, ", ").replace(/\?/g, "? ").trimRight(); + raw = raw.replace(/\s{2,}/g, " "); + return { transformed: raw }; + } + + export function tryGetValidCapture(matches: RegExpExecArray | null, matchIndex: number): string | undefined { + let captured: string; + if (!matches || !(captured = matches[matchIndex])) { + return undefined; + } + const lower = captured.toLowerCase(); + if (/to come/.test(lower)) { + return undefined; + } + if (lower.includes("xxx")) { + return undefined; + } + if (!captured.toLowerCase().replace(/[….\s]+/g, "").length) { + return undefined; + } + return captured; + } + + export function capitalize(word: string): string { + const clean = word.trim(); + if (!clean.length) { + return word; + } + return word.charAt(0).toUpperCase() + word.slice(1); + } + +} + +const RegexMap = new Map>([ + ["title", { + exp: /contact\s+(.*)Short Description:/ + }], + ["company", { + exp: /Company:\s+([^\|]*)\s+\|/, + transformer: (raw: string) => ({ transformed: raw.replace(/\./g, "") }) + }], + ["year", { + exp: /Year:\s+([^\|]*)\s+\|/, + transformer: Utilities.numberValue + }], + ["primaryKey", { + exp: /Primary:\s+(.*)(Secondary|Additional):/, + transformer: Utilities.collectUniqueTokens + }], + ["secondaryKey", { + exp: /(Secondary|Additional):\s+([^\{\}]*)Links/, + transformer: Utilities.collectUniqueTokens, + matchIndex: 2 + }], + ["originalPrice", { + exp: /Original Price \(USD\)\:\s+\$([0-9\.]+)/, + transformer: Utilities.numberValue + }], + ["degreesOfFreedom", { + exp: /Degrees of Freedom:\s+([0-9]+)/, + transformer: Utilities.numberValue + }], + ["dimensions", { + exp: /Dimensions\s+\(L x W x H\):\s+([0-9\.]+\s+x\s+[0-9\.]+\s+x\s+[0-9\.]+\s\([A-Za-z]+\))/, + transformer: (raw: string) => { + const [length, width, group] = raw.split(" x "); + const [height, unit] = group.split(" "); + return { + transformed: { + dim_length: Number(length), + dim_width: Number(width), + dim_height: Number(height), + dim_unit: unit.replace(/[\(\)]+/g, "") + } + }; + } + }], + ["shortDescription", { + exp: /Short Description:\s+(.*)Bill Buxton[’']s Notes/, + transformer: Utilities.correctSentences + }], + ["longDescription", { + exp: /Bill Buxton[’']s Notes(.*)Device Details/, + transformer: Utilities.correctSentences + }], +]); + +const outDir = path.resolve(__dirname, "json"); +const imageDir = path.resolve(__dirname, "../../../server/public/files/images/buxton"); +const successOut = "buxton.json"; +const failOut = "incomplete.json"; +const deviceKeys = Array.from(RegexMap.keys()); + +export default async function executeImport() { + [outDir, imageDir].forEach(dir => { + rimraf.sync(dir); + mkdirSync(dir); + }); + return parseFiles(); +} + +async function parseFiles(): Promise { + const sourceDirectory = path.resolve(`${__dirname}/source`); + + const candidates = readdirSync(sourceDirectory).filter(file => file.endsWith(".doc") || file.endsWith(".docx")).map(file => `${sourceDirectory}/${file}`); + const imported = await Promise.all(candidates.map(async path => ({ path, body: await extractFileContents(path) }))); + const data = imported.map(({ path, body }) => analyze(path, body)); + + const masterDevices: DeviceDocument[] = []; + const masterErrors: any[] = []; + + data.forEach(({ device, errors }) => { + if (device) { + masterDevices.push(device); + } else { + masterErrors.push(errors); + } + }); + const total = candidates.length; + if (masterDevices.length + masterErrors.length !== total) { + throw new Error(`Encountered a ${masterDevices.length} to ${masterErrors.length} mismatch in device / error split!`); + } + + console.log(); + await writeOutputFile(successOut, masterDevices, total, true); + await writeOutputFile(failOut, masterErrors, total, false); + console.log(); + + return masterDevices; +} + +async function extractFileContents(pathToDocument: string): Promise<{ body: string, images: string[] }> { + const zip = new StreamZip({ file: pathToDocument, storeEntries: true }); + const contents = await new Promise((resolve, reject) => { + zip.on('ready', () => { + let body = ""; + zip.stream("word/document.xml", (error: any, stream: any) => { + if (error) { + reject(error); + } + stream.on('data', (chunk: any) => body += chunk.toString()); + stream.on('end', () => resolve(body)); + }); + }); + }); + const images = (await writeImages(zip)).map(name => `http://localhost:1050/files/images/buxton/${name}`); + zip.close(); + let body = ""; + const components = contents.toString().split(''); + const content = tags[1].replace(/<.*$/, ""); + body += content; + } + return { body, images }; +} + +async function writeImages(zip: any): Promise { + const entryNames = Object.values(zip.entries()).map(({ name }) => name); + const resolved: { mediaPath: string, ext: string }[] = []; + let initialWritePath: string; + entryNames.forEach(name => { + const matches = /^word\/media\/\w+\.(jpeg|jpg|png|gif)/.exec(name); + matches && resolved.push({ mediaPath: name, ext: matches[1] }); + }); + return Promise.all(resolved.map(async ({ mediaPath, ext }) => { + const outName = `upload_${Utils.GenerateGuid()}.${ext}`; + const initialWrite = await new Promise((resolve, reject) => { + zip.stream(mediaPath, (error: any, stream: any) => { + if (error) { + console.error(error); + return reject(error); + } + initialWritePath = `${imageDir}/${outName}`; + const writeStream = createWriteStream(initialWritePath); + stream.on('end', () => resolve(outName)); + stream.on('error', reject); + stream.pipe(writeStream); + }); + }); + const resizers = [ + { resizer: sharp().rotate(), suffix: SizeSuffix.Original }, + ...Object.values(DashUploadUtils.Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + const { pngs, jpgs } = AcceptibleMedia; + if (pngs.includes(ext)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(ext)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } + for (const { resizer, suffix } of resizers) { + await new Promise(resolve => { + const filename = InjectSize(outName, suffix); + console.log(filename); + createReadStream(initialWritePath).pipe(resizer).pipe(createWriteStream(`${imageDir}/${filename}`)) + .on('close', resolve) + .on('error', error => { + console.log(red(error)); + resolve(); + }); + }); + } + unlinkSync(initialWritePath); + return initialWrite; + })); +} + +function analyze(pathToDocument: string, { body, images }: DocumentContents): AnalysisResult { + const filename = path.basename(pathToDocument).replace("Bill_Notes_", ""); + console.log(`Parsing ${filename}...`); + + const device: any = {}; + const errors: any = { filename }; + + for (const key of deviceKeys) { + const { exp, transformer, matchIndex } = RegexMap.get(key)!; + const matches = exp.exec(body); + + let captured = Utilities.tryGetValidCapture(matches, matchIndex ?? 1); + if (!captured) { + errors[key] = `ERR__${key.toUpperCase()}__: outer match ${matches === null ? "wasn't" : "was"} captured.`; + continue; + } + + captured = captured.replace(/\s{2,}/g, " "); + if (transformer) { + const { error, transformed } = transformer(captured); + if (error) { + errors[key] = `__ERR__${key.toUpperCase()}__TRANSFORM__: ${error}`; + continue; + } + captured = transformed; + } + + device[key] = captured; + } + + const errorKeys = Object.keys(errors); + if (errorKeys.length > 1) { + console.log(red(`\n@ ${cyan(filename.toUpperCase())}...`)); + errorKeys.forEach(key => key !== "filename" && console.log(red(errors[key]))); + return { errors }; + } + + device.__images = images; + + return { device }; +} + +async function writeOutputFile(relativePath: string, data: any[], total: number, success: boolean) { + console.log(yellow(`Encountered ${data.length} ${success ? "valid" : "invalid"} documents out of ${total} candidates. Writing ${relativePath}...`)); + return new Promise((resolve, reject) => { + const destination = path.resolve(outDir, relativePath); + const contents = JSON.stringify(data, undefined, 4); + writeFile(destination, contents, err => err ? reject(err) : resolve()); + }); +} \ No newline at end of file diff --git a/src/scraping/buxton/json/buxton.json b/src/scraping/buxton/json/buxton.json deleted file mode 100644 index 8371f2cf2..000000000 --- a/src/scraping/buxton/json/buxton.json +++ /dev/null @@ -1,116 +0,0 @@ -[ - { - "title": "3Dconnexion CadMan 3D Motion Controller", - "company": "3Dconnexion", - "year": 2003, - "primaryKey": [ - "Joystick" - ], - "secondaryKey": [ - "Isometric", - "Joystick" - ], - "originalPrice": 399, - "degreesOfFreedom": 6, - "dimensions": { - "length": 175, - "width": 122, - "height": 43, - "unit": "mm" - }, - "shortDescription": "The CadMan is a 6 degree of freedom (DOF) joystick controller. It represented a significant step towards making this class of is controller affordable. It was mainly directed at 3D modelling and animation and was a “next generation” of the Magellan controller, which is also in the collection.", - "longDescription": "The CadMan is a 6 degree of freedom (DOF) joystick controller. It represented a significant step towards making this class of is controller more affordable. It was mainly directed at 3D modelling and animation and was a “next generation” of the Magellan/SpaceMouse controller, which is also in the collection. Like the Magellan, this is an isometric rate-control joystick. That is, it rests in a neutral central position, not sending and signal. When a force is applied to it, it emits a signal indicating the direction and strength of that force. This signal can then be mapped to a parameter of a selected object, such as a sphere, and – for example – cause that sphere to rotate for as long as, and as fast as, and in the direction determined by, the duration, force, and direction of the applied force. When released, it springs back to neutral position. Note that the force does not need to be directed along a single DOF. In fact, a core feature of the device is that one can simultaneously and independently apply force that asserts control over more than one DOF, and furthermore, vary those forces dynamically. As an aid to understanding, let me walk through some of the underlying concepts at play here by using a more familiar device: a computer mouse. If you move a mouse in a forward/backward direction, the mouse pointer on the screen moves between the screen’s top and bottom. If you think of the screen as a piece of graph paper, that corresponds to moving along the “Y” axis. That is one degree of freedom. On the other hand, you could move the mouse left and right, which causes the mouse to move between the left and right side of the screen. That would correspond to moving along the graph paper’s “X” axis – a second degree of freedom. Yet, you can also move the mouse diagonally. This is an example of independently controlling two degrees of freedom. Now imagine that if you lifted your mouse off your desktop, that your computer could dynamically sense its height as you did so. This would constitute a “flying mouse” (the literal translation of the German word for a “Bat”, which Canadian colleague, Colin Ware, applied to just such a mouse which he built in 1988). If you moved your Bat vertically up and down, perpendicular to the desktop, you would be controlling movement along the “Z” axis - a third degree of freedom. Having already seen that we can move a mouse diagonally, we have established that we need not be constrained to only moving along a single axis. That extends to the movement of our Bat and movement along the “Z” axis. We can control our hand movement in dependently in any or all directions in 3D space. But how does one reconcile the fact that we call the CadMan a “3D controller, and yet also describe it as having 6 degrees of freedom? After all, the example this far demonstrates that our Bat, as described thus far, has freedom on movement in 3 Dimensions. While true, we can extend our example to prove that that freedom to move in 3D is also highly constrained. To demonstrate this, move your hand in 3D space on and above your desktop. However, do so keeping your palm flat, parallel to the desktop with your fingers pointing directly forward. In so doing, you are still moving in 3D. Now, while moving, twist your wrist, while moving the hand, such that your palm is alternatively exposed to the left and right side. This constitutes rotation around the “Y” axis. A fourth DOF. Now add a waving motion to your hand, as if it were a paper airplane diving up and down, while also rocking left and right. But keep your fingers pointing forward. You have now added a fifth DOF, rotation around the “X” axis. Finally, add a twist to your wrist so that your fingers are no longer constrained to pointing forward. This is the sixth degree of freedom, rotation around the “Z” axis. Now don’t be fooled, this exercise could continue. We are not restricted to even six DOF. Imagine doing the above, but where the movement and rotations are measured relative to the Bat’s position and orientation, rather than to the holding/controlling hand, per se. One could imagine the Bat having a scroll wheel, like the one on most mice today. Furthermore, while flying your Bat around in 3D, that wheel could easily be rolled in either forward or backward, and thereby control the size of whatever was being controlled. Hence, with one hand we could assert simultaneous and independent control over 7 DOF in 3D space. This exercise has two intended take-aways. The first is a better working understanding between the notion of Degree of Freedom (DOF) and Dimension in space. Hopefully, the confusion frequently encountered when 3D and 6DOF are used in close context, can now be eliminated. Second, is that, with appropriate sensing, the human hand is capable of exercising control over far more degrees of freedom that six. And if we use the two hands together, the potential number of DOF that one can control goes even further. Finally, it is important to add one more take-away – one which both emerges from, and is frequently encountered when discussing, the previous two. That is, do not equate exercising simultaneous control over a high number of DOF with consciously doing the same number of different things all at once. The example that used to be thrown at me when I started talking about coordinated simultaneously bi-manual action went along the lines of, “Psychology tells us that we cannot do multiple things at once, for example, simultaneously tapping your head and rubbing your stomach. ”Well, first, I can tap my head with one hand while rubbing my stomach with the other. But that is not the point. The whole essence of skill – motor-sensory and cognitive – is “chunking” or task integration. When one appears to be doing many different things at once, if they are skilled, they are consciously doing only one thing. Playing a chord on the piano, for example, or skiing down the hill. Likewise, in flying your imaginary BAT in the previous exercise with the scroll wheel, were you doing 7 things at once, or one thing with 7 DOF? And if you had a Bat in each hand, does that mean you are now doing 14 things at once, or are you doing one thing with 14 DOF? Let me provide a different way of answering this question: if you have ever played air guitar, or “conducted” the orchestra that you are listening to on the radio, you are exercising control over more than 14 DOF. And you are doing exactly what I just said, “playing air guitar” or “conducting an orchestra”. One thing – at the conscious level, which is what matters – despite almost any one thing being able to be deconstructed into hundreds of sub-tasks. As I said the essence of skill: aggregation, or chunking. What is most important for both tool designers and users to be mindful of, is the overwhelming influence that our choice and design of tools impacts the degree to which such integration or chunking can take place. The degree to which the tool matches both the skills that we have already acquired through a lifetime of living in the everyday world, and the demands of the intended task, the more seamless that task can be performed, the more “natural” it will feel, and the less learning will be required. In my experience, it brought particular value when used bimanually, in combination with a mouse, where the preferred hand performed conventional pointing, selection and dragging tasks, while the non-preferred hand could manipulate the parameters of the thing being selected. First variation of the since the 2001 formation of 3Dconnextion. The CadMan came in 5 colours: smoke, orange, red, blue and green. See the notes for the LogiCad3D Magellan for more details on this class of device. It is the “parent” of the CadMan, and despite the change in company name, it comes from the same team." - }, - { - "title": "Adesso ACK-540UB USB Mini-Touch Keyboard with Touchpad", - "company": "Adesso", - "year": 2005, - "primaryKey": [ - "Keyboard" - ], - "secondaryKey": [ - "Pad", - "Touch" - ], - "originalPrice": 59.95, - "degreesOfFreedom": 2, - "dimensions": { - "length": 287, - "width": 140, - "height": 35.5, - "unit": "mm" - }, - "shortDescription": "The Mini-Touch Keyboard is a surprisingly rare device: a laptop-style, small-footprint keyboard with a centrally mounted touch-pad. .", - "longDescription": "First released in 2003 with a PS/2 connector (ACK-540PW & ACK-540PB). USB version released in 2006 in either black (ACK-540UB) or white (ACK-540UW). Marketed under different brands, including SolidTek: http: //www. tigerdirect. com/applications/searchtools/item-details. asp? EdpNo=1472243https: //acecaddigital. com/index. php/products/keyboards/mini-keyboards/kb-540 Deltaco: https: //www. digitalimpuls. no/logitech/116652/deltaco-minitastatur-med-touchpad-usb" - }, - { - "title": "Contour Design UniTrap ", - "company": "Contour Design", - "year": 1999, - "primaryKey": [ - "Re-skin" - ], - "secondaryKey": [ - "Mouse" - ], - "originalPrice": 14.99, - "degreesOfFreedom": 2, - "dimensions": { - "length": 130.5, - "width": 75.7, - "height": 43, - "unit": "mm" - }, - "shortDescription": "This is a plastic shell within which the round Apple iMac G3 “Hockey Puck” mouse can be fit. While the G3 Mouse worked well mechanically, when gripped its round shape gave few cues as to its orientation. Hence, if you moved your hand up, the screen pointer may well have moved diagonally. By reskinning it with the inexpensive Contour UniTrap, the problem went away without the need to buy a whole new mouse.", - "longDescription": "Also add back pointers from devices re-skinned" - }, - { - "title": "Depraz Swiss Mouse", - "company": "Depraz", - "year": 1980, - "primaryKey": [ - "Mouse" - ], - "secondaryKey": [ - "Ball", - "Chord", - "Keyboard", - "Mouse" - ], - "originalPrice": 295, - "degreesOfFreedom": 2, - "dimensions": { - "length": 50.8, - "width": 76.2, - "height": 114.3, - "unit": "mm" - }, - "shortDescription": "This mouse is one of the first commercially available mice to be sold publicly. It is known as the Swiss mouse, and yes, the roller mechanism was designed by a Swiss watchmaker. Coincidentally, the company that made it, Depraz, is based in Apples, Switzerland. Their success in selling this mouse is what caused Logitech to switch from a software development shop to one of the world’s leading suppliers of mice and other input devices.", - "longDescription": "DePraz began manufacturing in 1980, but following design built in 1979. Logitech started selling it in 1982. It was one of the first mass produced mice, one of the first available ball mice, as well as to have an optical shaft encoder – thereby improving linearity. An interesting fact, given its Swiss heritage, is that its designer, André Guignard, was trained as a Swiss watch maker. Unlike most modern mice, the DePraz, or “Swiss” mouse had a quasi-hemispherical shape. Hence, it was held in a so-called “power-grip”, much as one would grip a horizontally held ball – the thumb and small finger applying pressure on each side, with added support from the weight/friction of the palm on the back of the mouse. In this posture, the three middle fingers naturally positioning themselves over the three buttons mounted at the lower edge of the front. Largely freed of grip pressure, by grace of thumb and little finger, the middle fingers had essentially freedom of motion to independently operate the buttons. Each having a dedicated finger, the buttons could be easily pushed independently or in any combination. Like the three valves on a trumpet, this ability to “chord” extended the three physical buttons to have the power of seven. The down-side of this “turtle shell” form factor is that it placed the hand in a posture in which mouse movement relied more of the larger muscle groups of the arm to wrist, rather than wrist to fingers – the latter being the approach taken in most subsequent mice. The original Swiss Mouse was developed at École Polytechnique Fédérale de Lausanne by a project led by Jean-Daniel Nicoud, who was also responsible for the development of its optical shaft encoder. To augment their revenue stream, Logitech, then a software and hardware consulting company for the publishing industry, acquired marketing rights for North America. Mouse revenue quickly overshadowed that from software. In 1983, Logitech acquired DePraz, named the Swiss Mouse the “P4”, and grew to become one of the largest input device manufacturer in the world. One curious coincidence is that they were founded in the town of Apples, Switzerland." - }, - { - "title": "One Laptop Per Child (OLPC) XO-1", - "company": "One Laptop Per Child (OLPC)", - "year": 2007, - "primaryKey": [ - "Computer" - ], - "secondaryKey": [ - "Keyboard", - "Laptop", - "Pad", - "Slate", - "Touch" - ], - "originalPrice": 199, - "degreesOfFreedom": 2, - "dimensions": { - "length": 242, - "width": 228, - "height": 30, - "unit": "mm" - }, - "shortDescription": "The OLPC XO-1 is very innovative device that nevertheless raises serious issues about technology and social responsibility. It is included in the collection primarily as a warning against technological hubris, and the fact that no technologies are neutral from a social-cultural perspective.", - "longDescription": "IntroductionI have this computer in my collection as a reminder of the delicate relationship between object and purpose, and how no matter how well one does on the former, it will likely have no impact on making a wanting concept achieve the stated (and even valid) purpose any better. I include it in the collection as a cautionary tale of how the object may help sell a concept, regardless how ill-conceived – even to those who should know better, had they applied the most basic critical thinking. For consumers, investors and designers, its story serves as a cautionary reminder to the importance of cultivating and retaining a critical mind and questioning perspective, regardless of how intrinsically seductive or well-intentioned a technology may be. From the perspective of hardware and software, what the One Laptop Per Child (OLPC) project was able to accomplish is impressive. In general, the team delivered a computer that could be produced at a remarkably low price – even if about double that which was targeted. Specifically, the display, for example, is innovative, and stands out due to its ability to work both in the bright sun (reflective) as well as in poorly lit spaces (emissive) – something that goes beyond pretty much anything else that is available on today’s (2017) slate computers or e-readers. In short, some excellent work went into this machine, something that is even more impressive, given the nature of the organization from which it emerged. The industrial design was equally impressive. Undertaken by Yves Behar’s FuseprojectUltimately, however, the machine was a means to an end, not the end itself. Rather than a device, the actual mission of the OLPC project was: … to empower the world's poorest children through education. Yet, as described by in their materials, the computer was intended to play a key role in this: With access to this type of tool [the computer], children are engaged in their own education, and learn, share, and create together. They become connected to each other, to the world and to a brighter future. Hence, making a suitable computer suitable to that purpose and the conditions where it would be used, at a price point that would enable broad distribution, was a key part of the project. The Underlying Belief System of the OLPC ProjectSince they are key to the thinking behind the OLPC project, I believe if fair to frame my discussion around the following four questions: Will giving computers to kids in the developing world improve their education? Will having a thus better-educated youth help bring a society out of poverty? Can that educational improvement be accomplished by giving the computers to the kids, with no special training for teachers? Should this be attempted on a global scale without any advance field trials or pilot studies? From the perspective of the OLPC project, the answer to every one of these questions is an unequivocal “yes”. In fact, as we shall see, any suggestion to the contrary is typically answered by condescension and/or mockery. The answers appear to be viewed as self-evident and not worth even questioning. Those who have not subscribed to this doctrine might call such a viewpoint hubris. What staggers me is how the project got so far without the basic assumptions being more broadly questioned, much less such questions being seriously addressed by the proponents. How did seemingly otherwise people commit to the project, through their labour or financial investment, given the apparently naïve and utopian approach that it took? Does the desire to do good cloud judgment that much? Are we that dazzled by a cool technology or big hairy audacious goal? Or by a charismatic personality? To explain my concern, and what this artifact represents to me, let me just touch on the four assumptions on which the project was founded. Will giving computers to kids in the developing world improve education? The literature on this question is, at best, mixed. What is clear is that one cannot make any assumption that such improvements will occur, regardless of whether one is talking about the developing world or suburban USA. For example, in January 2011, The World Bank published the following study: Can Computers Help Students Learn? From Evidence to Policy, January 2011, Number 4, The World Bank. A public-private partnership in Colombia, called Computers for Education, was created in 2002 to increase the availability of computers in public schools for use in education. Since starting, the program has installed more than 73, 000 computers in over 6, 300 public schools in more than 1, 000 municipalities. By 2008, over 2 million students and 83, 000 teachers had taken part. This document reports on a two-year study to determine the impact of the program on student performance. Students in schools that received the computers and teacher training did not do measurably better on tests than students in the control group. Nor was there a positive effect on other measures of learning. Researchers did not find any difference in test scores when they looked at specific components of math and language studies, such as algebra and geometry, and grammar and paraphrase ability in Spanish. But report also notes that results of such studies are mixed: Studies on the relationship between using computers in the classroom and improved test scores in developing countries give mixed results: A review of Israel’s Tomorrow-98 program in the mid-1990s, which put computers in schools across the country, did not find any impact on math and Hebrew language scores. But in India, a study of a computer-assisted learning program showed a significant positive impact on math scores. One thing researchers agree on, more work is needed in this field. Before moving on, a search of the literature will show that these results are consistent with those that were available in the literature at the time that the project was started. The point that I am making is not that the OLPC project could not be made to work; rather, that it was wrong to assume that it would do so without spending at least as much time designing the process to bring that about, as was expended designing the computer itself. Risk is fine, and something that can be mitigated. But diving in under the assumption that it would just work is not calculated risk, it is gambling - with other people’s lives, education and money. Will a better educated population help bring a society out of poverty? I am largely going to punt on this question. The fact is, I would be hard pressed to argue against education. But let us grant that improving education in the developing world is a good thing. The appropriate question is: is the approach of the OLPC project a reasonable or responsible way to disburse the limited resources that are available to address the educational challenges of the developing world? At the very least, I would suggest that this is a topic worthy of debate. An a priori assumption that giving computers is the right solution is akin to the, “If you build it they will come” approach seen in the movie, Field of Dreams. The problem here is that this is not a movie. There are real lives and futures that are at stake here – lives of those who cannot afford to see the movie, much less have precious resources spent on projects that are not well thought through. Can that improvement be accomplished by just giving the computers to the kids without training teachers? Remarkably, the OLPC Project’s answer is an explicit, “Yes”. In a TED talk filmed in December 2007, the founder of the OLPC initiative, Nicholas Negroponte states: “When people tell me, you know, who’s going to teach the teachers to teach the kids, I say to myself, “What planet do you come from? ” Okay, there’s not a person in this room [the TED Conference], I don’t care how techy you are, there’s not a person in this room that doesn’t give their laptop or cell phone to a kid to help them debug it. Okay, we all need help, even those of us who are very seasoned. ”Let us leave aside the naïvete of this statement stemming from the lack of distinction between ability to use applications and devices versus the ability to create and shape them. A failure of logic remains in that those unseasoned kids are part of “us”, as in “we all need help”. Where do the kids go for help? To other kids? What if they don’t know? Often they won’t. After all, the question may well have to do with a concept in calculus, rather than how to use the computer. What then? No answer is offered. Rather, those who dare raise the serious and legitimate concerns regarding teacher preparation are mockingly dismissed as coming from another planet! Well, perhaps they are. But in that case, there should at least be some debate as to who lives on which planet. Is it the people raising the question or the one dismissing the concern that lives in the real world of responsible thought and action? Can this all be accomplished without any advance field trials? Should one just immediately commit to international deployment of the program? As recently as September 2009, Negroponte took part in a panel discussion where he spoke on this matter. He states: I'd like you to imagine that I told you \"I have a technology that is going to change the quality of life. \" And then I tell you \"Really the right thing to do is to set up a pilot project to test my technology. And then the second thing to do is, once the pilot has been running for some period of time, is to go and measure very carefully the benefits of that technology. \"And then I am to tell you that what we are going to is very scientifically evaluate this technology, with control groups - giving it to some, giving it to others. And this all is very reasonable until I tell you the technology is electricity. And you say \"Wait, you don't have to do that!\"But you don't have to do that with laptops and learning either. And the fact that somebody in the room would say the impact is unclear is to me amazing - unbelievably amazing. There's not a person in this room who hasn't bought a laptop for their child, if they could afford it. And you don't know somebody who hasn't done it, if they can afford it. So there's only one question on the table and that's, “How to afford it? ” That's the only question. There is no other question - it's just the economics. And so, when One Laptop Per Child started, I didn't have the picture quite as clear as that, but we did focus on trying to get the price down. We did focus on those things. Unfortunately, Negroponte demonstrates his lack of understanding of both the history of electricity and education in this example. His historical mistake is this: yes, it was pretty obvious that electricity could bring many benefits to society. But what happened when Edison did exactly what Negroponte advocates? He almost lost his company due to his complete (but mistaken) conviction that DC, rather the AC was the correct technology to pursue. As with electricity, yes, it is rather obvious that education could bring significant benefits to the developing world. But in order to avoid making the same kind of expensive mistake that Edison did, perhaps one might want to do one’s best to make sure that the chosen technology is the AC, rather than DC, of education. A little more research, and a little less hubris might have put the investments in Edison and the OLPC to much better use. But the larger question is this: in what way is it responsible for the wealthy western world to advocate an untested and expensive (in every sense) technological solution on the poorest nations in the world? If history has taught us anything, it has taught us that just because our intentions are good, the same is not necessarily true for consequences of our actions. Later in his presentation, Negroponte states: … our problems are swimming against very naïve views of education. With this, I have to agree. It is just whose views on education are naïve, and how can such views emerge from MIT, no less, much less pass with so little critical scrutiny by the public, the press, participants, and funders? In an interview with Paul Marks, published in the New Scientist in December 2008, we see the how the techno-centric aspect of the project plays into the ostensible human centric purpose of the project. Negroponte’s retort regarding some of the initial skepticism that the project provoked was this: “When we first said we could build a laptop for $100 it was viewed as unrealistic and so 'anti-market' and so 'anti' the current laptops which at the time were around $1000 each, \" Negroponte said. \"It was viewed as pure bravado - but look what happened: the netbook market has developed in our wake. \" The project's demands for cheaper components such as keyboards, and processors nudged the industry into finding ways to cut costs, he says. \"What started off as a revolution became a culture. \"Surprise, yes, computers get smaller, faster, and cheaper over the course of time, and yes, one can even grant that the OLPC project may have accelerated that inevitable move. And, I have already stated my admiration and respect for the quality of the technology that was developed. But in the context of the overall objectives of the project, the best that one can say is, “Congratulations on meeting a milestone. ” However, by the same token, one might also legitimately question if starting with the hardware was not an instance of putting the cart before the horse. Yes, it is obviously necessary to have portable computers in the first place, before one can introduce them into the classroom, home, and donate them to children in the developing world. But it is also the case that small portable computers were already in existence and at the time that the project was initiated. While a factor of ten more expensive than the eventual target price, they were both available and adequate to support limited preliminary testing of the underlying premises of the project in an affordable manner. That is, before launching into a major - albeit well-intentioned – hardware development project, it may have been prudent to have tested the underlying premises of its motivation. Here we have to return to the raison d’être of the initiative: … to empower the world's poorest children through educationHence, the extent to which this is achieved from a given investment must be the primary metric of success, as well as the driving force of the project. Yet, that is clearly not what happened. Driven by a blind Edisonian belief in their un-tested premise, the project’s investments were overwhelmingly on the side of technology rather than pedagogy. Perhaps the nature and extent of the naïve (but well-meaning) utopian dream underlying the project is captured in the last part of the interview, above: Negroponte believes that empowering children and their parents with the educational resources offered by computers and the Internet will lead to informed decisions that improve democracy. Indeed, it has led to some gentle ribbing between himself and his brother: John Negroponte - currently deputy secretary of state in the outgoing Bush administration and the first ever director of national intelligence at the National Security Agency. \"I often joke with John that he can bring democracy his way - and I'll bring it mine, \" he says. Apparently providing inexpensive laptops to children in the developing world is not only going to raise educational standards, eradicate poverty, it is also going to bring democracy! All that, with no mention of the numerous poor non-democratic countries that have literacy levels equal to or higher than the USA (Cuba might be one reasonable example). The words naïve technological-utopianism come to mind. I began by admitting that I was conflicted in terms of this project. From the purely technological perspective, there is much to admire in the project’s accomplishments. Sadly, that was not the project’s primary objective. What appears to be missing throughout is an inability to distinguish between the technology and the purpose to which is was intended to serve. My concern in this regard is reflected in a paper by Warschauer & Ames(2010). The analysis reveals that provision of individual laptops is a utopian vision for the children in the poorest countries, whose educational and social futures could be more effectively improved if the same investments were instead made on more sustainable and proven interventions. Middle- and high-income countries may have a stronger rationale for providing individual laptops to children, but will still want to eschew OLPC’s technocentric vision. In summary, OLPC represents the latest in a long line of technologically utopian development schemes that have unsuccessfully attempted to solve complex social problems with overly simplistic solutions. There is a delicate relationship between technology and society, culture, ethics, and values. What this case study reflects is the fact that technologies are not neutral. They never are. Hence, technological initiatives must be accompanied by appropriate social, cultural and ethical considerations – especially in projects such as this where the technologies are being introduced into particularly vulnerable societies. That did not happen here, The fact that this project got the support that it did, and has gone as far as it has, given the way it was approached, is why this reminder – in the form of this device – is included in the collection. And if anyone ever wonders why I am so vocal about the need for public discourse around technology, one need look no further than the OLPC project." - } -] \ No newline at end of file diff --git a/src/scraping/buxton/json/incomplete.json b/src/scraping/buxton/json/incomplete.json deleted file mode 100644 index 4b05a2a86..000000000 --- a/src/scraping/buxton/json/incomplete.json +++ /dev/null @@ -1,468 +0,0 @@ -[ - { - "filename": "3DMag.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "3DPlus.docx", - "year": "ERR__YEAR__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "3DSpace.docx", - "year": "ERR__YEAR__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "3Dconnexion_SpaceNavigator.docx", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "3MErgo.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "ADB2.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "AWrock.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Abaton.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Active.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "AlphaSmart_Pro.docx", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "Apple_ADB_Mouse.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured." - }, - { - "filename": "Apple_Mac_Portable-Katy’s MacBook Air-2.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Apple_Mac_Portable-Katy’s MacBook Air.docx", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Apple_Scroll_Mouse.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Apple_iPhone.docx", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Brailler.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Brewster_Stereoscope.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "CasioTC500.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Citizen_LC_909.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Citizen_LC_913.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured." - }, - { - "filename": "Citizen_LCl_914.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "CoolPix.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "Cross.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Dymo_MK-6.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Emotiv.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Explorer.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Falcon.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "Freeboard.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match was captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "FujitsuPalm.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "FujitsuTouch.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "GRiD1550-Katy’s MacBook Air-2.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "GRiD1550-Katy’s MacBook Air.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "GRiD1550.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "Genius_Ring_Mouse.docx", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "HTC_Touch.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured." - }, - { - "filename": "Helios-Klimax.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Honeywell_T86.docx", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "IBMTrack.docx", - "year": "ERR__YEAR__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "IBM_Convertable-Katy’s MacBook Air-2.docx", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "IBM_Convertable-Katy’s MacBook Air.docx", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "IBM_Convertable.docx", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "IBM_PS2_Mouse.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "IBM_Simon.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value." - }, - { - "filename": "IDEO.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Joyboard.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "Kensington_SB_TB-Mouse.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Leatherman_Tread.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "M1.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured." - }, - { - "filename": "MS-1_Stereoscope.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "MWB_Braille_Writer.docx", - "company": "ERR__COMPANY__: outer match wasn't captured.", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "MaltronLH.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "Marine_Band_Harmonica.docx", - "company": "ERR__COMPANY__: outer match was captured.", - "year": "ERR__YEAR__: outer match was captured.", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Matrox.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "Metaphor_Kbd.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "Metaphor_Mouse.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Motorola_DynaTAC.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "NewO.docx", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Newton120.docx", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "Nikon_Coolpix-100.docx", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured." - }, - { - "filename": "Numonics_Mgr_Mouse.docx", - "company": "ERR__COMPANY__: outer match was captured.", - "year": "ERR__YEAR__: outer match was captured.", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "PadMouse.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "PowerTrack.docx", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "ProAgio.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Pulsar_time_Computer.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "Ring.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "SafeType_Kbd.docx", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "Samsung_SPH-A500.docx", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "SurfMouse.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured." - }, - { - "filename": "TPARCtab.docx", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "Thumbelina.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured." - }, - { - "filename": "adecm.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "eMate.docx", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "gravis.docx", - "year": "ERR__YEAR__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "iGesture.docx", - "year": "__ERR__YEAR__TRANSFORM__: NaN cannot be parsed to a numeric value.", - "primaryKey": "ERR__PRIMARYKEY__: outer match was captured.", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "iGrip.docx", - "shortDescription": "ERR__SHORTDESCRIPTION__: outer match was captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - }, - { - "filename": "iLiad.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "degreesOfFreedom": "ERR__DEGREESOFFREEDOM__: outer match wasn't captured." - }, - { - "filename": "round.docx", - "secondaryKey": "ERR__SECONDARYKEY__: outer match wasn't captured.", - "originalPrice": "ERR__ORIGINALPRICE__: outer match wasn't captured.", - "dimensions": "ERR__DIMENSIONS__: outer match wasn't captured.", - "longDescription": "ERR__LONGDESCRIPTION__: outer match was captured." - } -] \ No newline at end of file diff --git a/src/scraping/buxton/node_scraper.ts b/src/scraping/buxton/node_scraper.ts deleted file mode 100644 index ab6c9dcb2..000000000 --- a/src/scraping/buxton/node_scraper.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { readdirSync, writeFile, existsSync, mkdirSync } from "fs"; -import * as path from "path"; -import { red, cyan, yellow, green } from "colors"; -import { Opt } from "../../new_fields/Doc"; -const StreamZip = require('node-stream-zip'); - -export interface DeviceDocument { - title: string; - shortDescription: string; - longDescription: string; - company: string; - year: number; - originalPrice: number; - degreesOfFreedom: number; - dimensions: string; - primaryKey: string; - secondaryKey: string; -} - -interface AnalysisResult { - device?: DeviceDocument; - errors?: any; -} - -type Converter = (raw: string) => { transformed?: T, error?: string }; - -interface Processor { - exp: RegExp; - matchIndex?: number; - transformer?: Converter; -} - -const RegexMap = new Map>([ - ["title", { - exp: /contact\s+(.*)Short Description:/ - }], - ["company", { - exp: /Company:\s+([^\|]*)\s+\|/, - transformer: (raw: string) => ({ transformed: raw.replace(/\./g, "") }) - }], - ["year", { - exp: /Year:\s+([^\|]*)\s+\|/, - transformer: numberValue - }], - ["primaryKey", { - exp: /Primary:\s+(.*)(Secondary|Additional):/, - transformer: collectUniqueTokens - }], - ["secondaryKey", { - exp: /(Secondary|Additional):\s+([^\{\}]*)Links/, - transformer: collectUniqueTokens, - matchIndex: 2 - }], - ["originalPrice", { - exp: /Original Price \(USD\)\:\s+\$([0-9\.]+)/, - transformer: numberValue - }], - ["degreesOfFreedom", { - exp: /Degrees of Freedom:\s+([0-9]+)/, - transformer: numberValue - }], - ["dimensions", { - exp: /Dimensions\s+\(L x W x H\):\s+([0-9\.]+\s+x\s+[0-9\.]+\s+x\s+[0-9\.]+\s\([A-Za-z]+\))/, - transformer: (raw: string) => { - const [length, width, group] = raw.split(" x "); - const [height, unit] = group.split(" "); - return { - transformed: { - length: Number(length), - width: Number(width), - height: Number(height), - unit: unit.replace(/[\(\)]+/g, "") - } - }; - } - }], - ["shortDescription", { - exp: /Short Description:\s+(.*)Bill Buxton[’']s Notes/, - transformer: correctSentences - }], - ["longDescription", { - exp: /Bill Buxton[’']s Notes(.*)Device Details/, - transformer: correctSentences - }], -]); - -function numberValue(raw: string) { - const transformed = Number(raw); - if (isNaN(transformed)) { - return { error: `${transformed} cannot be parsed to a numeric value.` }; - } - return { transformed }; -} - -function collectUniqueTokens(raw: string) { - return { transformed: Array.from(new Set(raw.replace(/,|\s+and\s+/g, " ").split(/\s+/).map(token => token.toLowerCase().trim()))).map(capitalize).sort() }; -} - -function correctSentences(raw: string) { - raw = raw.replace(/\./g, ". ").replace(/\:/g, ": ").replace(/\,/g, ", ").replace(/\?/g, "? ").trimRight(); - raw = raw.replace(/\s{2,}/g, " "); - return { transformed: raw }; -} - -const outDir = path.resolve(__dirname, "json"); -const successOut = "buxton.json"; -const failOut = "incomplete.json"; -const deviceKeys = Array.from(RegexMap.keys()); - -function printEntries(zip: any) { - const { entriesCount } = zip; - console.log(`Recognized ${entriesCount} entr${entriesCount === 1 ? "y" : "ies"}.`); - for (const entry of Object.values(zip.entries())) { - const desc = entry.isDirectory ? 'directory' : `${entry.size} bytes`; - console.log(`${entry.name}: ${desc}`); - } -} - -async function wordToPlainText(pathToDocument: string): Promise { - const zip = new StreamZip({ file: pathToDocument, storeEntries: true }); - const contents = await new Promise((resolve, reject) => { - zip.on('ready', () => { - let body = ""; - zip.stream("word/document.xml", (error: any, stream: any) => { - if (error) { - reject(error); - } - stream.on('data', (chunk: any) => body += chunk.toString()); - stream.on('end', () => { - resolve(body); - zip.close(); - }); - }); - }); - }); - let body = ""; - const components = contents.toString().split(''); - const content = tags[1].replace(/<.*$/, ""); - body += content; - } - return body; -} - -function tryGetValidCapture(matches: RegExpExecArray | null, matchIndex: number): Opt { - let captured: string; - if (!matches || !(captured = matches[matchIndex])) { - return undefined; - } - const lower = captured.toLowerCase(); - if (/to come/.test(lower)) { - return undefined; - } - if (lower.includes("xxx")) { - return undefined; - } - if (!captured.toLowerCase().replace(/[….\s]+/g, "").length) { - return undefined; - } - return captured; -} - -function capitalize(word: string): string { - const clean = word.trim(); - if (!clean.length) { - return word; - } - return word.charAt(0).toUpperCase() + word.slice(1); -} - -function analyze(path: string, body: string): AnalysisResult { - const device: any = {}; - - const segments = path.split("/"); - const filename = segments[segments.length - 1].replace("Bill_Notes_", ""); - - const errors: any = { filename }; - - for (const key of deviceKeys) { - const { exp, transformer, matchIndex } = RegexMap.get(key)!; - const matches = exp.exec(body); - - let captured = tryGetValidCapture(matches, matchIndex ?? 1); - if (!captured) { - errors[key] = `ERR__${key.toUpperCase()}__: outer match ${matches === null ? "wasn't" : "was"} captured.`; - continue; - } - - captured = captured.replace(/\s{2,}/g, " "); - if (transformer) { - const { error, transformed } = transformer(captured); - if (error) { - errors[key] = `__ERR__${key.toUpperCase()}__TRANSFORM__: ${error}`; - continue; - } - captured = transformed; - } - - device[key] = captured; - } - - const errorKeys = Object.keys(errors); - if (errorKeys.length > 1) { - console.log(red(`\n@ ${cyan(filename.toUpperCase())}...`)); - errorKeys.forEach(key => key !== "filename" && console.log(red(errors[key]))); - return { errors }; - } - - return { device }; -} - -async function parseFiles(): Promise { - const sourceDirectory = path.resolve(`${__dirname}/source`); - const candidates = readdirSync(sourceDirectory).filter(file => file.endsWith(".doc") || file.endsWith(".docx")).map(file => `${sourceDirectory}/${file}`); - const imported = await Promise.all(candidates.map(async path => ({ path, body: await wordToPlainText(path) }))); - // const imported = [{ path: candidates[10], body: await extract(candidates[10]) }]; - const data = imported.map(({ path, body }) => analyze(path, body)); - const masterDevices: DeviceDocument[] = []; - const masterErrors: any[] = []; - data.forEach(({ device, errors }) => { - if (device) { - masterDevices.push(device); - } else { - masterErrors.push(errors); - } - }); - const total = candidates.length; - if (masterDevices.length + masterErrors.length !== total) { - throw new Error(`Encountered a ${masterDevices.length} to ${masterErrors.length} mismatch in device / error split!`); - } - console.log(); - await writeOutputFile(successOut, masterDevices, total, true); - await writeOutputFile(failOut, masterErrors, total, false); - console.log(); - - return masterDevices; -} - -async function writeOutputFile(relativePath: string, data: any[], total: number, success: boolean) { - console.log(yellow(`Encountered ${data.length} ${success ? "valid" : "invalid"} documents out of ${total} candidates. Writing ${relativePath}...`)); - return new Promise((resolve, reject) => { - const destination = path.resolve(outDir, relativePath); - const contents = JSON.stringify(data, undefined, 4); - writeFile(destination, contents, err => err ? reject(err) : resolve()); - }); -} - -export async function main() { - if (!existsSync(outDir)) { - mkdirSync(outDir); - } - return parseFiles(); -} - -main(); \ No newline at end of file diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index dbf274e93..e590a5b85 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -4,7 +4,7 @@ import { exec } from 'child_process'; import { command_line } from "../ActionUtilities"; import RouteSubscriber from "../RouteSubscriber"; import { red } from "colors"; -import { main } from "../../scraping/buxton/node_scraper"; +import executeImport from "../../scraping/buxton/final/BuxtonImporter"; export default class UtilManager extends ApiManager { @@ -40,29 +40,29 @@ export default class UtilManager extends ApiManager { } }); - register({ - method: Method.GET, - subscription: "/buxton", - secureHandler: async ({ res }) => { - const cwd = './src/scraping/buxton'; + // register({ + // method: Method.GET, + // subscription: "/buxton", + // secureHandler: async ({ res }) => { + // const cwd = './src/scraping/buxton'; - const onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; - const onRejected = (err: any) => { console.error(err.message); res.send(err); }; - const tryPython3 = (reason: any) => { - console.log("Initial scraper failed for the following reason:"); - console.log(red(reason.Error)); - console.log("Falling back to python3..."); - return command_line('python3 scraper.py', cwd).then(onResolved, onRejected); - }; + // const onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; + // const onRejected = (err: any) => { console.error(err.message); res.send(err); }; + // const tryPython3 = (reason: any) => { + // console.log("Initial scraper failed for the following reason:"); + // console.log(red(reason.Error)); + // console.log("Falling back to python3..."); + // return command_line('python3 scraper.py', cwd).then(onResolved, onRejected); + // }; - return command_line('python scraper.py', cwd).then(onResolved, tryPython3); - }, - }); + // return command_line('python scraper.py', cwd).then(onResolved, tryPython3); + // }, + // }); register({ method: Method.GET, - subscription: "/newBuxton", - secureHandler: async ({ res }) => res.send(await main()) + subscription: "/buxton", + secureHandler: async ({ res }) => res.send(await executeImport()) }); register({ diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 71775bed6..896b88631 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -58,7 +58,7 @@ export class CurrentUserUtils { { title: "todo item", icon: "check", ignoreClick: true, drag: 'getCopy(this.dragFactory, true)', dragFactory: notes[notes.length - 1] }, { title: "web page", icon: "globe-asia", ignoreClick: true, drag: 'Docs.Create.WebDocument("https://en.wikipedia.org/wiki/Hedgehog", {_width: 300, _height: 300, title: "New Webpage" })' }, { title: "cat image", icon: "cat", ignoreClick: true, drag: 'Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", { _width: 200, title: "an image of a cat" })' }, - { title: "buxton", icon: "faObjectGroup", ignoreClick: true, drag: "Docs.Create.Buxton()" }, + { title: "buxton", icon: "cloud-upload-alt", ignoreClick: true, drag: "Docs.Create.Buxton()" }, { title: "record", icon: "microphone", ignoreClick: true, drag: `Docs.Create.AudioDocument("${nullAudio}", { _width: 200, title: "ready to record audio" })` }, { title: "clickable button", icon: "bolt", ignoreClick: true, drag: 'Docs.Create.ButtonDocument({ _width: 150, _height: 50, title: "Button" })' }, { title: "presentation", icon: "tv", click: 'openOnRight(Doc.UserDoc().curPresentation = getCopy(this.dragFactory, true))', drag: `Doc.UserDoc().curPresentation = getCopy(this.dragFactory,true)`, dragFactory: emptyPresentation }, -- cgit v1.2.3-70-g09d2