diff --git a/src/gdoc.ts b/src/gdoc.ts index 62795c130..e6464b4e7 100644 --- a/src/gdoc.ts +++ b/src/gdoc.ts @@ -42,7 +42,7 @@ export async function parse( file: VFile, fetch: boolean = true ): Promise { - const json = dump(file) + const json = await dump(file) const gdoc = JSON.parse(json) return parseDocument(gdoc, fetch) } diff --git a/src/html.ts b/src/html.ts index 434dbb804..a91300eef 100644 --- a/src/html.ts +++ b/src/html.ts @@ -68,7 +68,7 @@ export const mediaTypes = ['text/html'] * @returns A promise that resolves to a `stencila.Node` */ export async function parse(file: VFile): Promise { - const html = dump(file) + const html = await dump(file) const dom = new jsdom.JSDOM(html) const document = dom.window.document collapse(document) diff --git a/src/index.ts b/src/index.ts index ac3f0329f..824282f34 100644 --- a/src/index.ts +++ b/src/index.ts @@ -299,18 +299,19 @@ export async function write( /** * Convert content from one format to another. * - * @param inp The input content (raw or file path). - * @param out The output file path. + * @param input The input content (raw or file path). + * @param outputPath The output file path. * @param options Conversion options e.g `from` and `to`: to specify the formats to convert from/to * @returns The converted content, or file path (for converters that only write to files). */ export async function convert( - inp: string, - out?: string, + input: string, + outputPath?: string, options: { [key: string]: any } = {} ): Promise { - const node = await read(inp, options.from) - let file = await unparse(node, out, options.to) - if (out) await vfile.write(file, out) - return file.contents ? vfile.dump(file) : file.path + const inputFile = vfile.create(input) + const node = await parse(inputFile, input, options.from) + const outputFile = await unparse(node, outputPath, options.to) + if (outputPath) await vfile.write(outputFile, outputPath) + return outputFile.contents ? vfile.dump(outputFile) : outputFile.path } diff --git a/src/json.ts b/src/json.ts index b1c784a17..2051f3f98 100644 --- a/src/json.ts +++ b/src/json.ts @@ -8,7 +8,7 @@ import { dump, load, VFile } from './vfile' export const mediaTypes = ['application/json'] export async function parse(file: VFile): Promise { - return JSON.parse(dump(file)) + return JSON.parse(await dump(file)) } export async function unparse(node: stencila.Node): Promise { diff --git a/src/json5.ts b/src/json5.ts index 75b185b84..e9295dff9 100644 --- a/src/json5.ts +++ b/src/json5.ts @@ -48,7 +48,7 @@ export const mediaTypes = ['application/json5'] * @returns A promise that resolves to a Stencila `Node` */ export async function parse(file: VFile): Promise { - return json5.parse(dump(file)) + return json5.parse(await dump(file)) } /** diff --git a/src/pandoc.ts b/src/pandoc.ts index ac95cece1..408337c9e 100644 --- a/src/pandoc.ts +++ b/src/pandoc.ts @@ -3,7 +3,7 @@ import childProcess from 'child_process' import { pandocDataDir, pandocPath } from './boot' import * as Pandoc from './pandoc-types' import * as rpng from './rpng' -import { create, dump, load, VFile, write } from './vfile' +import { create, load, VFile, write } from './vfile' export { InputFormat, OutputFormat } from './pandoc-types' @@ -26,11 +26,13 @@ export async function parse( ensureFile: boolean = false ): Promise { const args = [`--from=${from}`, `--to=json`].concat(options) - let content = dump(file) + + let content = file.contents if (!content || ensureFile) { if (ensureFile && !file.path) throw new Error('Must supply a file') args.push(`${file.path}`) } + const json = await run(content, args) const pdoc = JSON.parse(json) return parseDocument(pdoc) @@ -62,7 +64,6 @@ export async function unparse( const pdoc = unparseArticle(node as stencila.Article) await Promise.all(unparsePromises) - const json = JSON.stringify(pdoc) const args = [`--from=json`, `--to=${to}`].concat(options) if ((filePath && filePath !== '-') || ensureFile) { let output @@ -74,12 +75,13 @@ export async function unparse( args.push(`--output=${output}`) } + const json = JSON.stringify(pdoc) const content = await run(json, args) - // If content was output, then load that into a vfile, - // otherwise the vfile, simply has path to the file created + // If content was outputted, then load that into a vfile, + // otherwise the vfile simply has path to the file created if (content) return load(content) - else return create({ path: filePath }) + else return create(undefined, { path: filePath }) } /** @@ -95,13 +97,14 @@ let unparsePromises: Promise[] = [] /** * Run the Pandoc binary */ -function run(input: string, args: string[]): Promise { +function run(input: string | Buffer, args: string[]): Promise { args.push(`--data-dir=${pandocDataDir}`) if (process.env.DEBUG) { console.log(`Running ${pandocPath} with args:\n ${args.join('\n ')}`) } return new Promise((resolve, reject) => { const child = childProcess.spawn(pandocPath, args) + let stdout = '' let stderr = '' child.stdout.on('data', data => { @@ -121,8 +124,13 @@ function run(input: string, args: string[]): Promise { child.on('error', err => { reject(err) }) - child.stdin.write(input) - child.stdin.end() + + if (input && input.length) { + child.stdin.write(input, err => { + if (err) return reject(err) + child.stdin.end() + }) + } }) } diff --git a/src/tdp.ts b/src/tdp.ts index 30b027fc9..32eac3f2e 100644 --- a/src/tdp.ts +++ b/src/tdp.ts @@ -36,7 +36,7 @@ export const extNames = [ export async function parse(file: VFile): Promise { let pkg: datapackage.Package if (file.path) pkg = await datapackage.Package.load(file.path) - else pkg = await datapackage.Package.load(JSON.parse(dump(file))) + else pkg = await datapackage.Package.load(JSON.parse(await dump(file))) // Parse resources const parts = await Promise.all(pkg.resources.map( @@ -199,7 +199,7 @@ async function unparseCreativeWork( profile: 'tabular-data-resource', name: datatable.name || 'Unnamed', - data: dump(await csv.unparse(datatable)), + data: await dump(await csv.unparse(datatable)), format: 'csv', mediatype: 'text/csv', encoding: 'utf-8', diff --git a/src/vfile.ts b/src/vfile.ts index 9db431b10..a33b3a986 100644 --- a/src/vfile.ts +++ b/src/vfile.ts @@ -12,9 +12,17 @@ export type VFile = vfile.VFile /** * Create a virtual file * - * @param options to `vfile` see https://github.com/vfile/vfile#vfileoptions + * @param contents Raw string contents of `VFile`, or file path + * @param options Options to `vfile` see https://github.com/vfile/vfile#vfileoptions */ -export function create(options: any = {}): VFile { +export function create( + contents?: string, + options: { [key: string]: any } = {} +): VFile { + if (contents) { + if (isPath(contents)) options = { ...options, path: contents } + else options = { ...options, contents } + } return vfile(options) } @@ -30,10 +38,18 @@ export function load(contents: VFileContents): VFile { /** * Dump a string from the contents of a virtual files * + * If the file has no `contents` but does have a file + * `path`, then `read` the virtual file. + * * @param file The virtual file to dump */ -export function dump(vfile: VFile): string { - return vfile.contents ? vfile.contents.toString() : '' +export async function dump(vfile: VFile): Promise { + if (vfile.contents) return vfile.toString() + if (vfile.path) { + const readFile = await toVFile.read(vfile.path) + return readFile.toString() + } + return '' } /** @@ -66,7 +82,7 @@ export async function write(vfile: VFile, path: string): Promise { if (!path) throw new Error('Argument `path` is required') if (path === '-') { - console.log(dump(vfile)) + console.log(await dump(vfile)) } else if (path && vfile.contents) { vfile.path = path await toVFile.write(vfile) diff --git a/src/yaml.ts b/src/yaml.ts index 914a36433..9e2ed4edc 100644 --- a/src/yaml.ts +++ b/src/yaml.ts @@ -9,7 +9,7 @@ import { dump, load, VFile } from './vfile' export const mediaTypes = ['text/yaml'] export async function parse(file: VFile): Promise { - return yaml.safeLoad(dump(file)) + return yaml.safeLoad(await dump(file)) } export async function unparse(node: stencila.Node): Promise { diff --git a/tests/csv.test.ts b/tests/csv.test.ts index bf59da5aa..6b4032c8c 100644 --- a/tests/csv.test.ts +++ b/tests/csv.test.ts @@ -99,7 +99,7 @@ test('parse', async () => { }) test('unparse', async () => { - expect(dump(await unparse(simple.node))).toEqual(simple.content) - expect(dump(await unparse(named.node))).toEqual(named.content) - expect(dump(await unparse(formulas.node))).toEqual(formulas.content) + expect(await dump(await unparse(simple.node))).toEqual(simple.content) + expect(await dump(await unparse(named.node))).toEqual(named.content) + expect(await dump(await unparse(formulas.node))).toEqual(formulas.content) }) diff --git a/tests/gdoc.test.ts b/tests/gdoc.test.ts index 23ffefe48..edc532ee8 100644 --- a/tests/gdoc.test.ts +++ b/tests/gdoc.test.ts @@ -7,7 +7,7 @@ test('parse', async () => { }) test('unparse', async () => { - const u = async (node: any) => JSON.parse(dump(await unparse(node))) + const u = async (node: any) => JSON.parse(await dump(await unparse(node))) expect(await u(kitchenSink.node)).toEqual(kitchenSink.gdoc) }) diff --git a/tests/html.test.ts b/tests/html.test.ts index 1c4822ba0..1a2ec3a36 100644 --- a/tests/html.test.ts +++ b/tests/html.test.ts @@ -7,7 +7,7 @@ test('parse', async () => { }) test('unparse', async () => { - expect(dump(await unparse(kitchenSink.node))).toEqual(kitchenSink.html) + expect(await dump(await unparse(kitchenSink.node))).toEqual(kitchenSink.html) }) // An example intended for testing progressively added parser/unparser pairs diff --git a/tests/issues/74-write-epipe.docx b/tests/issues/74-write-epipe.docx new file mode 100644 index 000000000..50b83394a Binary files /dev/null and b/tests/issues/74-write-epipe.docx differ diff --git a/tests/issues/74-write-epipe.test.ts b/tests/issues/74-write-epipe.test.ts new file mode 100644 index 000000000..2db0c7a24 --- /dev/null +++ b/tests/issues/74-write-epipe.test.ts @@ -0,0 +1,33 @@ +import fs from 'fs-extra' +import path from 'path' +import { convert } from '../../src' + +test('issue 74', async () => { + /** + * The [issue](https://github.com/stencila/convert/issues/74) + * was reported with respect to converting to HTML + * but it also occurs when converting to JSON. + * + * $ npx ts-node --files src/cli ./tests/issues/74-write-epipe.docx --to json + * + * Error: write EPIPE + * at WriteWrap.afterWrite (net.js:836:14) + * + * This suggests it's a problem within `read` and + * [this issue](https://github.com/nodejs/node/issues/947) + * suggest that it is a problem with use of `stdin` of `stdout` + * within `pandoc.run`. + */ + + const docx = path.join(__dirname, '74-write-epipe.docx') + const html = path.join(__dirname, '74-write-epipe.html') + + // Before fix, this fails with `Error: write EPIPE` + // https://travis-ci.org/stencila/convert/jobs/537572367#L172 + // https://ci.appveyor.com/project/nokome/convert/builds/24825307#L66 + convert(docx, html) + + // Cleanup + await fs.remove(html) + await fs.remove(docx + '.media') +}) diff --git a/tests/pandoc.test.ts b/tests/pandoc.test.ts index e9032915d..09e317d92 100644 --- a/tests/pandoc.test.ts +++ b/tests/pandoc.test.ts @@ -22,7 +22,7 @@ test('parse', async () => { }) test('unparse', async () => { - const u = async (node: any) => JSON.parse(dump(await unparse(node))) + const u = async (node: any) => JSON.parse(await dump(await unparse(node))) expect(await u(kitchenSink.node)).toEqual(kitchenSink.pdoc) }) @@ -566,7 +566,7 @@ test('rpngs', async () => { expect(await parse(load(JSON.stringify(pdoc)))).toEqual(node) // Skipping this until resolve how to deal with output RPNG paths - //expect(JSON.parse(dump(await unparse(node)))).toEqual(pdoc) + //expect(JSON.parse(await dump(await unparse(node)))).toEqual(pdoc) }) // A very simple test of the approach to typing Pandoc nodes diff --git a/tests/tdp.test.ts b/tests/tdp.test.ts index daf614ceb..62a51a4d1 100644 --- a/tests/tdp.test.ts +++ b/tests/tdp.test.ts @@ -137,11 +137,11 @@ test('parse', async () => { }) test('unparse', async () => { - const actual = JSON.parse(dump(await unparse(periodic.node))) + const actual = JSON.parse(await dump(await unparse(periodic.node))) // Pretend that we unparsed to a a filePath i.e. that // the data was written to disk. delete actual.resources[0].data actual.resources[0].path = 'periodic-table.csv' - const expected = JSON.parse(dump(await read(periodic.file))) + const expected = JSON.parse(await dump(await read(periodic.file))) expect(actual).toEqual(expected) })