-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
8 changed files
with
297 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| import debugm from 'debug'; | ||
| const debug = debugm('eczoo_sitegen.scripts.helperEcZooLoader'); | ||
|
|
||
| import fs from 'fs'; | ||
| import path from 'path'; | ||
|
|
||
| import _ from 'lodash'; | ||
|
|
||
| import { ZooDbDataLoaderHandler } from '@phfaist/zoodb'; | ||
| import { createEcZooDb } from '@errorcorrectionzoo/eczoodb/eczoodb.js'; | ||
| import { createEcZooYamlDbDataLoader } from '@errorcorrectionzoo/eczoodb/load_yamldb.js'; | ||
|
|
||
| import { zoo_permalinks } from '@errorcorrectionzoo/eczoodb/permalinks.js'; | ||
|
|
||
| import { get_eczoo_full_options } from '@errorcorrectionzoo/eczoodb/fullopts.js'; | ||
|
|
||
| import { fileURLToPath } from 'url'; | ||
| const __filename = fileURLToPath(import.meta.url); | ||
| const __dirname = path.dirname(__filename); | ||
|
|
||
|
|
||
|
|
||
| export async function loadEcZoo({ dataDir, } = {}) | ||
| { | ||
| debug(`loadEcZoo(), dataDir=${dataDir}`); | ||
|
|
||
| dataDir ??= path.join(__dirname, '..', '..', 'eczoo_data'); | ||
|
|
||
| debug(`Using dataDir=‘${dataDir}’`); | ||
|
|
||
| const eczoodbopts = _.merge( | ||
| { | ||
| fs, | ||
| fs_data_dir: dataDir, | ||
| }, | ||
| get_eczoo_full_options({ | ||
| citationsinfo_cache_dir: path.join(__dirname, '..', '_zoodb_citations_cache'), | ||
| }), | ||
| { | ||
| flm_options: { | ||
| resources: { | ||
| rename_figure_template: | ||
| (f) => `fig-${f.b32hash(24)}.pdf`, | ||
| }, | ||
| }, | ||
| zoo_permalinks: { | ||
| object: (object_type, object_id) => { | ||
| return 'https://errorcorrectionzoo.org' | ||
| + zoo_permalinks.object(object_type, object_id); | ||
| }, | ||
| graphics_resource: (graphics_resource) => | ||
| `__abstract_fig_reference__/${graphics_resource.src_url}` | ||
| }, | ||
| }, | ||
| ); | ||
|
|
||
| let eczoodb = await createEcZooDb(eczoodbopts); | ||
| const yaml_loader = await createEcZooYamlDbDataLoader(eczoodb); | ||
| const loader_handler = new ZooDbDataLoaderHandler( | ||
| yaml_loader, | ||
| { | ||
| throw_reload_errors: false, // for when in devel mode with eleventy | ||
| } | ||
| ); | ||
|
|
||
| await eczoodb.install_zoo_loader_handler(loader_handler); | ||
|
|
||
| await eczoodb.load(); | ||
|
|
||
| return eczoodb; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| { | ||
| "name": "@errorcorrectionzoo/scripts", | ||
| "version": "0.0.1", | ||
| "packageManager": "yarn@3.3.0", | ||
| "private": true, | ||
| "type": "module", | ||
| "dependencies": { | ||
| "@errorcorrectionzoo/eczoodb": ">=0.0.1", | ||
| "@phfaist/zoodb": "https://github.com/phfaist/zoodb.git#main", | ||
| "debug": "^4.3.4", | ||
| "lodash": "^4.17.21", | ||
| "yargs": "^17.7.2" | ||
| }, | ||
| "peerDependencies": { | ||
| "@errorcorrectionzoo/eczoodb": "*", | ||
| "@phfaist/zoodb": "*" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| import debugm from 'debug'; | ||
| const debug = debugm('eczoo_sitegen.scripts.query_bib_references'); | ||
|
|
||
| import fs from 'fs'; | ||
|
|
||
| import yargs from 'yargs'; | ||
| import { hideBin } from 'yargs/helpers'; | ||
|
|
||
| import { loadEcZoo } from './helperEcZooLoader.js'; | ||
|
|
||
| // | ||
| // Core script function. Loads the zoo and performs the desired analysis. | ||
| // | ||
| async function runmain(args) | ||
| { | ||
| process.stderr.write('runmain(): loading zoo... (might take a couple minutes)\n'); | ||
| const eczoodb = await loadEcZoo({ dataDir: args.dataDir }); | ||
|
|
||
| process.stderr.write('runmain(): zoo is now loaded!\n'); | ||
|
|
||
| // | ||
| // Zoo is loaded (eczoodb). Query anything we need from it at this point. | ||
| // | ||
|
|
||
| const encountered_citations = eczoodb.zoo_flm_processor.scanner.get_encountered('citations'); | ||
|
|
||
| let data = {}; | ||
|
|
||
| for (const encountered_citation of encountered_citations) { | ||
| //debug(`encountered citation, `, encountered_citation); | ||
| const { cite_prefix, cite_key } = encountered_citation; | ||
| const { object_type, object_id, source_path } = | ||
| encountered_citation.encountered_in.resource_info; | ||
| if (args.citePrefix && args.citePrefix !== cite_prefix) { | ||
| continue; | ||
| } | ||
| if (args.domain) { | ||
| if (object_type !== 'code') { | ||
| continue; | ||
| } | ||
| const code = eczoodb.objects.code[object_id]; | ||
| const domains = eczoodb.code_parent_domains(code, { find_domain_id: args.domain }); | ||
| if (domains.length !== 1 || domains[0] !== args.domain) { | ||
| continue; | ||
| } | ||
| } | ||
| if (data[cite_prefix] == null) { | ||
| data[cite_prefix] = {}; | ||
| } | ||
| if (data[cite_prefix][cite_key] == null) { | ||
| data[cite_prefix][cite_key] = []; | ||
| } | ||
| data[cite_prefix][cite_key].push({ source_path }); | ||
| } | ||
|
|
||
| // | ||
| // Our relevant data is stored in `data`. Now we output the information the | ||
| // way the user requested it. | ||
| // | ||
|
|
||
| let outputData = ''; | ||
|
|
||
| if (args.format === 'json') { | ||
|
|
||
| let data2 = _.merge({}, data); | ||
| if (!args.includeSourcePath) { | ||
| for (const cite_prefix of [...Object.keys(data)]) { | ||
| for (const cite_key of [...Object.keys(data[cite_prefix])]) { | ||
| data[cite_prefix][cite_key] = true; | ||
| } | ||
| } | ||
| } | ||
| outputData = JSON.stringify(data2, null, 4); | ||
|
|
||
| } else if (args.format === 'txt') { | ||
|
|
||
| let cite_prefix_list = Object.keys(data); | ||
| cite_prefix_list.sort(); | ||
| for (const cite_prefix of cite_prefix_list) { | ||
| const db = data[cite_prefix]; | ||
| let cite_key_list = Object.keys(db); | ||
| cite_key_list.sort(); | ||
| for (const cite_key of cite_key_list) { | ||
| const db2 = db[cite_key]; | ||
| outputData += `${cite_prefix}:${cite_key}`; | ||
| if (args.includeSourcePath) { | ||
| outputData += ` :\n` + db2.map( (x) => `\t\t${x.source_path}\n` ).join(''); | ||
| } else { | ||
| outputData += `\n`; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| if ( ! args.output ) { | ||
| process.stdout.write(outputData); | ||
| } else { | ||
| process.stderr.write(`Writing output to ${args.output}\n`); | ||
| fs.writeFileSync( args.output, outputData ); | ||
| } | ||
|
|
||
| return; | ||
| } | ||
|
|
||
|
|
||
| // | ||
| // Main function. Parse command-line arguments and call runmain(). | ||
| // | ||
| async function main() | ||
| { | ||
| const args = yargs(hideBin(process.argv)) | ||
| .scriptName('query_bib_references') | ||
| .usage('Usage: $0 [options]') | ||
| .options({ | ||
| 'data-dir': { | ||
| alias: 'd', | ||
| default: null, | ||
| describe: "Data repository folder (defaults to sibling `eczoo_data` folder)", | ||
| }, | ||
| 'cite-prefix': { | ||
| alias: 'p', | ||
| default: null, | ||
| describe: "Only include citations with the given cite_prefix (e.g. 'doi' or 'arxiv')", | ||
| }, | ||
| 'domain': { | ||
| alias: 'D', | ||
| default: null, | ||
| describe: "Only include citations in codes that belong to the given domain_id", | ||
| }, | ||
| 'format': { | ||
| alias: 'f', | ||
| default: 'txt', | ||
| describe: "Output format ('txt' or 'json')", | ||
| }, | ||
| 'output': { | ||
| alias: 'o', | ||
| default: null, | ||
| describe: "Output file to write to (default stdout)", | ||
| }, | ||
| 'include-source-path': { | ||
| alias: 's', | ||
| default: true, | ||
| boolean: true, | ||
| describe: "Whether or not to include the files where the citation was encountered", | ||
| }, | ||
| }) | ||
| .strictOptions() | ||
| .argv | ||
| ; | ||
|
|
||
| await runmain(args); | ||
| } | ||
|
|
||
| await main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters