From 82597fca427c6fd4f78500e44a9cf239efbadb2f Mon Sep 17 00:00:00 2001 From: Simon Hong Date: Thu, 16 Apr 2020 15:20:02 +0900 Subject: [PATCH] Generate super referral mapping table component --- package.json | 3 + .../generateNTPSuperReferrerMappingTable.js | 64 ++++++++++ ...geNTPSuperReferrerMappingTableComponent.js | 109 ++++++++++++++++++ scripts/uploadComponent.js | 2 +- 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 scripts/generateNTPSuperReferrerMappingTable.js create mode 100644 scripts/packageNTPSuperReferrerMappingTableComponent.js diff --git a/package.json b/package.json index 3bdb3357..51e4f859 100755 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "data-files-speedreader": "npm run --prefix ./node_modules/speedreader data-files", "generate-ntp-sponsored-images": "node scripts/generateNTPSponsoredImages.js", "generate-ntp-super-referrer": "node scripts/generateNTPSuperReferrer.js", + "generate-ntp-super-referrer-mapping-table": "node scripts/generateNTPSuperReferrerMappingTable.js", "generate-parameter-updates": "node scripts/generateClientModelParameterFiles.js", "lint": "standard", "import-cws-components": "node ./scripts/importCWSComponents", @@ -51,6 +52,7 @@ "package-local-data-files": "node ./scripts/packageComponent --type local-data-files-updater", "package-ntp-sponsored-images": "node ./scripts/packageNTPSponsoredImagesComponents", "package-ntp-super-referrer": "node scripts/packageNTPSuperReferrerComponent.js", + "package-ntp-super-referrer-mapping-table": "node scripts/packageNTPSuperReferrerMappingTableComponent.js", "package-speedreader": "node ./scripts/packageComponent --type speedreader-updater", "upload-ethereum-remote-client": "node ./scripts/uploadComponent --type ethereum-remote-client", "package-parameter-updates": "node ./scripts/packageClientModelParameterComponent", @@ -62,6 +64,7 @@ "upload-local-data-files": "node ./scripts/uploadComponent --type local-data-files-updater", "upload-ntp-sponsored-images-components": "node ./scripts/uploadComponent --type ntp-sponsored-images", "upload-ntp-super-referrer-component": "node ./scripts/uploadComponent --type ntp-super-referrer", + "upload-ntp-super-referrer-component-mapping-table": "node ./scripts/uploadComponent --type ntp-super-referrer-mapping-table", "upload-parameter-updates": "node ./scripts/uploadComponent --type client-model-parameters", "upload-speedreader": "node ./scripts/uploadComponent --type speedreader-updater" }, diff --git a/scripts/generateNTPSuperReferrerMappingTable.js b/scripts/generateNTPSuperReferrerMappingTable.js new file mode 100644 index 00000000..dac90187 --- /dev/null +++ b/scripts/generateNTPSuperReferrerMappingTable.js @@ -0,0 +1,64 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const path = require('path') +const mkdirp = require('mkdirp') +const fs = require('fs-extra') +const request = require('request') +const commander = require('commander') + +const jsonSchemaVersion = 1 + +const createDataJsonFile = (path, body) => { + fs.writeFileSync(path, body) +} + +function downloadMappingTableJsonFile (jsonFileUrl, targetFilePath) { + return new Promise(function (resolve, reject) { + let jsonFileBody = '{}' + + request(jsonFileUrl, async function (error, response, body) { + if (error) { + console.error(`Error from ${jsonFileUrl}:`, error) + return reject(error) + } + if (response && response.statusCode === 200) { + jsonFileBody = body + } + + const data = JSON.parse(jsonFileBody) + // Make sure the data has a schema version so that clients can opt to parse or not + const incomingSchemaVersion = data.schemaVersion + if (!incomingSchemaVersion) { + // Source has no schema version, assume and set current version. + // TODO(petemill): Don't allow this once the source is established to always + // have a schema version. + data.schemaVersion = jsonSchemaVersion + } else if (incomingSchemaVersion !== jsonSchemaVersion) { + // We don't support this file format + console.error(`Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`) + return reject(error) + } + + createDataJsonFile(targetFilePath, JSON.stringify(data)) + + resolve() + }) + }) +} + +async function generateNTPSuperReferrerMappingTable (dataUrl) { + const rootResourceDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table') + mkdirp.sync(rootResourceDir) + + console.log(`Downloading ${dataUrl}...`) + const targetFilePath = path.join(rootResourceDir, 'mapping-table.json') + await downloadMappingTableJsonFile(dataUrl, targetFilePath) +} + +commander + .option('-d, --data-url ', 'url that refers to data that has ntp super referrer') + .parse(process.argv) + +generateNTPSuperReferrerMappingTable(commander.dataUrl) diff --git a/scripts/packageNTPSuperReferrerMappingTableComponent.js b/scripts/packageNTPSuperReferrerMappingTableComponent.js new file mode 100644 index 00000000..c4815e8c --- /dev/null +++ b/scripts/packageNTPSuperReferrerMappingTableComponent.js @@ -0,0 +1,109 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const childProcess = require('child_process') +const commander = require('commander') +const fs = require('fs-extra') +const mkdirp = require('mkdirp') +const path = require('path') +const replace = require('replace-in-file') +const util = require('../lib/util') + +const stageFiles = (version, outputDir) => { + // Copy resources and manifest file to outputDir. + const resourceDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table', '/') + console.log('copy dir:', resourceDir, ' to:', outputDir) + fs.copySync(resourceDir, outputDir) + + // Fix up the manifest version + const originalManifest = getOriginalManifest() + const outputManifest = path.join(outputDir, 'manifest.json') + console.log('copy manifest file: ', originalManifest, ' to: ', outputManifest) + const replaceOptions = { + files: outputManifest, + from: /0\.0\.0/, + to: version + } + fs.copyFileSync(originalManifest, outputManifest) + replace.sync(replaceOptions) +} + +const generateManifestFile = (publicKey) => { + const manifestFile = getOriginalManifest() + const manifestContent = { + description: 'Brave NTP Super Referrer mapping table component', + key: publicKey, + manifest_version: 2, + name: `Brave NTP Super Referrer mapping table`, + version: '0.0.0' + } + fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) +} + +const getOriginalManifest = () => { + return path.join(path.resolve(), 'build','ntp-super-referrer', 'mapping-table-manifest.json') +} + +const generatePublicKeyAndID = (privateKeyFile) => { + childProcess.execSync(`openssl rsa -in ${privateKeyFile} -pubout -out public.pub`) + try { + // read contents of the file + const data = fs.readFileSync('public.pub', 'UTF-8'); + + // split the contents by new line + const lines = data.split(/\r?\n/); + let pubKeyString = '' + lines.forEach((line) => { + if (!line.includes('-----')) + pubKeyString += line + }); + console.log(`publicKey: ${pubKeyString}`) + const id = util.getIDFromBase64PublicKey(pubKeyString) + console.log(`componentID: ${id}`) + return [pubKeyString, id] + } catch (err) { + console.error(err); + } +} + +const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile) => { + const originalManifest = getOriginalManifest() + const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table') + const stagingDir = path.join(rootBuildDir, 'staging') + const crxOutputDir = path.join(rootBuildDir, 'output') + mkdirp.sync(stagingDir) + mkdirp.sync(crxOutputDir) + util.getNextVersion(endpoint, region, componentID).then((version) => { + const crxFile = path.join(crxOutputDir, `ntp-super-referrer-mapping-table.crx`) + stageFiles(version, stagingDir) + util.generateCRXFile(binary, crxFile, privateKeyFile, stagingDir) + console.log(`Generated ${crxFile} with version number ${version}`) + }) +} + +util.installErrorHandlers() + +commander + .option('-b, --binary ', 'Path to the Chromium based executable to use to generate the CRX file') + .option('-k, --key ', 'file containing private key for signing crx file') + .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '')// If setup locally, use http://localhost:8000 + .option('-r, --region ', 'The AWS region to use', 'us-east-2') + .parse(process.argv) + +let privateKeyFile = '' +if (fs.existsSync(commander.key)) { + privateKeyFile = commander.key +} else { + throw new Error('Missing or invalid private key') +} + +if (!commander.binary) { + throw new Error('Missing Chromium binary: --binary') +} + +util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { + const [publicKey, componentID] = generatePublicKeyAndID(privateKeyFile) + generateManifestFile(publicKey) + generateCRXFile(commander.binary, commander.endpoint, commander.region, componentID, privateKeyFile) +}) diff --git a/scripts/uploadComponent.js b/scripts/uploadComponent.js index f3b04e04..21a1e8d7 100644 --- a/scripts/uploadComponent.js +++ b/scripts/uploadComponent.js @@ -12,7 +12,7 @@ util.installErrorHandlers() commander .option('-d, --crx-directory ', 'directory containing multiple crx files to upload') .option('-f, --crx-file ', 'crx file to upload', 'extension.crx') - .option('-t, --type ', 'component extension type', /^(ad-block-updater|https-everywhere-updater|local-data-files-updater|ethereum-remote-client|ntp-sponsored-images|ntp-super-referrer|speedreader-updater|tor-client-updater|client-model-parameters)$/i, 'ad-block-updater') + .option('-t, --type ', 'component extension type', /^(ad-block-updater|https-everywhere-updater|local-data-files-updater|ethereum-remote-client|ntp-sponsored-images|ntp-super-referrer|ntp-super-referrer-mapping-table|speedreader-updater|tor-client-updater|client-model-parameters)$/i, 'ad-block-updater') .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '')// If setup locally, use http://localhost:8000 .option('-r, --region ', 'The AWS region to use', 'us-east-2') .parse(process.argv)