From ddbf29e94fc9fdc98975a28ef0d1dae99626939f Mon Sep 17 00:00:00 2001 From: Simon Hong Date: Tue, 10 Mar 2020 17:57:53 +0900 Subject: [PATCH] Generate NTP Super Referreer(SR) component --- package.json | 5 +- scripts/generateNTPSuperReferrer.js | 102 ++++++++++++++++++ scripts/packageNTPSuperReferrerComponent.js | 111 ++++++++++++++++++++ scripts/uploadComponent.js | 2 +- 4 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 scripts/generateNTPSuperReferrer.js create mode 100644 scripts/packageNTPSuperReferrerComponent.js diff --git a/package.json b/package.json index 83e0917e..67080de0 100755 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "data-files-extension-whitelist": "npm run --prefix ./node_modules/extension-whitelist data-files", "data-files-local-data-files": "npm run data-files-tracking-protection && npm run data-files-autoplay-whitelist && npm run data-files-extension-whitelist", "generate-ntp-sponsored-images": "node scripts/generateNTPSponsoredImages.js", + "generate-ntp-super-referrer": "node scripts/generateNTPSuperReferrer.js", "lint": "standard", "import-cws-components": "node ./scripts/importCWSComponents", "package-ethereum-remote-client": "node ./scripts/packageComponent --type ethereum-remote-client", @@ -46,6 +47,7 @@ "package-ipfs-daemon": "node ./scripts/packageIpfsDaemon", "package-local-data-files": "node ./scripts/packageComponent --type local-data-files-updater", "package-ntp-sponsored-images": "node ./scripts/packageNTPSponsoredImagesComponents", + "package-ntp-super-referrer": "node scripts/packageNTPSuperReferrerComponent.js", "upload-ethereum-remote-client": "node ./scripts/uploadComponent --type ethereum-remote-client", "upload-ad-block": "node ./scripts/uploadComponent --type ad-block-updater", "upload-ad-block-raw": "node ./scripts/uploadRawAdblock", @@ -53,7 +55,8 @@ "upload-ipfs-daemon": "node ./scripts/uploadIpfsDaemon", "upload-tor-client": "node ./scripts/uploadComponent --type tor-client-updater", "upload-local-data-files": "node ./scripts/uploadComponent --type local-data-files-updater", - "upload-ntp-sponsored-images-components": "node ./scripts/uploadComponent --type ntp-sponsored-images" + "upload-ntp-sponsored-images-components": "node ./scripts/uploadComponent --type ntp-sponsored-images", + "upload-ntp-super-referrer-component": "node ./scripts/uploadComponent --type ntp-super-referrer" }, "repository": { "type": "git", diff --git a/scripts/generateNTPSuperReferrer.js b/scripts/generateNTPSuperReferrer.js new file mode 100644 index 00000000..c6e2ed61 --- /dev/null +++ b/scripts/generateNTPSuperReferrer.js @@ -0,0 +1,102 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const path = require('path') +const mkdirp = require('mkdirp') +const fs = require('fs-extra') +const request = require('request') +const commander = require('commander') + +const jsonFileName = 'data.json' +const jsonSchemaVersion = 1 + +const createPhotoJsonFile = (path, body) => { + fs.writeFileSync(path, body) +} + +const getImageFileNameListFrom = (dataJsonObj) => { + let fileList = [] + if (dataJsonObj.logo) + fileList.push(dataJsonObj.logo.imageUrl) + + if (dataJsonObj.wallpapers) { + dataJsonObj.wallpapers.forEach((wallpaper) => { + fileList.push(wallpaper.imageUrl) + }) + } + if (dataJsonObj.topSites) { + dataJsonObj.topSites.forEach((topSiteObj) => { + fileList.push(topSiteObj.iconUrl) + }) + } + return fileList +} + +function downloadForRegion (jsonFileUrl, targetResourceDir) { + return new Promise(function (resolve, reject) { + const jsonFilePath = path.join(targetResourceDir, jsonFileName) + let jsonFileBody = '{}' + + // Download and parse data.json. + // If it doesn't exist, create with empty object. + request(jsonFileUrl, async function (error, response, body) { + if (error) { + console.error(`Error from ${jsonFileUrl}:`, error) + return reject(error) + } + if (response && response.statusCode === 200) { + jsonFileBody = body + } + + const data = JSON.parse(jsonFileBody) + // Make sure the data has a schema version so that clients can opt to parse or not + const incomingSchemaVersion = data.schemaVersion + if (!incomingSchemaVersion) { + // Source has no schema version, assume and set current version. + // TODO(petemill): Don't allow this once the source is established to always + // have a schema version. + data.schemaVersion = jsonSchemaVersion + } else if (incomingSchemaVersion !== jsonSchemaVersion) { + // We don't support this file format + console.error(`Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`) + return reject(error) + } + + createPhotoJsonFile(jsonFilePath, JSON.stringify(data)) + + // Download image files that specified in data.json + const imageFileNameList = getImageFileNameListFrom(data) + const downloadOps = imageFileNameList.map((imageFileName) => new Promise(resolve => { + const targetImageFilePath = path.join(targetResourceDir, imageFileName) + const targetImageFileUrl = new URL(imageFileName, jsonFileUrl).href + request(targetImageFileUrl) + .pipe(fs.createWriteStream(targetImageFilePath)) + .on('finish', () => { + console.log(targetImageFileUrl) + resolve() + }) + })) + await Promise.all(downloadOps) + resolve() + }) + }) +} + +async function generateNTPSuperReferrer (dataUrl, referrerName) { + const rootResourceDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources') + mkdirp.sync(rootResourceDir) + + console.log(`Downloading for ${referrerName}...`) + const targetResourceDir = path.join(rootResourceDir, referrerName) + mkdirp.sync(targetResourceDir) + const jsonFileUrl = `${dataUrl}superreferrer/${referrerName}/${jsonFileName}` + await downloadForRegion(jsonFileUrl, targetResourceDir) +} + +commander + .option('-d, --data-url ', 'url that refers to data that has ntp super referrer') + .option('-n, --super-referrer-name ', 'super referrer name for this component') + .parse(process.argv) + +generateNTPSuperReferrer(commander.dataUrl, commander.superReferrerName) diff --git a/scripts/packageNTPSuperReferrerComponent.js b/scripts/packageNTPSuperReferrerComponent.js new file mode 100644 index 00000000..0f7e9967 --- /dev/null +++ b/scripts/packageNTPSuperReferrerComponent.js @@ -0,0 +1,111 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const childProcess = require('child_process') +const commander = require('commander') +const fs = require('fs-extra') +const mkdirp = require('mkdirp') +const path = require('path') +const replace = require('replace-in-file') +const util = require('../lib/util') + +const stageFiles = (superReferrerName, version, outputDir) => { + // Copy resources and manifest file to outputDir. + // Copy resource files + const resourceDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', superReferrerName, '/') + console.log('copy dir:', resourceDir, ' to:', outputDir) + fs.copySync(resourceDir, outputDir) + + // Fix up the manifest version + const originalManifest = getOriginalManifest(superReferrerName) + const outputManifest = path.join(outputDir, 'manifest.json') + console.log('copy manifest file: ', originalManifest, ' to: ', outputManifest) + const replaceOptions = { + files: outputManifest, + from: /0\.0\.0/, + to: version + } + fs.copyFileSync(originalManifest, outputManifest) + replace.sync(replaceOptions) +} + +const generateManifestFile = (superReferrerName, publicKey) => { + const manifestFile = getOriginalManifest(superReferrerName) + const manifestContent = { + description: 'Brave NTP Super Referrer component', + key: publicKey, + manifest_version: 2, + name: `Brave NTP Super Referrer (${superReferrerName})`, + version: '0.0.0' + } + fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) +} + +const getOriginalManifest = (superReferrerName) => { + return path.join(path.resolve(), 'build','ntp-super-referrer', `${superReferrerName}-manifest.json`) +} + +const generatePublicKeyAndID = (privateKeyFile) => { + childProcess.execSync(`openssl rsa -in ${privateKeyFile} -pubout -out public.pub`) + try { + // read contents of the file + const data = fs.readFileSync('public.pub', 'UTF-8'); + + // split the contents by new line + const lines = data.split(/\r?\n/); + let pubKeyString = '' + lines.forEach((line) => { + if (!line.includes('-----')) + pubKeyString += line + }); + console.log(`publicKey: ${pubKeyString}`) + const id = util.getIDFromBase64PublicKey(pubKeyString) + console.log(`componentID: ${id}`) + return [pubKeyString, id] + } catch (err) { + console.error(err); + } +} + +const generateCRXFile = (binary, endpoint, region, superReferrerName, componentID, privateKeyFile) => { + const originalManifest = getOriginalManifest(superReferrerName) + const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') + const stagingDir = path.join(rootBuildDir, 'staging', superReferrerName) + const crxOutputDir = path.join(rootBuildDir, 'output') + mkdirp.sync(stagingDir) + mkdirp.sync(crxOutputDir) + util.getNextVersion(endpoint, region, componentID).then((version) => { + const crxFile = path.join(crxOutputDir, `ntp-super-referrer-${superReferrerName}.crx`) + stageFiles(superReferrerName, version, stagingDir) + util.generateCRXFile(binary, crxFile, privateKeyFile, stagingDir) + console.log(`Generated ${crxFile} with version number ${version}`) + }) +} + +util.installErrorHandlers() + +commander + .option('-b, --binary ', 'Path to the Chromium based executable to use to generate the CRX file') + .option('-n, --super-referrer-name ', 'super referrer name for this component') + .option('-k, --key ', 'file containing private key for signing crx file') + .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '')// If setup locally, use http://localhost:8000 + .option('-r, --region ', 'The AWS region to use', 'us-east-2') + .parse(process.argv) + +let privateKeyFile = '' +if (fs.existsSync(commander.key)) { + privateKeyFile = commander.key +} else { + throw new Error('Missing or invalid private key') +} + +if (!commander.binary) { + throw new Error('Missing Chromium binary: --binary') +} + +util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { + const [publicKey, componentID] = generatePublicKeyAndID(privateKeyFile) + generateManifestFile(commander.superReferrerName, publicKey) + generateCRXFile(commander.binary, commander.endpoint, commander.region, commander.superReferrerName, componentID, privateKeyFile) +}) diff --git a/scripts/uploadComponent.js b/scripts/uploadComponent.js index 6d05d875..5c6f6ce1 100644 --- a/scripts/uploadComponent.js +++ b/scripts/uploadComponent.js @@ -12,7 +12,7 @@ util.installErrorHandlers() commander .option('-d, --crx-directory ', 'directory containing multiple crx files to upload') .option('-f, --crx-file ', 'crx file to upload', 'extension.crx') - .option('-t, --type ', 'component extension type', /^(ad-block-updater|https-everywhere-updater|local-data-files-updater|ethereum-remote-client|ntp-sponsored-images|tor-client-updater)$/i, 'ad-block-updater') + .option('-t, --type ', 'component extension type', /^(ad-block-updater|https-everywhere-updater|local-data-files-updater|ethereum-remote-client|ntp-sponsored-images|ntp-super-referrer|tor-client-updater)$/i, 'ad-block-updater') .option('-e, --endpoint ', 'DynamoDB endpoint to connect to', '')// If setup locally, use http://localhost:8000 .option('-r, --region ', 'The AWS region to use', 'us-east-2') .parse(process.argv)