-
Notifications
You must be signed in to change notification settings - Fork 35
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #97 from simonhong/super-referrer-package
Generate NTP Super Referrer(SR) component
- Loading branch information
Showing
5 changed files
with
242 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| /* This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this file, | ||
| * You can obtain one at http://mozilla.org/MPL/2.0/. */ | ||
|
|
||
| const path = require('path') | ||
| const mkdirp = require('mkdirp') | ||
| const fs = require('fs-extra') | ||
| const request = require('request') | ||
| const commander = require('commander') | ||
|
|
||
| const jsonFileName = 'data.json' | ||
| const jsonSchemaVersion = 1 | ||
|
|
||
| const createDataJsonFile = (path, body) => { | ||
| fs.writeFileSync(path, body) | ||
| } | ||
|
|
||
| const getImageFileNameListFrom = (dataJsonObj) => { | ||
| let fileList = [] | ||
| if (dataJsonObj.logo) | ||
| fileList.push(dataJsonObj.logo.imageUrl) | ||
|
|
||
| if (dataJsonObj.wallpapers) { | ||
| dataJsonObj.wallpapers.forEach((wallpaper) => { | ||
| fileList.push(wallpaper.imageUrl) | ||
| }) | ||
| } | ||
| if (dataJsonObj.topSites) { | ||
| dataJsonObj.topSites.forEach((topSiteObj) => { | ||
| fileList.push(topSiteObj.iconUrl) | ||
| }) | ||
| } | ||
| return fileList | ||
| } | ||
|
|
||
| function downloadForRegion (jsonFileUrl, targetResourceDir) { | ||
| return new Promise(function (resolve, reject) { | ||
| const jsonFilePath = path.join(targetResourceDir, jsonFileName) | ||
| let jsonFileBody = '{}' | ||
|
|
||
| // Download and parse data.json. | ||
| // If it doesn't exist, create with empty object. | ||
| request(jsonFileUrl, async function (error, response, body) { | ||
| if (error) { | ||
| console.error(`Error from ${jsonFileUrl}:`, error) | ||
| return reject(error) | ||
| } | ||
| if (response && response.statusCode === 200) { | ||
| jsonFileBody = body | ||
| } | ||
|
|
||
| const data = JSON.parse(jsonFileBody) | ||
| // Make sure the data has a schema version so that clients can opt to parse or not | ||
| const incomingSchemaVersion = data.schemaVersion | ||
| if (!incomingSchemaVersion) { | ||
| // Source has no schema version, assume and set current version. | ||
| // TODO(petemill): Don't allow this once the source is established to always | ||
| // have a schema version. | ||
| data.schemaVersion = jsonSchemaVersion | ||
| } else if (incomingSchemaVersion !== jsonSchemaVersion) { | ||
| // We don't support this file format | ||
| console.error(`Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`) | ||
| return reject(error) | ||
| } | ||
|
|
||
| createDataJsonFile(jsonFilePath, JSON.stringify(data)) | ||
|
|
||
| // Download image files that specified in data.json | ||
| const imageFileNameList = getImageFileNameListFrom(data) | ||
| const downloadOps = imageFileNameList.map((imageFileName) => new Promise(resolve => { | ||
| const targetImageFilePath = path.join(targetResourceDir, imageFileName) | ||
| const targetImageFileUrl = new URL(imageFileName, jsonFileUrl).href | ||
| request(targetImageFileUrl) | ||
| .pipe(fs.createWriteStream(targetImageFilePath)) | ||
| .on('finish', () => { | ||
| console.log(targetImageFileUrl) | ||
| resolve() | ||
| }) | ||
| })) | ||
| await Promise.all(downloadOps) | ||
| resolve() | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| async function generateNTPSuperReferrer (dataUrl, referrerName) { | ||
| const rootResourceDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources') | ||
| mkdirp.sync(rootResourceDir) | ||
|
|
||
| console.log(`Downloading for ${referrerName}...`) | ||
| const targetResourceDir = path.join(rootResourceDir, referrerName) | ||
| mkdirp.sync(targetResourceDir) | ||
| const jsonFileUrl = `${dataUrl}superreferrer/${referrerName}/${jsonFileName}` | ||
| await downloadForRegion(jsonFileUrl, targetResourceDir) | ||
| } | ||
|
|
||
| commander | ||
| .option('-d, --data-url <url>', 'url that refers to data that has ntp super referrer') | ||
| .option('-n, --super-referrer-name <name>', 'super referrer name for this component') | ||
| .parse(process.argv) | ||
|
|
||
| generateNTPSuperReferrer(commander.dataUrl, commander.superReferrerName) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| /* This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this file, | ||
| * You can obtain one at http://mozilla.org/MPL/2.0/. */ | ||
|
|
||
| const childProcess = require('child_process') | ||
| const commander = require('commander') | ||
| const fs = require('fs-extra') | ||
| const mkdirp = require('mkdirp') | ||
| const path = require('path') | ||
| const replace = require('replace-in-file') | ||
| const util = require('../lib/util') | ||
|
|
||
| const stageFiles = (superReferrerName, version, outputDir) => { | ||
| // Copy resources and manifest file to outputDir. | ||
| // Copy resource files | ||
| const resourceDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', superReferrerName, '/') | ||
| console.log('copy dir:', resourceDir, ' to:', outputDir) | ||
| fs.copySync(resourceDir, outputDir) | ||
|
|
||
| // Fix up the manifest version | ||
| const originalManifest = getOriginalManifest(superReferrerName) | ||
| const outputManifest = path.join(outputDir, 'manifest.json') | ||
| console.log('copy manifest file: ', originalManifest, ' to: ', outputManifest) | ||
| const replaceOptions = { | ||
| files: outputManifest, | ||
| from: /0\.0\.0/, | ||
| to: version | ||
| } | ||
| fs.copyFileSync(originalManifest, outputManifest) | ||
| replace.sync(replaceOptions) | ||
| } | ||
|
|
||
| const generateManifestFile = (superReferrerName, publicKey) => { | ||
| const manifestFile = getOriginalManifest(superReferrerName) | ||
| const manifestContent = { | ||
| description: 'Brave NTP Super Referrer component', | ||
| key: publicKey, | ||
| manifest_version: 2, | ||
| name: `Brave NTP Super Referrer (${superReferrerName})`, | ||
| version: '0.0.0' | ||
| } | ||
| fs.writeFileSync(manifestFile, JSON.stringify(manifestContent)) | ||
| } | ||
|
|
||
| const getOriginalManifest = (superReferrerName) => { | ||
| return path.join(path.resolve(), 'build','ntp-super-referrer', `${superReferrerName}-manifest.json`) | ||
| } | ||
|
|
||
| const generatePublicKeyAndID = (privateKeyFile) => { | ||
| childProcess.execSync(`openssl rsa -in ${privateKeyFile} -pubout -out public.pub`) | ||
| try { | ||
| // read contents of the file | ||
| const data = fs.readFileSync('public.pub', 'UTF-8'); | ||
|
|
||
| // split the contents by new line | ||
| const lines = data.split(/\r?\n/); | ||
| let pubKeyString = '' | ||
| lines.forEach((line) => { | ||
| if (!line.includes('-----')) | ||
| pubKeyString += line | ||
| }); | ||
| console.log(`publicKey: ${pubKeyString}`) | ||
| const id = util.getIDFromBase64PublicKey(pubKeyString) | ||
| console.log(`componentID: ${id}`) | ||
| return [pubKeyString, id] | ||
| } catch (err) { | ||
| console.error(err); | ||
| } | ||
| } | ||
|
|
||
| const generateCRXFile = (binary, endpoint, region, superReferrerName, componentID, privateKeyFile) => { | ||
| const originalManifest = getOriginalManifest(superReferrerName) | ||
| const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer') | ||
| const stagingDir = path.join(rootBuildDir, 'staging', superReferrerName) | ||
| const crxOutputDir = path.join(rootBuildDir, 'output') | ||
| mkdirp.sync(stagingDir) | ||
| mkdirp.sync(crxOutputDir) | ||
| util.getNextVersion(endpoint, region, componentID).then((version) => { | ||
| const crxFile = path.join(crxOutputDir, `ntp-super-referrer-${superReferrerName}.crx`) | ||
| stageFiles(superReferrerName, version, stagingDir) | ||
| util.generateCRXFile(binary, crxFile, privateKeyFile, stagingDir) | ||
| console.log(`Generated ${crxFile} with version number ${version}`) | ||
| }) | ||
| } | ||
|
|
||
| util.installErrorHandlers() | ||
|
|
||
| commander | ||
| .option('-b, --binary <binary>', 'Path to the Chromium based executable to use to generate the CRX file') | ||
| .option('-n, --super-referrer-name <name>', 'super referrer name for this component') | ||
| .option('-k, --key <file>', 'file containing private key for signing crx file') | ||
| .option('-e, --endpoint <endpoint>', 'DynamoDB endpoint to connect to', '')// If setup locally, use http://localhost:8000 | ||
| .option('-r, --region <region>', 'The AWS region to use', 'us-east-2') | ||
| .parse(process.argv) | ||
|
|
||
| let privateKeyFile = '' | ||
| if (fs.existsSync(commander.key)) { | ||
| privateKeyFile = commander.key | ||
| } else { | ||
| throw new Error('Missing or invalid private key') | ||
| } | ||
|
|
||
| if (!commander.binary) { | ||
| throw new Error('Missing Chromium binary: --binary') | ||
| } | ||
|
|
||
| util.createTableIfNotExists(commander.endpoint, commander.region).then(() => { | ||
| const [publicKey, componentID] = generatePublicKeyAndID(privateKeyFile) | ||
| generateManifestFile(commander.superReferrerName, publicKey) | ||
| generateCRXFile(commander.binary, commander.endpoint, commander.region, commander.superReferrerName, componentID, privateKeyFile) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters