From dd78f64c69a863a1be327af9e6c6dd810fd1681d Mon Sep 17 00:00:00 2001 From: Graham Tackley Date: Fri, 21 Feb 2025 16:08:37 +0000 Subject: [PATCH 1/2] refactor(ntp-si): read asset list explicitly The ntp-si packaging logic had intimate knowledge of the internal format of `photo.json` in order to determine the set of assets to be included in the crx. This PR shifts that responsibility to `ntp-si-assets`, which as of https://github.com/brave/ntp-si-assets/pull/1240 now publishes an additional file `assets.json` explicitly containing the list of files to be packaged. [example](https://mobile-data.s3.brave.com/GB/desktop/assets.json) This is a pre-req for rich NTTs, which will have a wider variety of assets. That isn't implemented server-side yet: this change is in preparation for reducing the number of places that are impacted when it is. So currently there should be no visible difference with this change: it's a pure refactor. Re #1039 --- .gitignore | 1 + lib/ntpUtil.js | 153 ++++++----------------- scripts/ntp-sponsored-images/generate.js | 2 +- 3 files changed, 41 insertions(+), 115 deletions(-) diff --git a/.gitignore b/.gitignore index f70f279b..64081b38 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ brave-lists leo-local-models # localstack .localstack/ +.vscode/ \ No newline at end of file diff --git a/lib/ntpUtil.js b/lib/ntpUtil.js index b5cdb813..6a37ff06 100644 --- a/lib/ntpUtil.js +++ b/lib/ntpUtil.js @@ -6,70 +6,7 @@ import childProcess from 'child_process' import fs from 'fs' import path from 'path' import util from '../lib/util.js' -import { Readable } from 'stream' -import { finished } from 'stream/promises' - -const jsonSchemaVersion = 1 - -/** -* @typedef {{ imageUrl: string }} Logo -* @typedef {{ logo?: Logo, imageUrl: string}} Wallpaper -* @typedef {{ logo: Logo, wallpapers: Wallpaper[]}} Campaign -* @typedef {Campaign & { campaigns?: Campaign[], campaigns2?: Campaign[], topSites?: { iconUrl }[] }} NTPAssetSchema -*/ - -const createPhotoJsonFile = (path, body) => { - fs.writeFileSync(path, body) -} - -/** - * - * @param {Campaign} campaign - * @returns {string[]} - */ -function getImageFileNameListFromCampaign (campaign) { - const fileList = new Set() - if (campaign.logo) { - fileList.add(campaign.logo.imageUrl) - } - if (campaign.wallpapers) { - campaign.wallpapers.forEach((wallpaper) => { - fileList.add(wallpaper.imageUrl) - // V2 feature - support per-wallpaper logo - if (wallpaper.logo && wallpaper.logo.imageUrl) { - fileList.add(wallpaper.logo.imageUrl) - } - }) - } - return Array.from(fileList.values()) -} - -/** - * - * @param {NTPAssetSchema} photoJsonObj - * @returns {string[]} - */ -const getImageFileNameListFrom = (photoJsonObj) => { - const fileList = new Set( - getImageFileNameListFromCampaign(photoJsonObj) - ) - if (photoJsonObj.campaigns) { - for (const campaign of photoJsonObj.campaigns) { - getImageFileNameListFromCampaign(campaign).forEach(s => fileList.add(s)) - } - } - if (photoJsonObj.campaigns2) { - for (const campaign of photoJsonObj.campaigns2) { - getImageFileNameListFromCampaign(campaign).forEach(s => fileList.add(s)) - } - } - if (photoJsonObj.topSites) { - photoJsonObj.topSites.forEach((topSiteObj) => { - fileList.add(topSiteObj.iconUrl) - }) - } - return Array.from(fileList.values()) -} +import { pipeline } from 'stream/promises' const generatePublicKeyAndID = (privateKeyFile) => { childProcess.execSync(`openssl rsa -in ${privateKeyFile} -pubout -out public.pub`) @@ -94,61 +31,49 @@ const generatePublicKeyAndID = (privateKeyFile) => { } } -const isValidSchemaVersion = (version) => { - return version === jsonSchemaVersion +const downloadFile = async (sourceUrl, dest) => { + const response = await fetch(sourceUrl) + + if (!response.ok) { + throw new Error(`download of ${sourceUrl} failed with code ${response.status}`) + } + + if (!response.body) { + throw new Error(`download of ${sourceUrl} failed with empty body`) + } + + const file = fs.createWriteStream(dest) + await pipeline(response.body, file) } -const prepareAssets = (jsonFileUrl, targetResourceDir, targetJsonFileName) => { - return new Promise(function (resolve, reject) { - let jsonFileBody = '{}' +const prepareAssets = async (jsonFileUrl, targetResourceDir) => { + const response = await fetch(jsonFileUrl) - // Download and parse jsonFileUrl. - // If it doesn't exist, create with empty object. - fetch(jsonFileUrl).then(async function (response) { - if (response.status === 200) { - jsonFileBody = await response.text() - } - let photoData = {} - try { - console.log(`Start - json file ${jsonFileUrl} parsing`) - photoData = JSON.parse(jsonFileBody) - } catch (err) { - console.error(`Invalid json file ${jsonFileUrl}`) - return reject(err) - } - console.log(`Done - json file ${jsonFileUrl} parsing`) - // Make sure the data has a schema version so that clients can opt to parse or not - let incomingSchemaVersion = photoData.schemaVersion - console.log(`Schema version: ${incomingSchemaVersion} from ${jsonFileUrl}`) - if (!incomingSchemaVersion) { - // Source has no schema version, assume and set current version. - // TODO(petemill): Don't allow this once the source is established to always - // have a schema version. - incomingSchemaVersion = jsonSchemaVersion - } else if (!isValidSchemaVersion(incomingSchemaVersion)) { - const error = `Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.` - console.error(error) - return reject(error) - } + if (!response.ok) { + throw new Error(`download of ${jsonFileUrl} failed with code ${response.status}`) + } - createPhotoJsonFile(path.join(targetResourceDir, 'photo.json'), JSON.stringify(photoData)) - - // Download image files that specified in jsonFileUrl - const imageFileNameList = getImageFileNameListFrom(photoData) - const downloadOps = imageFileNameList.map(async (imageFileName) => { - const targetImageFilePath = path.join(targetResourceDir, imageFileName) - const targetImageFileUrl = new URL(imageFileName, jsonFileUrl).href - const response = await fetch(targetImageFileUrl) - const ws = fs.createWriteStream(targetImageFilePath) - return finished(Readable.fromWeb(response.body).pipe(ws)) - .then(() => console.log(targetImageFileUrl)) - }) - await Promise.all(downloadOps) - resolve() - }).catch(error => { - throw new Error(`Error from ${jsonFileUrl}: ${error.cause}`) - }) + console.log(` Reading ${jsonFileUrl}`) + + // example file format: + // https://github.com/brave/ntp-si-assets/blob/966021c07cf1dcb58128c0b0487b8bd974f4eda8/resources/test-data/examples/assets.json + const json = await response.json() + + const allDownloads = json.assets.map(async asset => { + const targetAssetFilePath = path.join(targetResourceDir, asset.path) + const targetAssetFileUrl = new URL(asset.path, jsonFileUrl).href + + await downloadFile(targetAssetFileUrl, targetAssetFilePath) + + const hash = util.generateSHA256HashOfFile(targetAssetFilePath) + if (hash !== asset.sha256) { + throw new Error(`${targetAssetFileUrl}: hash does not match, expected ${asset.sha256} got ${hash}`) + } + + console.log(" " + targetAssetFileUrl) }) + + await Promise.all(allDownloads) } export default { diff --git a/scripts/ntp-sponsored-images/generate.js b/scripts/ntp-sponsored-images/generate.js index f03dbe70..06efd9a6 100644 --- a/scripts/ntp-sponsored-images/generate.js +++ b/scripts/ntp-sponsored-images/generate.js @@ -30,7 +30,7 @@ async function generateNTPSponsoredImages (dataUrl, targetComponents) { const targetResourceDir = path.join(rootResourceDir, regionPlatformName) mkdirp.sync(targetResourceDir) const regionPlatformPath = regionPlatformName.replace('-', '/') - const sourceJsonFileUrl = `${dataUrl}${regionPlatformPath}/photo.json` + const sourceJsonFileUrl = `${dataUrl}${regionPlatformPath}/assets.json` await ntpUtil.prepareAssets(sourceJsonFileUrl, targetResourceDir) } } From 4687e4cfb17bc17fa378cd5bbd49a2209f082494 Mon Sep 17 00:00:00 2001 From: Graham Tackley Date: Fri, 21 Feb 2025 16:25:24 +0000 Subject: [PATCH 2/2] fix lint --- lib/ntpUtil.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ntpUtil.js b/lib/ntpUtil.js index 6a37ff06..b8fd0480 100644 --- a/lib/ntpUtil.js +++ b/lib/ntpUtil.js @@ -70,7 +70,7 @@ const prepareAssets = async (jsonFileUrl, targetResourceDir) => { throw new Error(`${targetAssetFileUrl}: hash does not match, expected ${asset.sha256} got ${hash}`) } - console.log(" " + targetAssetFileUrl) + console.log(' ' + targetAssetFileUrl) }) await Promise.all(allDownloads)