diff --git a/lib/modules/datasource/api.ts b/lib/modules/datasource/api.ts index c61d696bfa18ff..8bb45960253cb1 100644 --- a/lib/modules/datasource/api.ts +++ b/lib/modules/datasource/api.ts @@ -19,6 +19,7 @@ import { GalaxyDatasource } from './galaxy'; import { GalaxyCollectionDatasource } from './galaxy-collection'; import { GitRefsDatasource } from './git-refs'; import { GitTagsDatasource } from './git-tags'; +import { GithubReleaseAttachmentsDatasource } from './github-release-attachments'; import { GithubReleasesDatasource } from './github-releases'; import { GithubTagsDatasource } from './github-tags'; import { GitlabPackagesDatasource } from './gitlab-packages'; @@ -76,6 +77,10 @@ api.set(GalaxyDatasource.id, new GalaxyDatasource()); api.set(GalaxyCollectionDatasource.id, new GalaxyCollectionDatasource()); api.set(GitRefsDatasource.id, new GitRefsDatasource()); api.set(GitTagsDatasource.id, new GitTagsDatasource()); +api.set( + GithubReleaseAttachmentsDatasource.id, + new GithubReleaseAttachmentsDatasource() +); api.set(GithubReleasesDatasource.id, new GithubReleasesDatasource()); api.set(GithubTagsDatasource.id, new GithubTagsDatasource()); api.set(GitlabPackagesDatasource.id, new GitlabPackagesDatasource()); diff --git a/lib/modules/datasource/github-releases/digest.spec.ts b/lib/modules/datasource/github-release-attachments/digest.spec.ts similarity index 80% rename from lib/modules/datasource/github-releases/digest.spec.ts rename to lib/modules/datasource/github-release-attachments/digest.spec.ts index 35fff7e25297ff..19264bc096da19 100644 --- a/lib/modules/datasource/github-releases/digest.spec.ts +++ b/lib/modules/datasource/github-release-attachments/digest.spec.ts @@ -1,17 +1,17 @@ import hasha from 'hasha'; import * as httpMock from '../../../../test/http-mock'; import type { GithubDigestFile } from '../../../util/github/types'; -import { GitHubReleaseMocker } from './test'; +import { GitHubReleaseAttachmentMocker } from './test'; -import { GithubReleasesDatasource } from '.'; +import { GithubReleaseAttachmentsDatasource } from '.'; -describe('modules/datasource/github-releases/digest', () => { +describe('modules/datasource/github-release-attachments/digest', () => { const packageName = 'some/dep'; - const releaseMock = new GitHubReleaseMocker( + const releaseMock = new GitHubReleaseAttachmentMocker( 'https://api.github.com', packageName ); - const githubReleases = new GithubReleasesDatasource(); + const githubReleaseAttachments = new GithubReleaseAttachmentsDatasource(); describe('findDigestAsset', () => { it('finds SHASUMS.txt file containing digest', async () => { @@ -21,7 +21,7 @@ describe('modules/datasource/github-releases/digest', () => { 'another-digest linux-arm64.tar.gz' ); - const digestAsset = await githubReleases.findDigestAsset( + const digestAsset = await githubReleaseAttachments.findDigestAsset( release, 'test-digest' ); @@ -40,7 +40,7 @@ describe('modules/datasource/github-releases/digest', () => { .get(`/repos/${packageName}/releases/download/v1.0.0/SHASUMS.txt`) .reply(200, ''); - const digestAsset = await githubReleases.findDigestAsset( + const digestAsset = await githubReleaseAttachments.findDigestAsset( release, 'test-digest' ); @@ -57,7 +57,7 @@ describe('modules/datasource/github-releases/digest', () => { }); const contentDigest = await hasha.async(content, { algorithm: 'sha256' }); - const digestAsset = await githubReleases.findDigestAsset( + const digestAsset = await githubReleaseAttachments.findDigestAsset( release, contentDigest ); @@ -67,7 +67,7 @@ describe('modules/datasource/github-releases/digest', () => { it('returns null when no assets available', async () => { const release = releaseMock.release('v1.0.0'); - const digestAsset = await githubReleases.findDigestAsset( + const digestAsset = await githubReleaseAttachments.findDigestAsset( release, 'test-digest' ); @@ -89,7 +89,7 @@ describe('modules/datasource/github-releases/digest', () => { 'v1.0.1', 'updated-digest asset.zip' ); - const digest = await githubReleases.mapDigestAssetToRelease( + const digest = await githubReleaseAttachments.mapDigestAssetToRelease( digestAsset, release ); @@ -106,7 +106,7 @@ describe('modules/datasource/github-releases/digest', () => { 'v1.0.1', 'updated-digest asset-1.0.1.zip' ); - const digest = await githubReleases.mapDigestAssetToRelease( + const digest = await githubReleaseAttachments.mapDigestAssetToRelease( digestAssetWithVersion, release ); @@ -118,7 +118,7 @@ describe('modules/datasource/github-releases/digest', () => { 'v1.0.1', 'moot-digest asset.tar.gz' ); - const digest = await githubReleases.mapDigestAssetToRelease( + const digest = await githubReleaseAttachments.mapDigestAssetToRelease( digestAsset, release ); @@ -127,7 +127,7 @@ describe('modules/datasource/github-releases/digest', () => { it('returns null when digest file not found', async () => { const release = releaseMock.release('v1.0.1'); - const digest = await githubReleases.mapDigestAssetToRelease( + const digest = await githubReleaseAttachments.mapDigestAssetToRelease( digestAsset, release ); @@ -151,7 +151,7 @@ describe('modules/datasource/github-releases/digest', () => { algorithm: 'sha256', }); - const digest = await githubReleases.mapDigestAssetToRelease( + const digest = await githubReleaseAttachments.mapDigestAssetToRelease( digestAsset, release ); @@ -160,7 +160,7 @@ describe('modules/datasource/github-releases/digest', () => { it('returns null when not found', async () => { const release = releaseMock.release('v1.0.1'); - const digest = await githubReleases.mapDigestAssetToRelease( + const digest = await githubReleaseAttachments.mapDigestAssetToRelease( digestAsset, release ); diff --git a/lib/modules/datasource/github-release-attachments/index.spec.ts b/lib/modules/datasource/github-release-attachments/index.spec.ts new file mode 100644 index 00000000000000..576bf7a004ff79 --- /dev/null +++ b/lib/modules/datasource/github-release-attachments/index.spec.ts @@ -0,0 +1,154 @@ +import { getDigest, getPkgReleases } from '..'; +import { mocked } from '../../../../test/util'; +import * as githubGraphql from '../../../util/github/graphql'; +import * as _hostRules from '../../../util/host-rules'; +import { GitHubReleaseAttachmentMocker } from './test'; +import { GithubReleaseAttachmentsDatasource } from '.'; + +jest.mock('../../../util/host-rules'); +const hostRules = mocked(_hostRules); + +const githubApiHost = 'https://api.github.com'; + +describe('modules/datasource/github-release-attachments/index', () => { + beforeEach(() => { + hostRules.hosts.mockReturnValue([]); + hostRules.find.mockReturnValue({ + token: 'some-token', + }); + }); + + describe('getReleases', () => { + it('returns releases', async () => { + jest.spyOn(githubGraphql, 'queryReleases').mockResolvedValueOnce([ + { + id: 1, + url: 'https://example.com', + name: 'some/dep2', + description: 'some description', + version: 'a', + releaseTimestamp: '2020-03-09T13:00:00Z', + }, + { + id: 2, + url: 'https://example.com', + name: 'some/dep2', + description: 'some description', + version: 'v', + releaseTimestamp: '2020-03-09T12:00:00Z', + }, + { + id: 3, + url: 'https://example.com', + name: 'some/dep2', + description: 'some description', + version: '1.0.0', + releaseTimestamp: '2020-03-09T11:00:00Z', + }, + { + id: 4, + url: 'https://example.com', + name: 'some/dep2', + description: 'some description', + version: 'v1.1.0', + releaseTimestamp: '2020-03-09T10:00:00Z', + }, + { + id: 5, + url: 'https://example.com', + name: 'some/dep2', + description: 'some description', + version: '2.0.0', + releaseTimestamp: '2020-04-09T10:00:00Z', + isStable: false, + }, + ]); + + const res = await getPkgReleases({ + datasource: GithubReleaseAttachmentsDatasource.id, + packageName: 'some/dep', + }); + + expect(res).toMatchObject({ + registryUrl: 'https://github.com', + releases: [ + { releaseTimestamp: '2020-03-09T11:00:00.000Z', version: '1.0.0' }, + { version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00.000Z' }, + { + version: '2.0.0', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + isStable: false, + }, + ], + sourceUrl: 'https://github.com/some/dep', + }); + }); + }); + + describe('getDigest', () => { + const packageName = 'some/dep'; + const currentValue = 'v1.0.0'; + const currentDigest = 'v1.0.0-digest'; + + const releaseMock = new GitHubReleaseAttachmentMocker( + githubApiHost, + packageName + ); + + it('requires currentDigest', async () => { + const digest = await getDigest( + { datasource: GithubReleaseAttachmentsDatasource.id, packageName }, + currentValue + ); + expect(digest).toBeNull(); + }); + + it('defaults to currentDigest when currentVersion is missing', async () => { + const digest = await getDigest( + { + datasource: GithubReleaseAttachmentsDatasource.id, + packageName, + currentDigest, + }, + currentValue + ); + expect(digest).toEqual(currentDigest); + }); + + it('returns updated digest in new release', async () => { + releaseMock.withDigestFileAsset( + currentValue, + `${currentDigest} asset.zip` + ); + const nextValue = 'v1.0.1'; + const nextDigest = 'updated-digest'; + releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`); + const digest = await getDigest( + { + datasource: GithubReleaseAttachmentsDatasource.id, + packageName, + currentValue, + currentDigest, + }, + nextValue + ); + expect(digest).toEqual(nextDigest); + }); + + // This is awkward, but I found returning `null` in this case to not produce an update + // I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision. + it('ignores failures verifying currentDigest', async () => { + releaseMock.release(currentValue); + const digest = await getDigest( + { + datasource: GithubReleaseAttachmentsDatasource.id, + packageName, + currentValue, + currentDigest, + }, + currentValue + ); + expect(digest).toEqual(currentDigest); + }); + }); +}); diff --git a/lib/modules/datasource/github-release-attachments/index.ts b/lib/modules/datasource/github-release-attachments/index.ts new file mode 100644 index 00000000000000..02516713ee424a --- /dev/null +++ b/lib/modules/datasource/github-release-attachments/index.ts @@ -0,0 +1,250 @@ +import is from '@sindresorhus/is'; +import hasha from 'hasha'; +import { logger } from '../../../logger'; +import { cache } from '../../../util/cache/package/decorator'; +import { queryReleases } from '../../../util/github/graphql'; +import type { + GithubDigestFile, + GithubRestAsset, + GithubRestRelease, +} from '../../../util/github/types'; +import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url'; +import { GithubHttp } from '../../../util/http/github'; +import { newlineRegex, regEx } from '../../../util/regex'; +import { Datasource } from '../datasource'; +import type { + DigestConfig, + GetReleasesConfig, + Release, + ReleaseResult, +} from '../types'; + +export const cacheNamespace = 'datasource-github-releases'; + +function inferHashAlg(digest: string): string { + switch (digest.length) { + case 64: + return 'sha256'; + default: + case 96: + return 'sha512'; + } +} + +export class GithubReleaseAttachmentsDatasource extends Datasource { + static readonly id = 'github-release-attachments'; + + override readonly defaultRegistryUrls = ['https://github.com']; + + override http: GithubHttp; + + constructor() { + super(GithubReleaseAttachmentsDatasource.id); + this.http = new GithubHttp(GithubReleaseAttachmentsDatasource.id); + } + + @cache({ + ttlMinutes: 1440, + namespace: 'datasource-github-releases', + key: (release: GithubRestRelease, digest: string) => + `${release.html_url}:${digest}`, + }) + async findDigestFile( + release: GithubRestRelease, + digest: string + ): Promise { + const smallAssets = release.assets.filter( + (a: GithubRestAsset) => a.size < 5 * 1024 + ); + for (const asset of smallAssets) { + const res = await this.http.get(asset.browser_download_url); + for (const line of res.body.split(newlineRegex)) { + const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2); + if (lineDigest === digest) { + return { + assetName: asset.name, + digestedFileName: lineFilename, + currentVersion: release.tag_name, + currentDigest: lineDigest, + }; + } + } + } + return null; + } + + @cache({ + ttlMinutes: 1440, + namespace: 'datasource-github-releases', + key: (asset: GithubRestAsset, algorithm: string) => + `${asset.browser_download_url}:${algorithm}:assetDigest`, + }) + async downloadAndDigest( + asset: GithubRestAsset, + algorithm: string + ): Promise { + const res = this.http.stream(asset.browser_download_url); + const digest = await hasha.fromStream(res, { algorithm }); + return digest; + } + + async findAssetWithDigest( + release: GithubRestRelease, + digest: string + ): Promise { + const algorithm = inferHashAlg(digest); + const assetsBySize = release.assets.sort( + (a: GithubRestAsset, b: GithubRestAsset) => { + if (a.size < b.size) { + return -1; + } + if (a.size > b.size) { + return 1; + } + return 0; + } + ); + + for (const asset of assetsBySize) { + const assetDigest = await this.downloadAndDigest(asset, algorithm); + if (assetDigest === digest) { + return { + assetName: asset.name, + currentVersion: release.tag_name, + currentDigest: assetDigest, + }; + } + } + return null; + } + + /** Identify the asset associated with a known digest. */ + async findDigestAsset( + release: GithubRestRelease, + digest: string + ): Promise { + const digestFile = await this.findDigestFile(release, digest); + if (digestFile) { + return digestFile; + } + + const asset = await this.findAssetWithDigest(release, digest); + return asset; + } + + /** Given a digest asset, find the equivalent digest in a different release. */ + async mapDigestAssetToRelease( + digestAsset: GithubDigestFile, + release: GithubRestRelease + ): Promise { + const current = digestAsset.currentVersion.replace(regEx(/^v/), ''); + const next = release.tag_name.replace(regEx(/^v/), ''); + const releaseChecksumAssetName = digestAsset.assetName.replace( + current, + next + ); + const releaseAsset = release.assets.find( + (a: GithubRestAsset) => a.name === releaseChecksumAssetName + ); + if (!releaseAsset) { + return null; + } + if (digestAsset.digestedFileName) { + const releaseFilename = digestAsset.digestedFileName.replace( + current, + next + ); + const res = await this.http.get(releaseAsset.browser_download_url); + for (const line of res.body.split(newlineRegex)) { + const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2); + if (lineFn === releaseFilename) { + return lineDigest; + } + } + } else { + const algorithm = inferHashAlg(digestAsset.currentDigest); + const newDigest = await this.downloadAndDigest(releaseAsset, algorithm); + return newDigest; + } + return null; + } + + /** + * Attempts to resolve the digest for the specified package. + * + * The `newValue` supplied here should be a valid tag for the GitHub release. + * Requires `currentValue` and `currentDigest`. + * + * There may be many assets attached to the release. This function will: + * - Identify the asset pinned by `currentDigest` in the `currentValue` release + * - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`). + * - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512. + * - Map the hashed asset to `newValue` and return the updated digest as a string + */ + override async getDigest( + { + packageName: repo, + currentValue, + currentDigest, + registryUrl, + }: DigestConfig, + newValue: string + ): Promise { + logger.debug( + { repo, currentValue, currentDigest, registryUrl, newValue }, + 'getDigest' + ); + if (!currentDigest) { + return null; + } + if (!currentValue) { + return currentDigest; + } + + const apiBaseUrl = getApiBaseUrl(registryUrl); + const { body: currentRelease } = await this.http.getJson( + `${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}` + ); + const digestAsset = await this.findDigestAsset( + currentRelease, + currentDigest + ); + let newDigest: string | null; + if (!digestAsset || newValue === currentValue) { + newDigest = currentDigest; + } else { + const { body: newRelease } = await this.http.getJson( + `${apiBaseUrl}repos/${repo}/releases/tags/${newValue}` + ); + newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease); + } + return newDigest; + } + + /** + * This function can be used to fetch releases with a customisable versioning + * (e.g. semver) and with releases. + * + * This function will: + * - Fetch all releases + * - Sanitize the versions if desired (e.g. strip out leading 'v') + * - Return a dependency object containing sourceUrl string and releases array + */ + async getReleases(config: GetReleasesConfig): Promise { + const releasesResult = await queryReleases(config, this.http); + const releases = releasesResult.map((item) => { + const { version, releaseTimestamp, isStable } = item; + const result: Release = { + version, + gitRef: version, + releaseTimestamp, + }; + if (is.boolean(isStable)) { + result.isStable = isStable; + } + return result; + }); + const sourceUrl = getSourceUrl(config.packageName, config.registryUrl); + return { sourceUrl, releases }; + } +} diff --git a/lib/modules/datasource/github-releases/test/index.ts b/lib/modules/datasource/github-release-attachments/test/index.ts similarity index 97% rename from lib/modules/datasource/github-releases/test/index.ts rename to lib/modules/datasource/github-release-attachments/test/index.ts index e7dfcc82c91828..84f6f3086c1e28 100644 --- a/lib/modules/datasource/github-releases/test/index.ts +++ b/lib/modules/datasource/github-release-attachments/test/index.ts @@ -2,7 +2,7 @@ import * as httpMock from '../../../../../test/http-mock'; import { partial } from '../../../../../test/util'; import type { GithubRestRelease } from '../../../../util/github/types'; -export class GitHubReleaseMocker { +export class GitHubReleaseAttachmentMocker { constructor( private readonly githubApiHost: string, private readonly packageName: string diff --git a/lib/modules/datasource/github-releases/index.spec.ts b/lib/modules/datasource/github-releases/index.spec.ts index f90efc018f4bfa..42f485fa589158 100644 --- a/lib/modules/datasource/github-releases/index.spec.ts +++ b/lib/modules/datasource/github-releases/index.spec.ts @@ -1,17 +1,14 @@ import { getDigest, getPkgReleases } from '..'; +import { mocked } from '../../../../test/util'; import * as githubGraphql from '../../../util/github/graphql'; import * as _hostRules from '../../../util/host-rules'; -import { GitHubReleaseMocker } from './test'; import { GithubReleasesDatasource } from '.'; jest.mock('../../../util/host-rules'); -const hostRules: any = _hostRules; - -const githubApiHost = 'https://api.github.com'; +const hostRules = mocked(_hostRules); describe('modules/datasource/github-releases/index', () => { beforeEach(() => { - jest.resetAllMocks(); hostRules.hosts.mockReturnValue([]); hostRules.find.mockReturnValue({ token: 'some-token', @@ -88,38 +85,48 @@ describe('modules/datasource/github-releases/index', () => { describe('getDigest', () => { const packageName = 'some/dep'; const currentValue = 'v1.0.0'; - const currentDigest = 'v1.0.0-digest'; - - const releaseMock = new GitHubReleaseMocker(githubApiHost, packageName); + const currentDigest = 'sha-of-v1'; + const newValue = 'v15.0.0'; + const newDigest = 'sha-of-v15'; - it('requires currentDigest', async () => { - const digest = await getDigest( - { datasource: GithubReleasesDatasource.id, packageName }, - currentValue - ); - expect(digest).toBeNull(); + beforeEach(() => { + jest.spyOn(githubGraphql, 'queryTags').mockResolvedValueOnce([ + { + version: 'v1.0.0', + gitRef: 'v1.0.0', + releaseTimestamp: '2021-01-01', + hash: 'sha-of-v1', + }, + { + version: 'v15.0.0', + gitRef: 'v15.0.0', + releaseTimestamp: '2022-10-01', + hash: 'sha-of-v15', + }, + ]); }); - it('defaults to currentDigest when currentVersion is missing', async () => { + it('should be independent of the current digest', async () => { const digest = await getDigest( { datasource: GithubReleasesDatasource.id, packageName, - currentDigest, + currentValue, }, - currentValue + newValue ); - expect(digest).toEqual(currentDigest); + expect(digest).toBe(newDigest); }); - it('returns updated digest in new release', async () => { - releaseMock.withDigestFileAsset( - currentValue, - `${currentDigest} asset.zip` + it('should be independent of the current value', async () => { + const digest = await getDigest( + { datasource: GithubReleasesDatasource.id, packageName }, + newValue ); - const nextValue = 'v1.0.1'; - const nextDigest = 'updated-digest'; - releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`); + expect(digest).toBe(newDigest); + }); + + it('returns updated digest in new release', async () => { const digest = await getDigest( { datasource: GithubReleasesDatasource.id, @@ -127,15 +134,12 @@ describe('modules/datasource/github-releases/index', () => { currentValue, currentDigest, }, - nextValue + newValue ); - expect(digest).toEqual(nextDigest); + expect(digest).toEqual(newDigest); }); - // This is awkward, but I found returning `null` in this case to not produce an update - // I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision. - it('ignores failures verifying currentDigest', async () => { - releaseMock.release(currentValue); + it('returns null if the new value/tag does not exist', async () => { const digest = await getDigest( { datasource: GithubReleasesDatasource.id, @@ -143,9 +147,9 @@ describe('modules/datasource/github-releases/index', () => { currentValue, currentDigest, }, - currentValue + 'unknown-tag' ); - expect(digest).toEqual(currentDigest); + expect(digest).toBeNull(); }); }); }); diff --git a/lib/modules/datasource/github-releases/index.ts b/lib/modules/datasource/github-releases/index.ts index 346fe27e0adca0..11714a8593be37 100644 --- a/lib/modules/datasource/github-releases/index.ts +++ b/lib/modules/datasource/github-releases/index.ts @@ -1,17 +1,9 @@ -// TODO: types (#7154) import is from '@sindresorhus/is'; -import hasha from 'hasha'; import { logger } from '../../../logger'; -import { cache } from '../../../util/cache/package/decorator'; import { queryReleases } from '../../../util/github/graphql'; -import type { - GithubDigestFile, - GithubRestAsset, - GithubRestRelease, -} from '../../../util/github/types'; -import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url'; +import { findCommitOfTag } from '../../../util/github/tags'; +import { getSourceUrl } from '../../../util/github/url'; import { GithubHttp } from '../../../util/http/github'; -import { newlineRegex, regEx } from '../../../util/regex'; import { Datasource } from '../datasource'; import type { DigestConfig, @@ -22,16 +14,6 @@ import type { export const cacheNamespace = 'datasource-github-releases'; -function inferHashAlg(digest: string): string { - switch (digest.length) { - case 64: - return 'sha256'; - default: - case 96: - return 'sha512'; - } -} - export class GithubReleasesDatasource extends Datasource { static readonly id = 'github-releases'; @@ -44,145 +26,17 @@ export class GithubReleasesDatasource extends Datasource { this.http = new GithubHttp(GithubReleasesDatasource.id); } - @cache({ - ttlMinutes: 1440, - namespace: 'datasource-github-releases', - key: (release: GithubRestRelease, digest: string) => - `${release.html_url}:${digest}`, - }) - async findDigestFile( - release: GithubRestRelease, - digest: string - ): Promise { - const smallAssets = release.assets.filter( - (a: GithubRestAsset) => a.size < 5 * 1024 - ); - for (const asset of smallAssets) { - const res = await this.http.get(asset.browser_download_url); - for (const line of res.body.split(newlineRegex)) { - const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2); - if (lineDigest === digest) { - return { - assetName: asset.name, - digestedFileName: lineFilename, - currentVersion: release.tag_name, - currentDigest: lineDigest, - }; - } - } - } - return null; - } - - @cache({ - ttlMinutes: 1440, - namespace: 'datasource-github-releases', - key: (asset: GithubRestAsset, algorithm: string) => - `${asset.browser_download_url}:${algorithm}:assetDigest`, - }) - async downloadAndDigest( - asset: GithubRestAsset, - algorithm: string - ): Promise { - const res = this.http.stream(asset.browser_download_url); - const digest = await hasha.fromStream(res, { algorithm }); - return digest; - } - - async findAssetWithDigest( - release: GithubRestRelease, - digest: string - ): Promise { - const algorithm = inferHashAlg(digest); - const assetsBySize = release.assets.sort( - (a: GithubRestAsset, b: GithubRestAsset) => { - if (a.size < b.size) { - return -1; - } - if (a.size > b.size) { - return 1; - } - return 0; - } - ); - - for (const asset of assetsBySize) { - const assetDigest = await this.downloadAndDigest(asset, algorithm); - if (assetDigest === digest) { - return { - assetName: asset.name, - currentVersion: release.tag_name, - currentDigest: assetDigest, - }; - } - } - return null; - } - - /** Identify the asset associated with a known digest. */ - async findDigestAsset( - release: GithubRestRelease, - digest: string - ): Promise { - const digestFile = await this.findDigestFile(release, digest); - if (digestFile) { - return digestFile; - } - - const asset = await this.findAssetWithDigest(release, digest); - return asset; - } - - /** Given a digest asset, find the equivalent digest in a different release. */ - async mapDigestAssetToRelease( - digestAsset: GithubDigestFile, - release: GithubRestRelease - ): Promise { - const current = digestAsset.currentVersion.replace(regEx(/^v/), ''); - const next = release.tag_name.replace(regEx(/^v/), ''); - const releaseChecksumAssetName = digestAsset.assetName.replace( - current, - next - ); - const releaseAsset = release.assets.find( - (a: GithubRestAsset) => a.name === releaseChecksumAssetName - ); - if (!releaseAsset) { - return null; - } - if (digestAsset.digestedFileName) { - const releaseFilename = digestAsset.digestedFileName.replace( - current, - next - ); - const res = await this.http.get(releaseAsset.browser_download_url); - for (const line of res.body.split(newlineRegex)) { - const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2); - if (lineFn === releaseFilename) { - return lineDigest; - } - } - } else { - const algorithm = inferHashAlg(digestAsset.currentDigest); - const newDigest = await this.downloadAndDigest(releaseAsset, algorithm); - return newDigest; - } - return null; - } - /** - * github.getDigest + * Attempts to resolve the digest for the specified package. * - * The `newValue` supplied here should be a valid tag for the GitHub release. - * Requires `currentValue` and `currentDigest`. + * The `newValue` supplied here should be a valid tag for the GitHub release. The digest + * of a GitHub release will be the underlying SHA of the release tag. * - * There may be many assets attached to the release. This function will: - * - Identify the asset pinned by `currentDigest` in the `currentValue` release - * - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`). - * - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512. - * - Map the hashed asset to `newValue` and return the updated digest as a string + * Some managers like Bazel will deal with individual artifacts from releases and handle + * the artifact checksum computation separately. This data-source does not know about + * specific artifacts being used, as that could vary per manager */ - override async getDigest( + override getDigest( { packageName: repo, currentValue, @@ -195,37 +49,13 @@ export class GithubReleasesDatasource extends Datasource { { repo, currentValue, currentDigest, registryUrl, newValue }, 'getDigest' ); - if (!currentDigest) { - return null; - } - if (!currentValue) { - return currentDigest; - } - const apiBaseUrl = getApiBaseUrl(registryUrl); - const { body: currentRelease } = await this.http.getJson( - `${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}` - ); - const digestAsset = await this.findDigestAsset( - currentRelease, - currentDigest - ); - let newDigest: string | null; - if (!digestAsset || newValue === currentValue) { - newDigest = currentDigest; - } else { - const { body: newRelease } = await this.http.getJson( - `${apiBaseUrl}repos/${repo}/releases/tags/${newValue}` - ); - newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease); - } - return newDigest; + return findCommitOfTag(registryUrl, repo, newValue, this.http); } /** - * github.getReleases - * - * This function can be used to fetch releases with a customisable versioning (e.g. semver) and with releases. + * This function can be used to fetch releases with a customizable versioning + * (e.g. semver) and with releases. * * This function will: * - Fetch all releases diff --git a/lib/modules/datasource/github-tags/index.ts b/lib/modules/datasource/github-tags/index.ts index 09d72813011716..f5e32f4f959403 100644 --- a/lib/modules/datasource/github-tags/index.ts +++ b/lib/modules/datasource/github-tags/index.ts @@ -2,6 +2,7 @@ import is from '@sindresorhus/is'; import { logger } from '../../../logger'; import { queryReleases, queryTags } from '../../../util/github/graphql'; import type { GithubReleaseItem } from '../../../util/github/graphql/types'; +import { findCommitOfTag } from '../../../util/github/tags'; import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url'; import { GithubHttp } from '../../../util/http/github'; import { Datasource } from '../datasource'; @@ -24,42 +25,6 @@ export class GithubTagsDatasource extends Datasource { this.http = new GithubHttp(GithubTagsDatasource.id); } - async getTagCommit( - registryUrl: string | undefined, - packageName: string, - tag: string - ): Promise { - logger.trace(`github-tags.getTagCommit(${packageName}, ${tag})`); - try { - const tags = await queryTags({ packageName, registryUrl }, this.http); - // istanbul ignore if - if (!tags.length) { - logger.debug( - `github-tags.getTagCommit(): No tags found for ${packageName}` - ); - } - const tagItem = tags.find(({ version }) => version === tag); - if (tagItem) { - if (tagItem.hash) { - return tagItem.hash; - } - logger.debug( - `github-tags.getTagCommit(): Tag ${tag} has no hash for ${packageName}` - ); - } else { - logger.debug( - `github-tags.getTagCommit(): Tag ${tag} not found for ${packageName}` - ); - } - } catch (err) { - logger.debug( - { githubRepo: packageName, err }, - 'Error getting tag commit from GitHub repo' - ); - } - return null; - } - async getCommit( registryUrl: string | undefined, githubRepo: string @@ -91,7 +56,7 @@ export class GithubTagsDatasource extends Datasource { newValue?: string ): Promise { return newValue - ? this.getTagCommit(registryUrl, repo!, newValue) + ? findCommitOfTag(registryUrl, repo!, newValue, this.http) : this.getCommit(registryUrl, repo!); } diff --git a/lib/util/github/tags.spec.ts b/lib/util/github/tags.spec.ts new file mode 100644 index 00000000000000..9747b8acf12846 --- /dev/null +++ b/lib/util/github/tags.spec.ts @@ -0,0 +1,78 @@ +import { GithubHttp } from '../http/github'; +import * as githubGraphql from './graphql'; +import { findCommitOfTag } from './tags'; + +describe('util/github/tags', () => { + describe('findCommitOfTag', () => { + const http = new GithubHttp(); + const queryTagsSpy = jest.spyOn(githubGraphql, 'queryTags'); + + it('should be able to find the hash of a Git tag', async () => { + queryTagsSpy.mockResolvedValueOnce([ + { + version: 'v1.0.0', + gitRef: 'v1.0.0', + releaseTimestamp: '2021-01-01', + hash: '123', + }, + { + version: 'v2.0.0', + gitRef: 'v2.0.0', + releaseTimestamp: '2022-01-01', + hash: 'abc', + }, + ]); + + const commit = await findCommitOfTag( + undefined, + 'some-org/repo', + 'v2.0.0', + http + ); + expect(commit).toBe('abc'); + }); + + it('should support passing a custom registry URL', async () => { + queryTagsSpy.mockResolvedValueOnce([]); + + const commit = await findCommitOfTag( + 'https://my-enterprise-github.dev', + 'some-org/repo', + 'v2.0.0', + http + ); + expect(commit).toBeNull(); + expect(githubGraphql.queryTags).toHaveBeenCalledWith( + { + packageName: 'some-org/repo', + registryUrl: 'https://my-enterprise-github.dev', + }, + http + ); + }); + + it('should return `null` if the tag does not exist', async () => { + queryTagsSpy.mockResolvedValueOnce([]); + + const commit = await findCommitOfTag( + undefined, + 'some-org/repo', + 'v2.0.0', + http + ); + expect(commit).toBeNull(); + }); + + it('should gracefully return `null` if tags cannot be queried', async () => { + queryTagsSpy.mockRejectedValue(new Error('some error')); + + const commit = await findCommitOfTag( + undefined, + 'some-org/repo', + 'v2.0.0', + http + ); + expect(commit).toBeNull(); + }); + }); +}); diff --git a/lib/util/github/tags.ts b/lib/util/github/tags.ts new file mode 100644 index 00000000000000..51101958af8413 --- /dev/null +++ b/lib/util/github/tags.ts @@ -0,0 +1,39 @@ +import { logger } from '../../logger'; +import type { GithubHttp } from '../http/github'; +import { queryTags } from './graphql'; + +export async function findCommitOfTag( + registryUrl: string | undefined, + packageName: string, + tag: string, + http: GithubHttp +): Promise { + logger.trace(`github/tags.findCommitOfTag(${packageName}, ${tag})`); + try { + const tags = await queryTags({ packageName, registryUrl }, http); + if (!tags.length) { + logger.debug( + `github/tags.findCommitOfTag(): No tags found for ${packageName}` + ); + } + const tagItem = tags.find(({ version }) => version === tag); + if (tagItem) { + if (tagItem.hash) { + return tagItem.hash; + } + logger.debug( + `github/tags.findCommitOfTag: Tag ${tag} has no hash for ${packageName}` + ); + } else { + logger.debug( + `github/tags.findCommitOfTag: Tag ${tag} not found for ${packageName}` + ); + } + } catch (err) { + logger.debug( + { githubRepo: packageName, err }, + 'Error getting tag commit from GitHub repo' + ); + } + return null; +}