From d98db19dfbc0925b6e20cfcd3bc1ace9ff5ea50e Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 20 Jun 2023 22:11:17 +0300 Subject: [PATCH 1/3] refactor(go): Change iteration over major versions --- .../datasource/go/releases-goproxy.spec.ts | 53 ++++++++-- lib/modules/datasource/go/releases-goproxy.ts | 97 ++++++++++--------- 2 files changed, 94 insertions(+), 56 deletions(-) diff --git a/lib/modules/datasource/go/releases-goproxy.spec.ts b/lib/modules/datasource/go/releases-goproxy.spec.ts index 55250e597fadb4..06feefb8a766ab 100644 --- a/lib/modules/datasource/go/releases-goproxy.spec.ts +++ b/lib/modules/datasource/go/releases-goproxy.spec.ts @@ -1,3 +1,4 @@ +import { codeBlock } from 'common-tags'; import { Fixtures } from '../../../../test/fixtures'; import * as httpMock from '../../../../test/http-mock'; import { GithubReleasesDatasource } from '../github-releases'; @@ -340,7 +341,10 @@ describe('modules/datasource/go/releases-goproxy', () => { .get('/@v/list') .reply( 200, - ['v1.0.0 2018-08-13T15:31:12Z', 'v1.0.1', ' \n'].join('\n') + codeBlock` + v1.0.0 2018-08-13T15:31:12Z + v1.0.1 + ` ) .get('/@v/v1.0.1.info') .reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' }) @@ -366,13 +370,21 @@ describe('modules/datasource/go/releases-goproxy', () => { httpMock .scope(`${baseUrl}/github.com/google/btree`) .get('/@v/list') - .reply(200, 'v1.0.0\nv1.0.1\n') + .reply( + 200, + codeBlock` + v1.0.0 + v1.0.1 + ` + ) .get('/@v/v1.0.0.info') .replyWithError('unknown') .get('/@v/v1.0.1.info') .reply(410) .get('/v2/@v/list') - .reply(200); + .reply(200) + .get('/v3/@v/list') + .reply(404); const res = await datasource.getReleases({ packageName: 'github.com/google/btree', @@ -395,7 +407,13 @@ describe('modules/datasource/go/releases-goproxy', () => { httpMock .scope(`${baseUrl}/github.com/google/btree`) .get('/@v/list') - .reply(200, 'v1.0.0\nv1.0.1\n') + .reply( + 200, + codeBlock` + v1.0.0 + v1.0.1 + ` + ) .get('/@v/v1.0.0.info') .reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' }) .get('/@v/v1.0.1.info') @@ -436,7 +454,13 @@ describe('modules/datasource/go/releases-goproxy', () => { httpMock .scope(`${baseUrl}/github.com/google/btree`) .get('/@v/list') - .reply(200, 'v1.0.0\nv1.0.1\n') + .reply( + 200, + codeBlock` + v1.0.0 + v1.0.1 + ` + ) .get('/@v/v1.0.0.info') .reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' }) .get('/@v/v1.0.1.info') @@ -580,13 +604,24 @@ describe('modules/datasource/go/releases-goproxy', () => { httpMock .scope(`${baseUrl}/github.com/google/btree`) .get('/@v/list') - .reply(200, 'v1.0.0\nv1.0.1\n') + .reply( + 200, + codeBlock` + v1.0.0 + v1.0.1 + ` + ) .get('/@v/v1.0.0.info') .reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' }) .get('/@v/v1.0.1.info') .reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' }) .get('/v2/@v/list') - .reply(200, 'v2.0.0\n') + .reply( + 200, + codeBlock` + v2.0.0 + ` + ) .get('/v2/@v/v2.0.0.info') .reply(200, { Version: 'v2.0.0', Time: '2020-10-16T16:15:28Z' }) .get('/v3/@v/list') @@ -679,7 +714,9 @@ describe('modules/datasource/go/releases-goproxy', () => { httpMock .scope(`${baseUrl}/github.com/google/btree`) .get('/@v/list') - .reply(200); + .reply(200) + .get('/v2/@v/list') + .reply(404); const res = await datasource.getReleases({ packageName: 'github.com/google/btree', diff --git a/lib/modules/datasource/go/releases-goproxy.ts b/lib/modules/datasource/go/releases-goproxy.ts index 7d00bfaa18eb45..140a06cbcf30ae 100644 --- a/lib/modules/datasource/go/releases-goproxy.ts +++ b/lib/modules/datasource/go/releases-goproxy.ts @@ -3,6 +3,7 @@ import { DateTime } from 'luxon'; import moo from 'moo'; import { logger } from '../../../logger'; import { cache } from '../../../util/cache/package/decorator'; +import { HttpError } from '../../../util/http'; import * as p from '../../../util/promises'; import { newlineRegex, regEx } from '../../../util/regex'; import { Datasource } from '../datasource'; @@ -14,7 +15,7 @@ import type { GoproxyItem, VersionInfo } from './types'; const parsedGoproxy: Record = {}; -const modRegex = regEx(`^(?.*?)(?:[./]v(?\\d+))?$`); +const modRegex = regEx(/^(?.*?)(?:[./]v(?\d+))?$/); export class GoProxyDatasource extends Datasource { static readonly id = 'go-proxy'; @@ -47,12 +48,6 @@ export class GoProxyDatasource extends Datasource { return result; } - const isGopkgin = packageName.startsWith('gopkg.in/'); - const majorSuffixSeparator = isGopkgin ? '.' : '/'; - const modParts = packageName.match(modRegex); - const baseMod = modParts?.groups?.baseMod ?? packageName; - const currentMajor = parseInt(modParts?.groups?.majorVersion ?? '0'); - for (const { url, fallback } of proxyList) { try { if (url === 'off') { @@ -62,43 +57,20 @@ export class GoProxyDatasource extends Datasource { break; } - const releases: Release[] = []; - for (let i = 0; ; i++) { - try { - const major = currentMajor + i; - let mod: string; - if (major < 2 && !isGopkgin) { - mod = baseMod; - i++; // v0 and v1 are the same module - } else { - mod = `${baseMod}${majorSuffixSeparator}v${major}`; - } - const result = await this.getVersionsWithInfo(url, mod); - if (result.length === 0) { - break; - } - releases.push(...result); - } catch (err) { - const statusCode = err?.response?.statusCode; - if (i > 0 && statusCode === 404) { - break; - } - throw err; - } - } - - if (releases.length) { + const res = await this.getVersionsWithInfo(url, packageName); + if (res.releases.length) { + result = res; try { const datasource = await BaseGoDatasource.getDatasource( packageName ); - const sourceUrl = getSourceUrl(datasource) ?? null; - result = { releases, sourceUrl }; + const sourceUrl = getSourceUrl(datasource); + if (sourceUrl) { + result.sourceUrl = sourceUrl; + } } catch (err) { logger.trace({ err }, `Can't get datasource for ${packageName}`); - result = { releases }; } - break; } } catch (err) { @@ -263,23 +235,52 @@ export class GoProxyDatasource extends Datasource { async getVersionsWithInfo( baseUrl: string, packageName: string - ): Promise { - const releasesIndex = await this.listVersions(baseUrl, packageName); - const releases = await p.map(releasesIndex, async (versionInfo) => { - const { version, releaseTimestamp } = versionInfo; + ): Promise { + const isGopkgin = packageName.startsWith('gopkg.in/'); + const majorSuffixSeparator = isGopkgin ? '.' : '/'; + const modParts = packageName.match(modRegex)?.groups; + const baseMod = modParts?.baseMod ?? packageName; + const packageMajor = parseInt(modParts?.majorVersion ?? '0'); - if (releaseTimestamp) { - return { version, releaseTimestamp }; + const result: ReleaseResult = { releases: [] }; + for (let major = packageMajor; ; major += 1) { + let pkg = `${baseMod}${majorSuffixSeparator}v${major}`; + if (!isGopkgin && major < 2) { + pkg = baseMod; + major += 1; // v0 and v1 are the same module } try { - return await this.versionInfo(baseUrl, packageName, version); + const res = await this.listVersions(baseUrl, pkg); + const releases = await p.map(res, async (versionInfo) => { + const { version, releaseTimestamp } = versionInfo; + + if (releaseTimestamp) { + return { version, releaseTimestamp }; + } + + try { + return await this.versionInfo(baseUrl, pkg, version); + } catch (err) { + logger.trace({ err }, `Can't obtain data from ${baseUrl}`); + return { version }; + } + }); + result.releases.push(...releases); } catch (err) { - logger.trace({ err }, `Can't obtain data from ${baseUrl}`); - return { version }; + if ( + err instanceof HttpError && + err.response?.statusCode === 404 && + major !== packageMajor + ) { + break; + } + + throw err; } - }); - return releases; + } + + return result; } static getCacheKey({ packageName }: GetReleasesConfig): string { From d9cb1a158ccda630634e507280d1adacd6f07011 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 20 Jun 2023 22:41:57 +0300 Subject: [PATCH 2/3] Fix coverage --- lib/modules/datasource/go/releases-goproxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/datasource/go/releases-goproxy.ts b/lib/modules/datasource/go/releases-goproxy.ts index 140a06cbcf30ae..a06c3520354486 100644 --- a/lib/modules/datasource/go/releases-goproxy.ts +++ b/lib/modules/datasource/go/releases-goproxy.ts @@ -239,7 +239,7 @@ export class GoProxyDatasource extends Datasource { const isGopkgin = packageName.startsWith('gopkg.in/'); const majorSuffixSeparator = isGopkgin ? '.' : '/'; const modParts = packageName.match(modRegex)?.groups; - const baseMod = modParts?.baseMod ?? packageName; + const baseMod = modParts?.baseMod ?? /* istanbul ignore next */ packageName; const packageMajor = parseInt(modParts?.majorVersion ?? '0'); const result: ReleaseResult = { releases: [] }; From 65734a616b7e91b6fb33eba3f38e6de4f5c7e3ca Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 21 Jun 2023 12:03:28 +0300 Subject: [PATCH 3/3] fix(go): Fetch data for `respectLatest` in goproxy --- .../datasource/go/releases-goproxy.spec.ts | 34 +++++++++++++++++-- lib/modules/datasource/go/releases-goproxy.ts | 24 +++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/go/releases-goproxy.spec.ts b/lib/modules/datasource/go/releases-goproxy.spec.ts index 06feefb8a766ab..20a1cefbde28ad 100644 --- a/lib/modules/datasource/go/releases-goproxy.spec.ts +++ b/lib/modules/datasource/go/releases-goproxy.spec.ts @@ -348,6 +348,8 @@ describe('modules/datasource/go/releases-goproxy', () => { ) .get('/@v/v1.0.1.info') .reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' }) + .get('/@latest') + .reply(200, { Version: 'v1.0.1' }) .get('/v2/@v/list') .reply(404); @@ -361,6 +363,7 @@ describe('modules/datasource/go/releases-goproxy', () => { { releaseTimestamp: '2019-10-16T16:15:28Z', version: 'v1.0.1' }, ], sourceUrl: 'https://github.com/google/btree', + tags: { latest: 'v1.0.1' }, }); }); @@ -381,9 +384,9 @@ describe('modules/datasource/go/releases-goproxy', () => { .replyWithError('unknown') .get('/@v/v1.0.1.info') .reply(410) + .get('/@latest') + .reply(200, { Version: 'v1.0.1' }) .get('/v2/@v/list') - .reply(200) - .get('/v3/@v/list') .reply(404); const res = await datasource.getReleases({ @@ -393,6 +396,7 @@ describe('modules/datasource/go/releases-goproxy', () => { expect(res).toEqual({ releases: [{ version: 'v1.0.0' }, { version: 'v1.0.1' }], sourceUrl: 'https://github.com/google/btree', + tags: { latest: 'v1.0.1' }, }); }); @@ -418,6 +422,8 @@ describe('modules/datasource/go/releases-goproxy', () => { .reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' }) .get('/@v/v1.0.1.info') .reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' }) + .get('/@latest') + .reply(200, { Version: 'v1.0.1' }) .get('/v2/@v/list') .reply(404); @@ -431,6 +437,7 @@ describe('modules/datasource/go/releases-goproxy', () => { { releaseTimestamp: '2019-10-16T16:15:28Z', version: 'v1.0.1' }, ], sourceUrl: 'https://github.com/google/btree', + tags: { latest: 'v1.0.1' }, }); }); @@ -465,6 +472,8 @@ describe('modules/datasource/go/releases-goproxy', () => { .reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' }) .get('/@v/v1.0.1.info') .reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' }) + .get('/@latest') + .reply(200, { Version: 'v1.0.1' }) .get('/v2/@v/list') .reply(404); @@ -478,6 +487,7 @@ describe('modules/datasource/go/releases-goproxy', () => { { releaseTimestamp: '2019-10-16T16:15:28Z', version: 'v1.0.1' }, ], sourceUrl: 'https://github.com/google/btree', + tags: { latest: 'v1.0.1' }, }); }); @@ -579,6 +589,8 @@ describe('modules/datasource/go/releases-goproxy', () => { .reply(200, ['v1.0.0 2018-08-13T15:31:12Z', 'v1.0.1'].join('\n')) .get('/@v/v1.0.1.info') .reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' }) + .get('/@latest') + .reply(200, { Version: 'v1.0.1' }) .get('/v2/@v/list') .reply(404); httpMock @@ -595,6 +607,7 @@ describe('modules/datasource/go/releases-goproxy', () => { { releaseTimestamp: '2018-08-13T15:31:12Z', version: 'v1.0.0' }, { releaseTimestamp: '2019-10-16T16:15:28Z', version: 'v1.0.1' }, ], + tags: { latest: 'v1.0.1' }, }); }); @@ -615,6 +628,8 @@ describe('modules/datasource/go/releases-goproxy', () => { .reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' }) .get('/@v/v1.0.1.info') .reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' }) + .get('/@latest') + .reply(200, { Version: 'v1.0.1' }) .get('/v2/@v/list') .reply( 200, @@ -624,6 +639,8 @@ describe('modules/datasource/go/releases-goproxy', () => { ) .get('/v2/@v/v2.0.0.info') .reply(200, { Version: 'v2.0.0', Time: '2020-10-16T16:15:28Z' }) + .get('/v2/@latest') + .reply(200, { Version: 'v2.0.0' }) .get('/v3/@v/list') .reply(404); @@ -638,6 +655,7 @@ describe('modules/datasource/go/releases-goproxy', () => { { releaseTimestamp: '2020-10-16T16:15:28Z', version: 'v2.0.0' }, ], sourceUrl: 'https://github.com/google/btree', + tags: { latest: 'v2.0.0' }, }); }); @@ -652,12 +670,16 @@ describe('modules/datasource/go/releases-goproxy', () => { .reply(200, { Version: 'v2.3.0', Time: '2020-05-06T23:08:38Z' }) .get('.v2/@v/v2.4.0.info') .reply(200, { Version: 'v2.4.0', Time: '2020-11-17T15:46:20Z' }) + .get('.v2/@latest') + .reply(200, { Version: 'v2.4.0' }) .get('.v3/@v/list') .reply(200, ['v3.0.0', 'v3.0.1', ' \n'].join('\n')) .get('.v3/@v/v3.0.0.info') .reply(200, { Version: 'v3.0.0', Time: '2022-05-21T10:33:21Z' }) .get('.v3/@v/v3.0.1.info') .reply(200, { Version: 'v3.0.1', Time: '2022-05-27T08:35:30Z' }) + .get('.v3/@latest') + .reply(200, { Version: 'v3.0.1' }) .get('.v4/@v/list') .reply(404); @@ -673,6 +695,7 @@ describe('modules/datasource/go/releases-goproxy', () => { { releaseTimestamp: '2022-05-27T08:35:30Z', version: 'v3.0.1' }, ], sourceUrl: 'https://github.com/go-yaml/yaml', + tags: { latest: 'v3.0.1' }, }); }); @@ -687,10 +710,14 @@ describe('modules/datasource/go/releases-goproxy', () => { .reply(200, { Version: 'v0.1.0', Time: '2017-01-01T00:00:00Z' }) .get('.v0/@v/v0.2.0.info') .reply(200, { Version: 'v0.2.0', Time: '2017-02-01T00:00:00Z' }) + .get('.v0/@latest') + .reply(200, { Version: 'v0.2.0' }) .get('.v1/@v/list') .reply(200, ['v1.0.0', '\n'].join('\n')) .get('.v1/@v/v1.0.0.info') .reply(200, { Version: 'v1.0.0', Time: '2018-01-01T00:00:00Z' }) + .get('.v1/@latest') + .reply(200, { Version: 'v1.0.0' }) .get('.v2/@v/list') .reply(404); @@ -705,6 +732,7 @@ describe('modules/datasource/go/releases-goproxy', () => { { releaseTimestamp: '2018-01-01T00:00:00Z', version: 'v1.0.0' }, ], sourceUrl: 'https://github.com/go-foo/foo', + tags: { latest: 'v1.0.0' }, }); }); @@ -715,6 +743,8 @@ describe('modules/datasource/go/releases-goproxy', () => { .scope(`${baseUrl}/github.com/google/btree`) .get('/@v/list') .reply(200) + .get('/@latest') + .reply(404) .get('/v2/@v/list') .reply(404); diff --git a/lib/modules/datasource/go/releases-goproxy.ts b/lib/modules/datasource/go/releases-goproxy.ts index a06c3520354486..74afa809d15883 100644 --- a/lib/modules/datasource/go/releases-goproxy.ts +++ b/lib/modules/datasource/go/releases-goproxy.ts @@ -6,6 +6,7 @@ import { cache } from '../../../util/cache/package/decorator'; import { HttpError } from '../../../util/http'; import * as p from '../../../util/promises'; import { newlineRegex, regEx } from '../../../util/regex'; +import goVersioning from '../../versioning/go-mod-directive'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; import { BaseGoDatasource } from './base'; @@ -232,6 +233,20 @@ export class GoProxyDatasource extends Datasource { return result; } + async getLatestVersion( + baseUrl: string, + packageName: string + ): Promise { + try { + const url = `${baseUrl}/${this.encodeCase(packageName)}/@latest`; + const res = await this.http.getJson(url); + return res.body.Version; + } catch (err) { + logger.debug({ err }, 'Failed to get latest version'); + return null; + } + } + async getVersionsWithInfo( baseUrl: string, packageName: string @@ -278,6 +293,15 @@ export class GoProxyDatasource extends Datasource { throw err; } + + const latestVersion = await this.getLatestVersion(baseUrl, pkg); + if (latestVersion) { + result.tags ??= {}; + result.tags.latest ??= latestVersion; + if (goVersioning.isGreaterThan(latestVersion, result.tags.latest)) { + result.tags.latest = latestVersion; + } + } } return result;