feat(platform/github): Support rewrite pagination links with the serv…

…er's base URL (#19888)

Co-authored-by: Sergei Zharinov <zharinov@users.noreply.github.com>
Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

docs/usage/self-hosted-experimental.md

@@ -104,6 +104,14 @@ Allowed values:
 
 Default value: `asc`.
 
+## `RENOVATE_X_REBASE_PAGINATION_LINKS`
+
+If set, Renovate will rewrite GitHub Enterprise Server's pagination responses to use the `endpoint` URL from the Renovate config.
+
+<!-- prettier-ignore -->
+!!! note
+    For the GitHub Enterprise Server platform only.
+
 ## `OTEL_EXPORTER_OTLP_ENDPOINT`
 
 If set, Renovate will export OpenTelemetry data to the supplied endpoint.

lib/util/http/github.spec.ts

@@ -52,6 +52,7 @@ describe('util/http/github', () => {
   let repoCache: RepoCacheData = {};
 
   beforeEach(() => {
+    delete process.env.RENOVATE_X_REBASE_PAGINATION_LINKS;
     githubApi = new GithubHttp();
     setBaseUrl(githubApiHost);
     jest.resetAllMocks();
@@ -229,6 +230,75 @@ describe('util/http/github', () => {
       expect(res.body).toEqual(['a']);
     });
 
+    it('rebases GHE Server pagination links', async () => {
+      process.env.RENOVATE_X_REBASE_PAGINATION_LINKS = '1';
+      // The origin and base URL which Renovate uses (from its config) to reach GHE:
+      const baseUrl = 'http://ghe.alternative.domain.com/api/v3';
+      setBaseUrl(baseUrl);
+      // The hostname from GHE settings, which users use through their browsers to reach GHE:
+      // https://docs.github.com/en/enterprise-server@3.5/admin/configuration/configuring-network-settings/configuring-a-hostname
+      const gheHostname = 'ghe.mycompany.com';
+      // GHE replies to paginated requests with a Link response header whose URLs have this base
+      const gheBaseUrl = `https://${gheHostname}/api/v3`;
+      const apiUrl = '/some-url?per_page=2';
+      httpMock
+        .scope(baseUrl)
+        .get(apiUrl)
+        .reply(200, ['a', 'b'], {
+          link: `<${gheBaseUrl}${apiUrl}&page=2>; rel="next", <${gheBaseUrl}${apiUrl}&page=3>; rel="last"`,
+        })
+        .get(`${apiUrl}&page=2`)
+        .reply(200, ['c', 'd'], {
+          link: `<${gheBaseUrl}${apiUrl}&page=3>; rel="next", <${gheBaseUrl}${apiUrl}&page=3>; rel="last"`,
+        })
+        .get(`${apiUrl}&page=3`)
+        .reply(200, ['e']);
+      const res = await githubApi.getJson(apiUrl, { paginate: true });
+      expect(res.body).toEqual(['a', 'b', 'c', 'd', 'e']);
+    });
+
+    it('preserves pagination links by default', async () => {
+      const baseUrl = 'http://ghe.alternative.domain.com/api/v3';
+      setBaseUrl(baseUrl);
+      const apiUrl = '/some-url?per_page=2';
+      httpMock
+        .scope(baseUrl)
+        .get(apiUrl)
+        .reply(200, ['a', 'b'], {
+          link: `<${baseUrl}${apiUrl}&page=2>; rel="next", <${baseUrl}${apiUrl}&page=3>; rel="last"`,
+        })
+        .get(`${apiUrl}&page=2`)
+        .reply(200, ['c', 'd'], {
+          link: `<${baseUrl}${apiUrl}&page=3>; rel="next", <${baseUrl}${apiUrl}&page=3>; rel="last"`,
+        })
+        .get(`${apiUrl}&page=3`)
+        .reply(200, ['e']);
+      const res = await githubApi.getJson(apiUrl, { paginate: true });
+      expect(res.body).toEqual(['a', 'b', 'c', 'd', 'e']);
+    });
+
+    it('preserves pagination links for github.com', async () => {
+      process.env.RENOVATE_X_REBASE_PAGINATION_LINKS = '1';
+      const baseUrl = 'https://api.github.com/';
+
+      setBaseUrl(baseUrl);
+      const apiUrl = 'some-url?per_page=2';
+      httpMock
+        .scope(baseUrl)
+        .get('/' + apiUrl)
+        .reply(200, ['a', 'b'], {
+          link: `<${baseUrl}${apiUrl}&page=2>; rel="next", <${baseUrl}${apiUrl}&page=3>; rel="last"`,
+        })
+        .get(`/${apiUrl}&page=2`)
+        .reply(200, ['c', 'd'], {
+          link: `<${baseUrl}${apiUrl}&page=3>; rel="next", <${baseUrl}${apiUrl}&page=3>; rel="last"`,
+        })
+        .get(`/${apiUrl}&page=3`)
+        .reply(200, ['e']);
+      const res = await githubApi.getJson(apiUrl, { paginate: true });
+      expect(res.body).toEqual(['a', 'b', 'c', 'd', 'e']);
+    });
+
     describe('handleGotError', () => {
       async function fail(
         code: number,

lib/util/http/github.ts

@@ -259,6 +259,11 @@ function setGraphqlPageSize(fieldName: string, newPageSize: number): void {
   }
 }
 
+function replaceUrlBase(url: URL, baseUrl: string): URL {
+  const relativeUrl = `${url.pathname}${url.search}`;
+  return new URL(relativeUrl, baseUrl);
+}
+
 export class GithubHttp extends Http<GithubHttpOptions> {
   constructor(hostType = 'github', options?: GithubHttpOptions) {
     super(hostType, options);
@@ -309,15 +314,27 @@ export class GithubHttp extends Http<GithubHttpOptions> {
         // Check if result is paginated
         const pageLimit = opts.pageLimit ?? 10;
         const linkHeader = parseLinkHeader(result?.headers?.link);
-        if (linkHeader?.next?.url && linkHeader?.last?.page) {
+        const next = linkHeader?.next;
+        if (next?.url && linkHeader?.last?.page) {
           let lastPage = parseInt(linkHeader.last.page, 10);
           // istanbul ignore else: needs a test
           if (!process.env.RENOVATE_PAGINATE_ALL && opts.paginate !== 'all') {
             lastPage = Math.min(pageLimit, lastPage);
           }
+          const baseUrl = opts.baseUrl;
+          const parsedUrl = new URL(next.url, baseUrl);
+          const rebasePagination =
+            !!baseUrl &&
+            !!process.env.RENOVATE_X_REBASE_PAGINATION_LINKS &&
+            // Preserve github.com URLs for use cases like release notes
+            parsedUrl.origin !== 'https://api.github.com';
+          const firstPageUrl = rebasePagination
+            ? replaceUrlBase(parsedUrl, baseUrl)
+            : parsedUrl;
           const queue = [...range(2, lastPage)].map(
             (pageNumber) => (): Promise<HttpResponse<T>> => {
-              const nextUrl = new URL(linkHeader.next!.url, opts.baseUrl);
+              // copy before modifying searchParams
+              const nextUrl = new URL(firstPageUrl);
               nextUrl.searchParams.set('page', String(pageNumber));
               return this.request<T>(
                 nextUrl,