Handle private repos (#13)

* fix gitlab server source info

* skip public repos by default with a flag

* update todo

src/commands/azuredevops.ts

@@ -50,7 +50,7 @@ export default class AzureDevOps extends Command {
     async run(): Promise<void> {
         const { flags } = (await this.parse(AzureDevOps));
 
-        const sourceInfo = AzureDevOps.getSourceInfo(':' + flags.token);
+        const sourceInfo = AzureDevOps.getSourceInfo(':' + flags.token, flags['include-public']);
 
         const apiManager = new AzureApiManager(sourceInfo, flags['ca-cert']);
         const runner = new AzureRunner(sourceInfo, flags, apiManager);
@@ -62,16 +62,17 @@ export default class AzureDevOps extends Command {
         await runner.execute();
     }
 
-    static getSourceInfo(token: string, baseUrl = 'https://dev.azure.com', sourceType = SourceType.AzureRepos): VcsSourceInfo {
+    static getSourceInfo(token: string, includePublic: boolean, url = 'https://dev.azure.com', sourceType = SourceType.AzureRepos): VcsSourceInfo {
         return {
-            sourceType: sourceType,
-            url: baseUrl,
-            token: token,
+            sourceType,
+            url,
+            token,
             repoTerm: 'repo',
             orgTerm: 'organization',
             orgFlagName: 'orgs',
             minPathLength: 3,
-            maxPathLength: 3
+            maxPathLength: 3,
+            includePublic
         };
     }
 }

src/commands/bitbucket-server.ts

@@ -36,24 +36,25 @@ export default class BitbucketServer extends Bitbucket {
 
         const serverUrl = getServerUrl(flags.hostname, flags.port, flags.protocol);
         const baseUrl = `${serverUrl}/rest/api/1.0`;
-        const sourceInfo = BitbucketServer.getSourceInfo(`${flags.username}:${flags.token}`, baseUrl);
+        const sourceInfo = BitbucketServer.getSourceInfo(`${flags.username}:${flags.token}`, flags['include-public'], baseUrl);
 
         const apiManager = new BitbucketServerApiManager(sourceInfo, flags['ca-cert']);
         const runner = new BitbucketServerRunner(sourceInfo, flags, apiManager);
 
         await runner.execute();
     }
 
-    static getSourceInfo(token: string, baseUrl: string, sourceType = SourceType.BitbucketServer): VcsSourceInfo {
+    static getSourceInfo(token: string, includePublic: boolean, url: string, sourceType = SourceType.BitbucketServer): VcsSourceInfo {
         return {
-            sourceType: sourceType,
-            url: baseUrl,
-            token: token,
+            sourceType,
+            url,
+            token,
             repoTerm: 'repo',
             orgTerm: 'project',
             orgFlagName: 'projects',
             minPathLength: 2,
-            maxPathLength: 2
+            maxPathLength: 2,
+            includePublic
         };
     }
 }

src/commands/bitbucket.ts

@@ -40,24 +40,25 @@ export default class Bitbucket extends Command {
     async run(): Promise<void> {
         const { flags } = await this.parse(Bitbucket);
 
-        const sourceInfo = Bitbucket.getSourceInfo(`${flags.username}:${flags.token}`);
+        const sourceInfo = Bitbucket.getSourceInfo(`${flags.username}:${flags.token}`, flags['include-public']);
 
         const apiManager = new BitbucketApiManager(sourceInfo, flags['requests-per-hour'], flags['ca-cert']);
         const runner = new BitbucketRunner(sourceInfo, flags, apiManager);
 
         await runner.execute();
     }
 
-    static getSourceInfo(token: string, baseUrl = 'https://api.bitbucket.org/2.0', sourceType = SourceType.Bitbucket): VcsSourceInfo {
+    static getSourceInfo(token: string, includePublic: boolean, url = 'https://api.bitbucket.org/2.0', sourceType = SourceType.Bitbucket): VcsSourceInfo {
         return {
-            sourceType: sourceType,
-            url: baseUrl,
-            token: token,
+            sourceType,
+            url,
+            token,
             repoTerm: 'repo',
             orgTerm: 'workspace',
             orgFlagName: 'workspaces',
             minPathLength: 2,
-            maxPathLength: 2
+            maxPathLength: 2,
+            includePublic
         };
     }
 }

src/commands/github-server.ts

@@ -28,7 +28,7 @@ export default class GithubServer extends Github {
 
         const serverUrl = getServerUrl(flags.hostname, flags.port, flags.protocol);
         const baseUrl = `${serverUrl}/api/v3`;
-        const sourceInfo = Github.getSourceInfo(flags.token, baseUrl, SourceType.GithubServer);
+        const sourceInfo = Github.getSourceInfo(flags.token, flags['include-public'], baseUrl, SourceType.GithubServer);
 
         const apiManager = new GithubServerApiManager(sourceInfo, flags['ca-cert']);
         const runner = new GithubServerRunner(sourceInfo, flags, apiManager);

src/commands/github.ts

@@ -30,24 +30,25 @@ export default class Github extends Command {
     async run(): Promise<void> {
         const { flags } = await this.parse(Github);
 
-        const sourceInfo = Github.getSourceInfo(flags.token);
+        const sourceInfo = Github.getSourceInfo(flags.token, flags['include-public']);
 
         const apiManager = new GithubApiManager(sourceInfo, flags['ca-cert']);
         const runner = new GithubRunner(sourceInfo, flags, apiManager);
 
         await runner.execute();
     }
 
-    static getSourceInfo(token: string, baseUrl = 'https://api.github.com', sourceType = SourceType.Github): VcsSourceInfo {
+    static getSourceInfo(token: string, includePublic: boolean, url = 'https://api.github.com', sourceType = SourceType.Github): VcsSourceInfo {
         return {
-            sourceType: sourceType,
-            url: baseUrl,
-            token: token,
+            sourceType,
+            url,
+            token,
             repoTerm: 'repo',
             orgTerm: 'organization',
             orgFlagName: 'orgs',
             minPathLength: 2,
-            maxPathLength: 2
+            maxPathLength: 2,
+            includePublic
         };
     }
 }

src/commands/gitlab-server.ts

@@ -26,7 +26,8 @@ export default class GitlabServer extends Gitlab {
 
         const serverUrl = getServerUrl(flags.hostname, flags.port, flags.protocol);
         const baseUrl = `${serverUrl}/api/v4`;
-        const sourceInfo = Gitlab.getSourceInfo(flags.token, baseUrl, SourceType.GithubServer);
+
+        const sourceInfo = Gitlab.getSourceInfo(flags.token, flags['include-public'], baseUrl, SourceType.GitlabServer);
 
         const apiManager = new GitlabServerApiManager(sourceInfo, flags['ca-cert']);
         const runner = new GitlabServerRunner(sourceInfo, flags, apiManager);

src/commands/gitlab.ts

@@ -32,24 +32,25 @@ export default class Gitlab extends Command {
     async run(): Promise<void> {
         const { flags } = await this.parse(Gitlab);
 
-        const sourceInfo = Gitlab.getSourceInfo(flags.token);
+        const sourceInfo = Gitlab.getSourceInfo(flags.token, flags['include-public']);
 
         const apiManager = new GitlabApiManager(sourceInfo, flags['ca-cert']);
         const runner = new GitlabRunner(sourceInfo, flags, apiManager);
 
         await runner.execute();
     }
 
-    static getSourceInfo(token: string, baseUrl = 'https://gitlab.com/api/v4', sourceType = SourceType.Gitlab): VcsSourceInfo {
+    static getSourceInfo(token: string, includePublic: boolean, url = 'https://gitlab.com/api/v4', sourceType = SourceType.Gitlab): VcsSourceInfo {
         return {
-            sourceType: sourceType,
-            url: baseUrl,
-            token: token,
+            sourceType,
+            url,
+            token,
             repoTerm: 'project',
             orgTerm: 'group',
             orgFlagName: 'groups',
             minPathLength: 2,
-            maxPathLength: 99
+            maxPathLength: 99,
+            includePublic
         };
     }
 }

src/commands/local.ts

@@ -2,14 +2,15 @@ import { Command, Flags } from '@oclif/core';
 import { CLIError } from '@oclif/errors';
 import { commonFlags } from '../common/flags';
 import { HelpGroup, SourceType } from '../common/types';
+import { deleteFlagKey } from '../common/utils';
 import { LocalApiManager } from '../vcs/local/local-api-manager';
 import { LocalRunner } from '../vcs/local/local-runner';
 
 export default class Local extends Command {
 
     static summary = 'Count active contributors in local directories using `git log`'
 
-    static description = ''
+    static description = 'Note that `local` mode has no way to determine if a repo is public or private. Thus, all repos will be counted, and this may cause different behavior in the platform, which does not include public repos in the contributor count.'
 
     static examples = [
         `$ <%= config.bin %> <%= command.id %> --repos . --skip-repos ./terragoat`,
@@ -41,7 +42,7 @@ export default class Local extends Command {
             aliases: ['skip-dir-file'],
             helpGroup: HelpGroup.REPO_SPEC
         }),
-        ...commonFlags
+        ...deleteFlagKey(commonFlags, 'include-public', 'ca-cert')
     };
 
     async run(): Promise<void> {

src/common/base-runner.ts

@@ -5,13 +5,11 @@ import { Commit, ContributorMap, Repo, RepoResponse, SourceInfo, VCSCommit, VcsS
 import { DEFAULT_DAYS, getXDaysAgoDate, isSslError, logError, LOGGER } from './utils';
 
 // TODO
-// - all branches? (no?)
-// - default to private only repos - should we explicitly check each repo for its visibility?
+// - all branches? (no?) - remove all branches from the BB call, remote git log --all
 // - document specific permissions needed
-// - public / private repos - include in output?
 // - document getting a cert chain
 // - some sort of errored repo list that is easy to review
-// - author vs committer
+// - author vs committer - use author always (not committer)
 // - clean up logging
 // - test on windows
 

src/common/flags.ts

@@ -32,6 +32,10 @@ export const commonFlags = {
     'exclude-empty': Flags.boolean({
         description: 'Do not include repos with no commits in the output',
         helpGroup: HelpGroup.OUTPUT
+    }),
+    'include-public': Flags.boolean({
+        description: 'The platform only counts contributors in private repos (and "internal" repos for some enterprise systems). If you wish to see contributors in public repos, for informational or other purposes, use this flag. This will also cause Redshirts to skip checking if a repo is public, so it can speed up the runtime if you know you are only supplying private repos, or mostly private repos, using --repos, --orgs, etc.',
+        helpGroup: HelpGroup.REPO_SPEC
     })
 };
 

src/common/rate-limit-vcs-api-manager.ts

@@ -9,6 +9,7 @@ export abstract class RateLimitVcsApiManager extends VcsApiManager {
     rateLimitRemainingHeader: string
     rateLimitResetHeader: string
     rateLimitEndpoint?: string
+    lastResponse?: AxiosResponse
 
     constructor(sourceInfo: VcsSourceInfo, rateLimitRemainingHeader: string, rateLimitResetHeader: string, rateLimitEndpoint?: string, certPath?: string) {
         super(sourceInfo, certPath);
@@ -66,17 +67,20 @@ export abstract class RateLimitVcsApiManager extends VcsApiManager {
         return this.getRateLimitStatus(response);
     }
 
-    getRateLimitStatus(response: AxiosResponse<any, any>): RateLimitStatus | undefined {
-        return this.rateLimitRemainingHeader in response.headers ? {
-            remaining: Number.parseInt(response.headers[this.rateLimitRemainingHeader]!),
-            reset: new Date(Number.parseInt(response.headers[this.rateLimitResetHeader]!) * 1000)
+    getRateLimitStatus(response?: AxiosResponse<any, any>): RateLimitStatus | undefined {
+        // returns the rate limit status from this response, if present, otherwise from this.lastResponse, otherwise undefined
+        const r = response || this.lastResponse || undefined;
+        return r && this.rateLimitRemainingHeader in r.headers ? {
+            remaining: Number.parseInt(r.headers[this.rateLimitRemainingHeader]!),
+            reset: new Date(Number.parseInt(r.headers[this.rateLimitResetHeader]!) * 1000)
         } : undefined;
     }
 
     async submitRequest(config: AxiosRequestConfig, previousResponse?: AxiosResponse): Promise<AxiosResponse> {
         await this.handleRateLimit(previousResponse);
         try {
-            return await this.axiosInstance.request(config);
+            this.lastResponse = await this.axiosInstance.request(config);
+            return this.lastResponse;
         } catch (error) {
             if (error instanceof AxiosError && error.response?.status === 429) {
                 // now we expect the rate limit details to be in the header
@@ -88,7 +92,7 @@ export abstract class RateLimitVcsApiManager extends VcsApiManager {
     }
 
     async handleRateLimit(response?: AxiosResponse): Promise<void> {
-        const rateLimitStatus = response ? this.getRateLimitStatus(response) : undefined;
+        const rateLimitStatus = this.getRateLimitStatus(response) || await this.checkRateLimitStatus();
         LOGGER.debug(`Rate limit remaining: ${rateLimitStatus ? rateLimitStatus.remaining : 'unknown'}`);
         // <= to handle a weird edge case I encountered but coult not reproduce
         // Not 0 for a small concurrency buffer for other uses of this token

src/common/throttled-vcs-api-manager.ts

@@ -3,6 +3,7 @@ import { RepoResponse, VcsSourceInfo } from './types';
 import https = require('https')
 import { VcsApiManager } from './vcs-api-manager';
 import Bottleneck from 'bottleneck';
+import { LOGGER } from './utils';
 
 
 export abstract class ThrottledVcsApiManager extends VcsApiManager {
@@ -17,7 +18,7 @@ export abstract class ThrottledVcsApiManager extends VcsApiManager {
             reservoirRefreshInterval: 3600 * 1000,
             reservoirRefreshAmount: requestsPerHour,
             maxConcurrent: 1,
-            minTime: Math.ceil(1000 * 60 * 60 / requestsPerHour)  // delay time in ms between requests
+            minTime: 100
         });
     }
 
@@ -37,6 +38,12 @@ export abstract class ThrottledVcsApiManager extends VcsApiManager {
     abstract getUserRepos(): Promise<RepoResponse[]>
 
     async submitRequest(config: AxiosRequestConfig, _?: AxiosResponse): Promise<AxiosResponse> {
-        return this.bottleneck.schedule(() => this.axiosInstance.request(config));
+        const response = await this.bottleneck.schedule(() => this.axiosInstance.request(config));
+
+        if (LOGGER.level === 'debug') {
+            LOGGER.debug(`Reservoir remaining: ${await this.bottleneck.currentReservoir()}`);
+        }
+
+        return response;
     }
 }

src/common/types.ts

@@ -10,6 +10,7 @@ export interface VcsSourceInfo extends SourceInfo {
     orgFlagName: string
     minPathLength: number
     maxPathLength: number
+    includePublic: boolean
 }
 
 export type RateLimitStatus = {

src/common/utils.ts

@@ -239,6 +239,11 @@ export const sleepUntilDateTime = async (until: Date): Promise<void> => {
     return new Promise(resolve => setTimeout(resolve, ms));
 };
 
+export const sleepForDuration = async (ms: number): Promise<void> => {
+    LOGGER.debug(`Sleeping for ${ms} ms`);
+    // eslint-disable-next-line no-promise-executor-return
+    return new Promise(resolve => setTimeout(resolve, ms));
+
 // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
 export const objectToString = (obj: any): string => {
     return `{${Object.keys(obj).map(k => `${k}: ${obj[k]}`).join(', ')}}`;

src/common/vcs-api-manager.ts

@@ -1,5 +1,5 @@
 import axios, { AxiosInstance, AxiosRequestConfig, AxiosResponse, RawAxiosRequestHeaders } from 'axios';
-import { RepoResponse, VcsSourceInfo } from './types';
+import { Repo, RepoResponse, VcsSourceInfo } from './types';
 import { getFileBuffer, LOGGER } from './utils';
 import https = require('https')
 import { ApiManager } from './api-manager';
@@ -32,6 +32,7 @@ export abstract class VcsApiManager extends ApiManager {
     abstract _getAxiosConfiguration(): any
     abstract getOrgRepos(group: string): Promise<RepoResponse[]>
     abstract getUserRepos(): Promise<RepoResponse[]>
+    abstract isRepoPublic(repo: Repo): Promise<boolean>;
     abstract submitRequest(config: AxiosRequestConfig, previousResponse?: AxiosResponse): Promise<AxiosResponse>
 
     hasMorePages(response: AxiosResponse): boolean {

src/common/vcs-runner.ts

@@ -1,7 +1,7 @@
 import { AxiosError } from 'axios';
 import { BaseRunner } from './base-runner';
 import { Repo, VcsSourceInfo } from './types';
-import { filterRepoList, getExplicitRepoList, getRepoListFromParams, LOGGER, stringToArr } from './utils';
+import { filterRepoList, getExplicitRepoList, getRepoListFromParams, logError, LOGGER, stringToArr } from './utils';
 import { VcsApiManager } from './vcs-api-manager';
 
 // TODO
@@ -37,12 +37,24 @@ export abstract class VcsRunner extends BaseRunner {
 
         if (orgsString) {
             repos = await this.getOrgRepos(orgsString);
-            LOGGER.debug(`Got repos from org(s): ${repos.map(r => `${r.owner}/${r.name}`)}`);
+            LOGGER.debug(`Got repos from ${this.sourceInfo.orgTerm}(s): ${repos.map(r => `${r.owner}/${r.name}`)}`);
         }
 
         const addedRepos = getExplicitRepoList(this.sourceInfo, repos, reposList, reposFile);
 
         if (addedRepos.length > 0) {
+            if (!this.sourceInfo.includePublic) {
+                LOGGER.debug(`--include-public was not set - getting the visibility of all explicitly specified ${this.sourceInfo.repoTerm}s`);
+                for (const repo of addedRepos) {
+                    try {
+                        // eslint-disable-next-line no-await-in-loop
+                        repo.private = !await this.apiManager.isRepoPublic(repo);
+                    } catch (error) {
+                        logError(error as Error, `An error occurred getting the visibility for the ${this.sourceInfo.repoTerm} ${repo.owner}/${repo.name}. It will be excluded from the list, because this will probably lead to an error later.`);
+                    }
+                }
+            }
+
             LOGGER.debug(`Added repos from --repo list: ${addedRepos.map(r => `${r.owner}/${r.name}`)}`);
             repos.push(...addedRepos);
         }
@@ -56,6 +68,21 @@ export abstract class VcsRunner extends BaseRunner {
 
         repos = filterRepoList(repos, skipRepos, this.sourceInfo.repoTerm);
 
+        // now that we have all the repos and their visibility, we can remove the public ones if needed
+        if (!this.sourceInfo.includePublic) {
+            repos = repos.filter(repo => {
+                if (repo.private === undefined) {
+                    LOGGER.debug(`Found ${this.sourceInfo.repoTerm} with unknown visibility: ${repo.owner}/${repo.name} - did it error out above? It will be skipped.`);
+                    return false;
+                } else if (repo.private) {
+                    return true;
+                } else {
+                    LOGGER.debug(`Skipping public ${this.sourceInfo.repoTerm}: ${repo.owner}/${repo.name}`);
+                    return false;
+                }
+            });
+        }
+
         return repos;
     }