feat: enable multi page star fetching for private vercel instances (a…

…nuraghazra#2159) * feat: enable multi-page stars' fetching for private vercel instances This commit enables multi-page stars' support from fetching on private Vercel instances. This feature can be disabled on the public Vercel instance by adding the `FETCH_SINGLE_PAGE_STARS=true` as an env variable in the public Vercel instance. This variable will not be present when people deploy their own Vercel instance, causing the code to fetch multiple star pages. * fix: improve stats multi-page fetching behavoir This commit makes sure that the GraphQL api is only called one time per 100 repositories. The old method added one unnecesairy GraphQL call. * docs: update documentation * style: improve code syntax Co-authored-by: Matteo Pierro <pierromatteo@gmail.com> * lol happy new year * docs: remove rate limit documentation for now Remove the `FETCH_SINGLE_PAGE_STARS` from documentation for now since it might confuse people. * fix: fix error in automatic merge * feat: make sure env variable is read Co-authored-by: Matteo Pierro <pierromatteo@gmail.com> Co-authored-by: Anurag <hazru.anurag@gmail.com>
eavilaes · Jan 23, 2023 · d7ad0eb · d7ad0eb · vercel · Jan 23, 2023
1 parent c8912fe
commit d7ad0eb
Show file tree

Hide file tree

Showing 4 changed files with 210 additions and 166 deletions.
diff --git a/readme.md b/readme.md
@@ -93,7 +93,7 @@ Visit <https://indiafightscorona.giveindia.org> and make a small donation to hel
     -   [Language Card Exclusive Options](#language-card-exclusive-options)
     -   [Wakatime Card Exclusive Option](#wakatime-card-exclusive-options)
 -   [Deploy Yourself](#deploy-on-your-own-vercel-instance)
-    - [Keep your fork up to date](#keep-your-fork-up-to-date)   
+    -   [Keep your fork up to date](#keep-your-fork-up-to-date)
 
 # GitHub Stats Card
 
@@ -264,7 +264,7 @@ You can customize the appearance of your `Stats Card` or `Repo Card` however you
 -   `border_radius` - Corner rounding on the card. Default: `4.5`.
 
 > **Warning**
-> We use caching to decrease the load on our servers (see https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.
+> We use caching to decrease the load on our servers (see <https://github.com/anuraghazra/github-readme-stats/issues/1471#issuecomment-1271551425>). Our cards have a default cache of 4 hours (14400 seconds). Also, note that the cache is clamped to a minimum of 4 hours and a maximum of 24 hours.
 
 ##### Gradient in bg_color
 
@@ -354,7 +354,7 @@ Use [show_owner](#customization) variable to include the repo's owner username
 The top languages card shows a GitHub user's most frequently used top language.
 
 > **Note**
-> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats._
+> Top Languages does not indicate my skill level or anything like that; it's a GitHub metric to determine which languages have the most code on GitHub. It is a new feature of github-readme-stats.
 
 ### Usage
 
@@ -498,7 +498,7 @@ By default, GitHub does not lay out the cards side by side. To do that, you can
 
 ## Deploy on your own Vercel instance
 
-#### [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)
+#### :film_projector: [Check Out Step By Step Video Tutorial By @codeSTACKr](https://youtu.be/n6d4KHSKqGk?t=107)
 
 > **Warning**
 > If you are on the [hobby (i.e. free)](https://vercel.com/pricing) Vercel plan, please make sure you change the `maxDuration` parameter in the [vercel.json](https://github.com/anuraghazra/github-readme-stats/blob/master/vercel.json) file from `30` to `10` (see [#1416](https://github.com/anuraghazra/github-readme-stats/issues/1416#issuecomment-950275476) for more information).

diff --git a/src/fetchers/stats-fetcher.js b/src/fetchers/stats-fetcher.js
@@ -1,5 +1,6 @@
 // @ts-check
 import axios from "axios";
+import * as dotenv from "dotenv";
 import githubUsernameRegex from "github-username-regex";
 import { calculateRank } from "../calculateRank.js";
 import { retryer } from "../common/retryer.js";
@@ -11,46 +12,74 @@ import {
   wrapTextMultiline,
 } from "../common/utils.js";
 
+dotenv.config();
+
+// GraphQL queries.
+const GRAPHQL_REPOS_FIELD = `
+  repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) {
+    totalCount
+    nodes {
+      name
+      stargazers {
+        totalCount
+      }
+    }
+    pageInfo {
+      hasNextPage
+      endCursor
+    }
+  }
+`;
+
+const GRAPHQL_REPOS_QUERY = `
+  query userInfo($login: String!, $after: String) {
+    user(login: $login) {
+      ${GRAPHQL_REPOS_FIELD}
+    }
+  }
+`;
+
+const GRAPHQL_STATS_QUERY = `
+  query userInfo($login: String!, $after: String) {
+    user(login: $login) {
+      name
+      login
+      contributionsCollection {
+        totalCommitContributions
+        restrictedContributionsCount
+      }
+      repositoriesContributedTo(first: 1, contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
+        totalCount
+      }
+      pullRequests(first: 1) {
+        totalCount
+      }
+      openIssues: issues(states: OPEN) {
+        totalCount
+      }
+      closedIssues: issues(states: CLOSED) {
+        totalCount
+      }
+      followers {
+        totalCount
+      }
+      ${GRAPHQL_REPOS_FIELD}
+    }
+  }
+`;
+
 /**
  * Stats fetcher object.
  *
  * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
  * @param {string} token GitHub token.
- * @returns {Promise<import('../common/types').StatsFetcherResponse>} Stats fetcher response.
+ * @returns {Promise<import('../common/types').Fetcher>} Stats fetcher response.
  */
 const fetcher = (variables, token) => {
+  const query = !variables.after ? GRAPHQL_STATS_QUERY : GRAPHQL_REPOS_QUERY;
   return request(
     {
-      query: `
-      query userInfo($login: String!) {
-        user(login: $login) {
-          name
-          login
-          contributionsCollection {
-            totalCommitContributions
-            restrictedContributionsCount
-          }
-          repositoriesContributedTo(contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) {
-            totalCount
-          }
-          pullRequests {
-            totalCount
-          }
-          openIssues: issues(states: OPEN) {
-            totalCount
-          }
-          closedIssues: issues(states: CLOSED) {
-            totalCount
-          }
-          followers {
-            totalCount
-          }
-          repositories(ownerAffiliations: OWNER) {
-            totalCount
-          }
-        }
-      }
-      `,
+      query,
       variables,
     },
     {
@@ -60,39 +89,42 @@ const fetcher = (variables, token) => {
 };
 
 /**
- * Fetch first 100 repositories for a given username.
+ * Fetch stats information for a given username.
  *
- * @param {import('axios').AxiosRequestHeaders} variables Fetcher variables.
- * @param {string} token GitHub token.
- * @returns {Promise<import('../common/types').StatsFetcherResponse>} Repositories fetcher response.
+ * @param {string} username Github username.
+ * @returns {Promise<import('../common/types').StatsFetcher>} GraphQL Stats object.
+ *
+ * @description This function supports multi-page fetching if the 'FETCH_MULTI_PAGE_STARS' environment variable is set to true.
  */
-const repositoriesFetcher = (variables, token) => {
-  return request(
-    {
-      query: `
-      query userInfo($login: String!, $after: String) {
-        user(login: $login) {
-          repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) {
-            nodes {
-              name
-              stargazers {
-                totalCount
-              }
-            }
-            pageInfo {
-              hasNextPage
-              endCursor
-            }
-          }
-        }
-      }
-      `,
-      variables,
-    },
-    {
-      Authorization: `bearer ${token}`,
-    },
-  );
+const statsFetcher = async (username) => {
+  let stats;
+  let hasNextPage = true;
+  let endCursor = null;
+  while (hasNextPage) {
+    const variables = { login: username, first: 100, after: endCursor };
+    let res = await retryer(fetcher, variables);
+    if (res.data.errors) return res;
+
+    // Store stats data.
+    const repoNodes = res.data.data.user.repositories.nodes;
+    if (!stats) {
+      stats = res;
+    } else {
+      stats.data.data.user.repositories.nodes.push(...repoNodes);
+    }
+
+    // Disable multi page fetching on public Vercel instance due to rate limits.
+    const repoNodesWithStars = repoNodes.filter(
+      (node) => node.stargazers.totalCount !== 0,
+    );
+    hasNextPage =
+      process.env.FETCH_MULTI_PAGE_STARS === "true" &&
+      repoNodes.length === repoNodesWithStars.length &&
+      res.data.data.user.repositories.pageInfo.hasNextPage;
+    endCursor = res.data.data.user.repositories.pageInfo.endCursor;
+  }
+
+  return stats;
 };
 
 /**
@@ -137,46 +169,6 @@ const totalCommitsFetcher = async (username) => {
   return 0;
 };
 
-/**
- * Fetch all the stars for all the repositories of a given username.
- *
- * @param {string} username GitHub username.
- * @param {array} repoToHide Repositories to hide.
- * @returns {Promise<number>} Total stars.
- */
-const totalStarsFetcher = async (username, repoToHide) => {
-  let nodes = [];
-  let hasNextPage = true;
-  let endCursor = null;
-  while (hasNextPage) {
-    const variables = { login: username, first: 100, after: endCursor };
-    let res = await retryer(repositoriesFetcher, variables);
-
-    if (res.data.errors) {
-      logger.error(res.data.errors);
-      throw new CustomError(
-        res.data.errors[0].message || "Could not fetch user",
-        CustomError.USER_NOT_FOUND,
-      );
-    }
-
-    const allNodes = res.data.data.user.repositories.nodes;
-    const nodesWithStars = allNodes.filter(
-      (node) => node.stargazers.totalCount !== 0,
-    );
-    nodes.push(...nodesWithStars);
-    // hasNextPage =
-    //   allNodes.length === nodesWithStars.length &&
-    //   res.data.data.user.repositories.pageInfo.hasNextPage;
-    hasNextPage = false; // NOTE: Temporarily disable fetching of multiple pages. Done because of #2130.
-    endCursor = res.data.data.user.repositories.pageInfo.endCursor;
-  }
-
-  return nodes
-    .filter((data) => !repoToHide[data.name])
-    .reduce((prev, curr) => prev + curr.stargazers.totalCount, 0);
-};
-
 /**
  * Fetch stats for a given username.
  *
@@ -203,7 +195,7 @@ const fetchStats = async (
     rank: { level: "C", score: 0 },
   };
 
-  let res = await retryer(fetcher, { login: username });
+  let res = await statsFetcher(username);
 
   // Catch GraphQL errors.
   if (res.data.errors) {
@@ -259,8 +251,15 @@ const fetchStats = async (
   stats.contributedTo = user.repositoriesContributedTo.totalCount;
 
   // Retrieve stars while filtering out repositories to be hidden
-  stats.totalStars = await totalStarsFetcher(username, repoToHide);
+  stats.totalStars = user.repositories.nodes
+    .filter((data) => {
+      return !repoToHide[data.name];
+    })
+    .reduce((prev, curr) => {
+      return prev + curr.stargazers.totalCount;
+    }, 0);
 
+  // @ts-ignore // TODO: Fix this.
   stats.rank = calculateRank({
     totalCommits: stats.totalCommits,
     totalRepos: user.repositories.totalCount,