From 77bd7f11577834754198e0f2dccaed3e834c53b6 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Fri, 21 Feb 2025 10:43:23 -0500 Subject: [PATCH] This got clobbered in a previous merge (#698) --- gatsby-node.ts | 355 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 233 insertions(+), 122 deletions(-) diff --git a/gatsby-node.ts b/gatsby-node.ts index b2201c1c..3bea5f90 100644 --- a/gatsby-node.ts +++ b/gatsby-node.ts @@ -1,6 +1,6 @@ /* eslint-disable no-await-in-loop */ import pg from 'pg'; -import { GatsbyNode } from 'gatsby'; +import { CreatePagesArgs, GatsbyNode } from 'gatsby'; import { createRemoteFileNode } from 'gatsby-source-filesystem'; import { SUPABASE_CONNECTION_STRING } from './config'; import { normalizeConnector } from './src/utils'; @@ -38,66 +38,28 @@ const successStoryTemplate = path.resolve( const connectorTemplate = path.resolve('./src/templates/connector/index.tsx'); const connectionTemplate = path.resolve('./src/templates/connection.tsx'); - const authorTemplate = path.resolve('./src/templates/author/index.tsx'); - const comparisonTemplate = path.resolve('./src/templates/etl-tools/index.tsx'); -export const createPages: GatsbyNode['createPages'] = async ({ - graphql, - actions, - reporter, -}) => { - const { createPage, createRedirect } = actions; - - const validateDataExistence = (data: any[] | undefined, name: string) => { - if (!Array.isArray(data) || data.length === 0) { - throw new Error( - `${name} array is either not an array or is empty.` - ); - } - }; +type CreateHelperParams = Pick< + CreatePagesArgs, + 'actions' | 'graphql' | 'reporter' +>; +type CreateHelper = (log: string, params: CreateHelperParams) => Promise; - createRedirect({ - fromPath: '/blogs', - toPath: '/blog', - }); +const QUERY_PANIC_MSG = 'GQL FAILURE LOADING :'; +const validateDataExistence = (data: any[] | undefined, name: string) => { + if (!Array.isArray(data) || data.length === 0) { + throw new Error(`${name} array is either not an array or is empty.`); + } +}; - // Get all strapi blog posts sorted by date - const blogPostsQuery = await graphql<{ - allStrapiBlogPost: { - nodes: { - updatedAt: any; - Slug: string; - id: string; - tags: { - Name: string; - Slug: string; - Type: string; - IsTab: boolean; - }[]; - }[]; - }; - }>(` - { - allStrapiBlogPost( - sort: { publishedAt: DESC } - filter: { publishedAt: { ne: null } } - ) { - nodes { - updatedAt - Slug - id - tags { - Name - Slug - Type - IsTab - } - } - } - } - `); +const createSuccessStories: CreateHelper = async ( + name, + { actions: { createPage }, graphql, reporter } +) => { + const startTime = performance.now(); + console.log(`Creation:Start:${name}`); const successStoryPages = await graphql<{ allStrapiCaseStudy: { @@ -116,10 +78,16 @@ export const createPages: GatsbyNode['createPages'] = async ({ } } `); + if (successStoryPages.errors) { + reporter.panicOnBuild( + `${QUERY_PANIC_MSG} ${name}`, + successStoryPages.errors + ); + return; + } const allSuccessStories = successStoryPages.data?.allStrapiCaseStudy.nodes; - - validateDataExistence(allSuccessStories, 'Success stories'); + validateDataExistence(allSuccessStories, name); allSuccessStories?.forEach((node) => { createPage({ @@ -131,6 +99,18 @@ export const createPages: GatsbyNode['createPages'] = async ({ }); }); + console.log( + `Creation:Finish:${name} took ${Math.ceil(performance.now() - startTime)}ms` + ); +}; + +const createVendorCompare: CreateHelper = async ( + name, + { actions: { createPage }, graphql, reporter } +) => { + const startTime = performance.now(); + console.log(`Creation:Start:${name}`); + const comparisonVendors = await graphql<{ allStrapiComparison: { nodes: Partial[]; @@ -145,9 +125,18 @@ export const createPages: GatsbyNode['createPages'] = async ({ } } `); + if (comparisonVendors.errors) { + reporter.panicOnBuild( + `${QUERY_PANIC_MSG} ${name}`, + comparisonVendors.errors + ); + return; + } const vendors = comparisonVendors.data?.allStrapiComparison.nodes; + validateDataExistence(vendors, name); + // Loop and process what we got (ignore blogs cause that is big and handled down below) if (vendors) { vendors.forEach((xVendor, i) => { vendors.slice(i + 1).forEach((yVendor) => { @@ -170,32 +159,68 @@ export const createPages: GatsbyNode['createPages'] = async ({ }); } + console.log( + `Creation:Finish:${name} took ${Math.ceil(performance.now() - startTime)}ms` + ); +}; + +const createBlogs: CreateHelper = async ( + name, + { actions: { createPage, createRedirect }, graphql, reporter } +) => { + const startTime = performance.now(); + console.log(`Creation:Start:${name}`); + + createRedirect({ + fromPath: '/blogs', + toPath: '/blog', + }); + + // Get all strapi blog posts sorted by date + const blogPostsQuery = await graphql<{ + allStrapiBlogPost: { + nodes: { + updatedAt: any; + Slug: string; + id: string; + tags: { + Name: string; + Slug: string; + Type: string; + IsTab: boolean; + }[]; + }[]; + }; + }>(` + { + allStrapiBlogPost( + sort: { publishedAt: DESC } + filter: { publishedAt: { ne: null } } + ) { + nodes { + updatedAt + Slug + id + tags { + Name + Slug + Type + IsTab + } + } + } + } + `); if (blogPostsQuery.errors) { reporter.panicOnBuild( - 'There was an error loading your blog posts.', + `${QUERY_PANIC_MSG} ${name}`, blogPostsQuery.errors ); return; } - if (comparisonVendors.errors) { - reporter.panicOnBuild( - 'There was an error loading your comparison vendors.', - comparisonVendors.errors - ); - return; - } - if (successStoryPages.errors) { - reporter.panicOnBuild( - 'There was an error loading your success stories.', - successStoryPages.errors - ); - return; - } const allBlogPosts = blogPostsQuery.data?.allStrapiBlogPost.nodes ?? []; - - validateDataExistence(allBlogPosts, 'Blog posts'); - validateDataExistence(vendors, 'Comparison Pages'); + validateDataExistence(allBlogPosts, name); const categories: { [key: string]: { @@ -323,6 +348,17 @@ export const createPages: GatsbyNode['createPages'] = async ({ }); } } + console.log( + `Creation:Finish:${name} took ${Math.ceil(performance.now() - startTime)}ms` + ); +}; + +const createConnectors: CreateHelper = async ( + name, + { actions: { createPage }, graphql, reporter } +) => { + const startTime = performance.now(); + console.log(`Creation:Start:${name}`); const connectors = await graphql<{ postgres: { @@ -351,6 +387,10 @@ export const createPages: GatsbyNode['createPages'] = async ({ } } `); + if (connectors.errors) { + reporter.panicOnBuild(`${QUERY_PANIC_MSG} ${name}`, connectors.errors); + return; + } const mapped_connectors = connectors.data?.postgres.allConnectors.nodes @@ -358,7 +398,7 @@ export const createPages: GatsbyNode['createPages'] = async ({ .map(normalizeConnector) .filter((connector) => connector !== undefined) ?? []; - validateDataExistence(mapped_connectors, 'Connectors'); + validateDataExistence(mapped_connectors, name); for (const normalized_connector of mapped_connectors) { if (!normalized_connector.slug) { @@ -391,6 +431,17 @@ export const createPages: GatsbyNode['createPages'] = async ({ } } } + console.log( + `Creation:Finish:${name} took ${Math.ceil(performance.now() - startTime)}ms` + ); +}; + +const createAuthors: CreateHelper = async ( + name, + { actions: { createPage }, graphql, reporter } +) => { + const startTime = performance.now(); + console.log(`Creation:Start:${name}`); const authors = await graphql<{ allStrapiAuthor: { @@ -407,18 +458,35 @@ export const createPages: GatsbyNode['createPages'] = async ({ } `); - if (authors.data?.allStrapiAuthor.nodes) { - for (const author of authors.data.allStrapiAuthor.nodes) { - createPage({ - path: getAuthorPathBySlug(author.slug), - component: authorTemplate, - context: { - id: author.id, - }, - }); - } + if (authors.errors) { + reporter.panicOnBuild(`${QUERY_PANIC_MSG} ${name}`, authors.errors); + return; } + const allAuthorsData = authors.data?.allStrapiAuthor.nodes ?? []; + validateDataExistence(allAuthorsData, name); + + for (const author of allAuthorsData) { + createPage({ + path: getAuthorPathBySlug(author.slug), + component: authorTemplate, + context: { + id: author.id, + }, + }); + } + console.log( + `Creation:Finish:${name} took ${Math.ceil(performance.now() - startTime)}ms` + ); +}; + +const createCompanyPosts: CreateHelper = async ( + name, + { actions: { createPage }, graphql, reporter } +) => { + const startTime = performance.now(); + console.log(`Creation:Start:${name}`); + const companyUpdatePostsQuery = await graphql<{ allStrapiCompanyUpdatePost: { nodes: { @@ -439,7 +507,7 @@ export const createPages: GatsbyNode['createPages'] = async ({ if (companyUpdatePostsQuery.errors) { reporter.panicOnBuild( - 'There was an error loading your company update posts.', + `${QUERY_PANIC_MSG} ${name}`, companyUpdatePostsQuery.errors ); return; @@ -447,8 +515,7 @@ export const createPages: GatsbyNode['createPages'] = async ({ const allCompanyUpdatePosts = companyUpdatePostsQuery.data?.allStrapiCompanyUpdatePost.nodes ?? []; - - validateDataExistence(allCompanyUpdatePosts, 'Company update posts'); + validateDataExistence(allCompanyUpdatePosts, name); allCompanyUpdatePosts.forEach((post) => { createPage({ @@ -459,6 +526,17 @@ export const createPages: GatsbyNode['createPages'] = async ({ }, }); }); + console.log( + `Creation:Finish:${name} took ${Math.ceil(performance.now() - startTime)}ms` + ); +}; + +const createSolutions: CreateHelper = async ( + name, + { actions: { createPage }, graphql, reporter } +) => { + const startTime = performance.now(); + console.log(`Creation:Start:${name}`); const solutionsQuery = await graphql<{ allStrapiSolution: { @@ -480,13 +558,14 @@ export const createPages: GatsbyNode['createPages'] = async ({ if (solutionsQuery.errors) { reporter.panicOnBuild( - 'There was an error loading your solutions.', + `${QUERY_PANIC_MSG} ${name}`, solutionsQuery.errors ); return; } const allSolutions = solutionsQuery.data?.allStrapiSolution.nodes ?? []; + validateDataExistence(allSolutions, name); const useCaseSolutions = allSolutions.filter((solution) => solution.slug.includes('/use-cases/') @@ -498,33 +577,65 @@ export const createPages: GatsbyNode['createPages'] = async ({ solution.slug.includes('/technology/') ); - validateDataExistence(allSolutions, 'Solutions'); - const solutionsUrlPrefix = '/solutions'; + await Promise.all([ + Promise.all( + useCaseSolutions.map((useCaseSolution) => + createPage({ + path: `${solutionsUrlPrefix}${useCaseSolution.slug}`, + component: useCaseSolutionsTemplate, + context: { id: useCaseSolution.id }, + }) + ) + ), - useCaseSolutions.map((useCaseSolution) => - createPage({ - path: `${solutionsUrlPrefix}${useCaseSolution.slug}`, - component: useCaseSolutionsTemplate, - context: { id: useCaseSolution.id }, - }) - ); - industrySolutions.map((industrySolution) => - createPage({ - path: `${solutionsUrlPrefix}${industrySolution.slug}`, - component: industrySolutionsTemplate, - context: { id: industrySolution.id }, - }) - ); - technologySolutions.map((technologySolution) => - createPage({ - path: `${solutionsUrlPrefix}${technologySolution.slug}`, - component: technologySolutionsTemplate, - context: { id: technologySolution.id }, - }) + Promise.all( + industrySolutions.map((industrySolution) => + createPage({ + path: `${solutionsUrlPrefix}${industrySolution.slug}`, + component: industrySolutionsTemplate, + context: { id: industrySolution.id }, + }) + ) + ), + + Promise.all( + technologySolutions.map((technologySolution) => + createPage({ + path: `${solutionsUrlPrefix}${technologySolution.slug}`, + component: technologySolutionsTemplate, + context: { id: technologySolution.id }, + }) + ) + ), + ]); + console.log( + `Creation:Finish:${name} took ${Math.ceil(performance.now() - startTime)}ms` ); }; +export const createPages: GatsbyNode['createPages'] = async ({ + graphql, + actions, + reporter, +}) => { + const createHelperParams: CreateHelperParams = { + actions, + graphql, + reporter, + }; + + await Promise.all([ + createConnectors('connectors', createHelperParams), + createBlogs('blog posts', createHelperParams), + createAuthors('authors', createHelperParams), + createCompanyPosts('company posts', createHelperParams), + createSolutions('solutions', createHelperParams), + createSuccessStories('success stories', createHelperParams), + createVendorCompare('vendor compare', createHelperParams), + ]); +}; + // Hacky hack :( // tl;dr `gatsby-source-pg` doesn't go through the normal gatsby `createNode` workflow. // As a result, we can't look up the nodes for connectors in order to download their logo. @@ -556,19 +667,17 @@ export const sourceNodes: GatsbyNode['sourceNodes'] = async ({ getCache, createContentDigest, }) => { - console.log('sourceNodes start'); + console.log('sourceNodes:start'); const pool = new pg.Pool({ connectionString: SUPABASE_CONNECTION_STRING, - connectionTimeoutMillis: 5 * 1000, + connectionTimeoutMillis: 8000, }); - console.log('-fetching start'); const connectors = await pool.query( 'select connectors.id as id, connectors.logo_url as logo_url from public.connectors;' ); - console.log('-fetching done'); - console.log('--loop start'); + let createdCount = 0; for (const conn of connectors.rows) { const usUrl = conn.logo_url?.['en-US']; @@ -590,15 +699,17 @@ export const sourceNodes: GatsbyNode['sourceNodes'] = async ({ contentDigest: createContentDigest(fileNode), }, }); + + createdCount += 1; } else { - console.error('---no logo', conn.id); + console.log('sourceNodes:missing connector logo', conn.id); } } - console.log('--loop done'); - await pool.end(); - console.log('sourceNodes done'); + console.log('sourceNodes:', { + completed: `${createdCount}/${connectors.rowCount}`, + }); }; export const createResolvers: GatsbyNode['createResolvers'] = async ({