Filter connectors better and make connector search page safer (#699)

* When searching this logo was for sure missing at time Need to exclude and filter out Kelkoo * Needed to import the graphql from gatsby making stuff a bit more safe updating typing to match what could happen * adding back in missing imports * adding imports * We handle missing tags in the normalize call so we don't have to filter * Marking anything that _could_ be undefined as optional * cleaning up logging * removing testing logs * Updating caniuse * Upgrading to the latest strapi plugin setting maximum parallel requests to stop hammering strapi
estuary · Feb 24, 2025 · 1d11d23 · 1d11d23
1 parent 0142aeb
commit 1d11d23
Show file tree

Hide file tree

Showing 9 changed files with 173 additions and 143 deletions.
diff --git a/gatsby-config.ts b/gatsby-config.ts
@@ -16,6 +16,8 @@ process.env.POSTGRAPHILE_PREPARED_STATEMENT_CACHE_SIZE = '1';
 const strapiConfig = {
     apiURL: process.env.STRAPI_API_URL,
     accessToken: process.env.STRAPI_TOKEN,
+    maxParallelRequests: 3,
+    version: 4, // They now assume v5
     collectionTypes: [
         'blog-post',
         'company-update-post',

diff --git a/package.json b/package.json
@@ -48,7 +48,7 @@
     "gatsby-remark-responsive-iframe": "^6.8.0",
     "gatsby-source-filesystem": "^5.13.1",
     "gatsby-source-pg": "^1.0.0-rc.0",
-    "gatsby-source-strapi": "^3.3.3",
+    "gatsby-source-strapi": "^5.0.2",
     "gatsby-transformer-inline-svg": "^1.2.0",
     "gatsby-transformer-rehype": "^2.0.1",
     "gatsby-transformer-remark": "^6.13.1",

diff --git a/src/components/Connectors/index.tsx b/src/components/Connectors/index.tsx
@@ -1,5 +1,5 @@
 import ChevronRight from '@mui/icons-material/ChevronRight';
-import { graphql, Link, useStaticQuery } from 'gatsby';
+import { graphql, useStaticQuery, Link } from 'gatsby';
 import { GatsbyImage, StaticImage } from 'gatsby-plugin-image';
 import { useMemo, useState } from 'react';
 import { useLunr } from 'react-lunr';
@@ -65,48 +65,66 @@ const ConnectorCard = ({
     slug,
     type,
     showType = false,
-}: ReturnType<typeof normalizeConnector> & { showType?: boolean }) => (
-    <Link to={`${slug}`}>
-        <div className={connectorCard}>
-            <div className={connectorCardTop}>
-                <GatsbyImage
-                    image={logo?.childImageSharp?.gatsbyImageData}
-                    alt={`${title} Logo`}
-                    className={clsx(connectorPostCardImage, 'icon-wrapper')}
-                    loading="eager"
-                />
-                {recommended || showType ? (
-                    <div style={{ flexGrow: 1 }} />
-                ) : null}
-                {recommended ? (
-                    <div>
-                        <p className={connectorPostCardRecommended}>
-                            RECOMMENDED
-                        </p>
-                    </div>
-                ) : null}
-                {showType ? (
-                    <>
-                        {recommended ? <div style={{ flexBasis: 4 }} /> : null}
+}: ReturnType<typeof normalizeConnector> & { showType?: boolean }) => {
+    return (
+        <Link to={`${slug}`}>
+            <div className={connectorCard}>
+                <div className={connectorCardTop}>
+                    {!logo?.childImageSharp?.gatsbyImageData ? (
+                        <div
+                            className={clsx(
+                                connectorPostCardImage,
+                                'icon-wrapper'
+                            )}
+                        />
+                    ) : (
+                        <GatsbyImage
+                            image={logo.childImageSharp.gatsbyImageData}
+                            alt={`${title} Logo`}
+                            className={clsx(
+                                connectorPostCardImage,
+                                'icon-wrapper'
+                            )}
+                            loading="eager"
+                        />
+                    )}
+                    {recommended || showType ? (
+                        <div style={{ flexGrow: 1 }} />
+                    ) : null}
+                    {recommended ? (
                         <div>
                             <p className={connectorPostCardRecommended}>
-                                {type === 'capture' ? 'SOURCE' : 'DESTINATION'}
+                                RECOMMENDED
                             </p>
                         </div>
-                    </>
+                    ) : null}
+                    {showType ? (
+                        <>
+                            {recommended ? (
+                                <div style={{ flexBasis: 4 }} />
+                            ) : null}
+                            <div>
+                                <p className={connectorPostCardRecommended}>
+                                    {type === 'capture'
+                                        ? 'SOURCE'
+                                        : 'DESTINATION'}
+                                </p>
+                            </div>
+                        </>
+                    ) : null}
+                </div>
+                <h4>{title}</h4>
+                {shortDescription?.length > 0 ? (
+                    <p>{truncate(shortDescription || '', 100)}</p>
                 ) : null}
+                <div style={{ flexGrow: 1 }} />
+                <span className={connectorCardReadMore}>
+                    Read More <ChevronRight />
+                </span>
             </div>
-            <h4>{title}</h4>
-            {shortDescription?.length > 0 ? (
-                <p>{truncate(shortDescription || '', 100)}</p>
-            ) : null}
-            <div style={{ flexGrow: 1 }} />
-            <span className={connectorCardReadMore}>
-                Read More <ChevronRight />
-            </span>
-        </div>
-    </Link>
-);
+        </Link>
+    );
+};
 
 export const Connectors = ({
     connectorType,
@@ -158,10 +176,6 @@ export const Connectors = ({
     const mappedConnectors = useMemo(
         () =>
             postgres.allConnectors.nodes
-                .filter(
-                    (connector) =>
-                        connector?.connectorTagsByConnectorIdList?.length > 0
-                )
                 .map(normalizeConnector)
                 .filter(
                     (connector) =>
@@ -193,14 +207,12 @@ export const Connectors = ({
         [postgres]
     );
 
-    const logosByConnectorId = useMemo(
-        () =>
-            Object.assign(
-                {},
-                ...mappedConnectors.map((con) => ({ [con.id]: con.logo }))
-            ),
-        [mappedConnectors]
-    );
+    const logosByConnectorId = useMemo(() => {
+        return Object.assign(
+            {},
+            ...mappedConnectors.map((con) => ({ [con.id]: con.logo }))
+        );
+    }, [mappedConnectors]);
 
     const [query, setQuery] = useState('');
     const results = useLunr(
@@ -240,14 +252,16 @@ export const Connectors = ({
                 </div>
                 <div className={connectorCards}>
                     {(query.length > 0 ? results : mappedConnectors).map(
-                        (connector) => (
-                            <ConnectorCard
-                                key={connector.id}
-                                {...connector}
-                                logo={logosByConnectorId[connector.id]}
-                                showType={showAllConnectors}
-                            />
-                        )
+                        (connector) => {
+                            return (
+                                <ConnectorCard
+                                    key={connector.id}
+                                    {...connector}
+                                    logo={logosByConnectorId[connector.id]}
+                                    showType={showAllConnectors}
+                                />
+                            );
+                        }
                     )}
                 </div>
             </div>

diff --git a/src/components/ConnectorsLink/index.tsx b/src/components/ConnectorsLink/index.tsx
@@ -55,10 +55,6 @@ const ConnectorsLink = ({
     const [captureConnectors, materializationConnectors] = useMemo(() => {
         const mapped: ReturnType<typeof normalizeConnector>[] = connectors
             .filter((connector) => connector.logo !== null)
-            .filter(
-                (connector) =>
-                    connector?.connectorTagsByConnectorIdList?.length > 0
-            )
             .map(normalizeConnector)
             .filter((connector) => connector !== undefined);
 

diff --git a/src/templates/connection.tsx b/src/templates/connection.tsx
@@ -44,25 +44,25 @@ const Connector = ({
         <Layout>
             <Hero
                 sourceConnector={{
-                    title: source_mapped.title,
-                    logo: source_mapped.logo,
+                    title: source_mapped?.title,
+                    logo: source_mapped?.logo,
                 }}
                 destConnector={{
-                    title: dest_mapped.title,
-                    logo: dest_mapped.logo,
+                    title: dest_mapped?.title,
+                    logo: dest_mapped?.logo,
                 }}
             />
             <FromConnector
-                title={source_mapped.title}
-                logo={source_mapped.logo}
-                longDescription={source_mapped.longDescription}
-                shortDescription={source_mapped.shortDescription}
+                title={source_mapped?.title}
+                logo={source_mapped?.logo}
+                longDescription={source_mapped?.longDescription}
+                shortDescription={source_mapped?.shortDescription}
             />
             <ToConnector
-                title={dest_mapped.title}
-                logo={dest_mapped.logo}
-                longDescription={dest_mapped.longDescription}
-                shortDescription={dest_mapped.shortDescription}
+                title={dest_mapped?.title}
+                logo={dest_mapped?.logo}
+                longDescription={dest_mapped?.longDescription}
+                shortDescription={dest_mapped?.shortDescription}
             />
             <Testimonials />
             <EstuaryFlowVideo />

diff --git a/src/templates/connector/Hero/index.tsx b/src/templates/connector/Hero/index.tsx
@@ -23,13 +23,13 @@ import {
 type HeroProps = {
     connector: {
         title: string;
-        logo: ImageDataLike;
+        logo: ImageDataLike | null | undefined;
         type: ConnectorType;
     };
 };
 
 const Hero = ({ connector: { title, logo, type } }: HeroProps) => {
-    const logoImage = getImage(logo);
+    const logoImage = logo ? getImage(logo) : null;
 
     return (
         <LightSwoopingLinesRightDirectionBackground>

diff --git a/src/templates/connector/index.tsx b/src/templates/connector/index.tsx
@@ -53,7 +53,7 @@ const Connector = ({
                         href: '/integrations',
                     },
                     {
-                        title: mappedConnector.title,
+                        title: mappedConnector?.title,
                     },
                 ]}
             />
@@ -101,7 +101,7 @@ export const Head = ({
             case 'capture':
                 return [
                     `Move ${mappedConnector?.title} to Any Destination, Real-time ETL & CDC`,
-                    `Effortlessly move ${mappedConnector.title} data to any destination in real-time or batch with Estuary's no-code ETL & CDC platform. Free and easy to use. Get started now.`,
+                    `Effortlessly move ${mappedConnector?.title} data to any destination in real-time or batch with Estuary's no-code ETL & CDC platform. Free and easy to use. Get started now.`,
                 ];
             default:
                 return [

diff --git a/src/utils.ts b/src/utils.ts
@@ -12,8 +12,12 @@ import {
 const CONNECTOR_IMAGE_RE = /(source|materialize)-([a-z0-9\-]+)/;
 
 export const normalizeConnector = (connector: any) => {
-    if (connector.imageName.includes('ghcr.io/estuary/dekaf')) {
+    if (
         // Exclude any Dekaf connector
+        connector.imageName.includes('ghcr.io/estuary/dekaf') ||
+        // Exclude connectors without a tag (Kelkoo)
+        connector?.connectorTagsByConnectorIdList?.length < 1
+    ) {
         return undefined;
     }