lax robots

src/routes/robots[.]txt.ts

@@ -7,16 +7,6 @@ export const loader: LoaderFunction = async ({ request }) => {
 # https://www.robotstxt.org/robotstxt.html
 User-agent: *
 Allow: /
-Crawl-delay: 10
-
-# Marketing & Analytics Bots
-User-agent: AdsBot-Google
-User-agent: AdsBot-Google-Mobile
-User-agent: Mediapartners-Google
-User-agent: LinkedInBot
-User-agent: facebookexternalhit
-User-agent: Twitterbot
-Allow: /
 
 # Security exclusions
 Disallow: /admin/
@@ -25,27 +15,6 @@ Disallow: /banking-applications/
 Disallow: /applications/
 Disallow: /staging/
 
-# Query parameters
-Disallow: /*?*
-Allow: /*?utm_*    # Google Analytics UTM parameters
-Allow: /*?fbclid*  # Facebook click identifier
-Allow: /*?li_*     # LinkedIn tracking parameters
-Allow: /*?twclid*  # Twitter click identifier
-
-# Block specific file types
-Disallow: /*.json$
-Disallow: /*.xml$
-
-# Media files
-User-agent: Googlebot-Image
-Allow: /images/
-Allow: /assets/images/
-Allow: /*.jpg$
-Allow: /*.jpeg$
-Allow: /*.gif$
-Allow: /*.png$
-Allow: /*.webp$
-
 Sitemap: ${origin}/sitemap.xml
 `;