🐛 only do the news search params and instant search #2682 (#2684)

* 🐛 only do the news search params and instant search * 🐛 missed disallow
equinor · Nov 20, 2024 · 4f00a54 · 4f00a54
1 parent 1f9b518
commit 4f00a54
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/web/pages/robots.txt.ts b/web/pages/robots.txt.ts
@@ -20,7 +20,11 @@ export const crawlableDomains = [
 
 const robots = (domain: string) => `User-agent: *
 ${crawlableDomains.includes(domain) ? 'Allow' : 'Disallow'}: /
-${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html \nDisallow: /*?*' : ''}
+${
+  domain === 'www.equinor.com'
+    ? `Disallow: /search \nDisallow: /search.html \nDisallow: /?*topic \nDisallow: /?*country \nDisallow: /?*year \nDisallow: /?*refinementList \nDisallow: /?*sortBy`
+    : ''
+}
 Sitemap: ${domain.startsWith('www') ? `https://${domain}` : domain}/sitemap.xml
 `