TASKS-1823 WebManager module

apps/web/app/robots.ts

@@ -0,0 +1,17 @@
+import type { MetadataRoute } from 'next';
+import { siteUrl } from '#web-runtime/lib/site-url';
+
+export default function robots(): MetadataRoute.Robots {
+  const origin = siteUrl();
+
+  return {
+    rules: {
+      userAgent: '*',
+      allow: '/',
+      // Preview URLs render unpublished drafts behind a signed token; they must never be indexed.
+      // The page also sends `noindex`, this just saves crawlers the fetch.
+      disallow: '/*/preview'
+    },
+    sitemap: new URL('/sitemap.xml', origin).href
+  };
+}