Add initial project files

apps/web/tests/robots.test.ts

@@ -0,0 +1,26 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import robots from '#app/robots';
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+});
+
+describe('robots.txt', () => {
+  it('lets crawlers in everywhere except preview URLs', () => {
+    vi.stubEnv('SITE_URL', 'https://crux.example.com');
+
+    const result = robots();
+
+    expect(result.rules).toEqual({
+      userAgent: '*',
+      allow: '/',
+      disallow: '/*/preview'
+    });
+  });
+
+  it('advertises the sitemap on the configured origin', () => {
+    vi.stubEnv('SITE_URL', 'https://crux.example.com');
+
+    expect(robots().sitemap).toBe('https://crux.example.com/sitemap.xml');
+  });
+});