Initial commit

dist/modules/seo/buildRobots.js

@@ -1,32 +0,0 @@
-/**
- * Builds robots rules that allow crawling everything except the admin panel and
- * API, and point crawlers at the sitemap.
- *
- * ```ts
- * // app/robots.ts
- * export default function robots() {
- *   return buildRobots({ baseUrl: process.env.NEXT_PUBLIC_SERVER_URL! })
- * }
- * ```
- *
- * The admin and API are disallowed by default — no reason for them in the index,
- * and it keeps crawl budget on real content.
- */ export function buildRobots({ baseUrl, disallow = [
-    '/admin',
-    '/api'
-], sitemap = true }) {
-    const origin = baseUrl.replace(/\/$/, '');
-    return {
-        rules: {
-            allow: '/',
-            disallow,
-            userAgent: '*'
-        },
-        ...sitemap ? {
-            sitemap: `${origin}/sitemap.xml`
-        } : {},
-        host: origin
-    };
-}
-
-//# sourceMappingURL=buildRobots.js.map