32 lines
836 B
JavaScript
32 lines
836 B
JavaScript
/**
|
|
* Builds robots rules that allow crawling everything except the admin panel and
|
|
* API, and point crawlers at the sitemap.
|
|
*
|
|
* ```ts
|
|
* // app/robots.ts
|
|
* export default function robots() {
|
|
* return buildRobots({ baseUrl: process.env.NEXT_PUBLIC_SERVER_URL! })
|
|
* }
|
|
* ```
|
|
*
|
|
* The admin and API are disallowed by default — no reason for them in the index,
|
|
* and it keeps crawl budget on real content.
|
|
*/ export function buildRobots({ baseUrl, disallow = [
|
|
'/admin',
|
|
'/api'
|
|
], sitemap = true }) {
|
|
const origin = baseUrl.replace(/\/$/, '');
|
|
return {
|
|
rules: {
|
|
allow: '/',
|
|
disallow,
|
|
userAgent: '*'
|
|
},
|
|
...sitemap ? {
|
|
sitemap: `${origin}/sitemap.xml`
|
|
} : {},
|
|
host: origin
|
|
};
|
|
}
|
|
|
|
//# sourceMappingURL=buildRobots.js.map
|