deno.land / x / lume@v2.1.4 / plugins / robots.ts
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899import { merge } from "../core/utils/object.ts";
import type Site from "../core/site.ts";
type Rule = { /** User-agent */ userAgent?: string; /** Crawl-delay */ crawlDelay?: string; /** Disallow */ disallow?: string; /** Disavow */ disavow?: string; /** Allow */ allow?: string; /** Host */ host?: string; /** Sitemap */ sitemap?: string; /** Clean-param */ cleanParam?: string;};
const ruleSort = [ "userAgent", "crawlDelay", "disallow", "disavow", "allow", "host", "sitemap", "cleanParam",];
export interface Options { /** The robots.txt file name */ filename: string; allow?: string[] | string; disallow?: string[] | string; rules?: Rule[];}
// Default optionsexport const defaults: Options = { filename: "/robots.txt", allow: "*",};
/** A plugin to generate a robots.txt after build */export default (userOptions?: Partial<Options>) => { const options: Options = merge(defaults, userOptions);
return (site: Site) => { site.addEventListener("beforeSave", async () => { const rules: Rule[] = []; const allow = typeof options.allow === "string" ? [options.allow] : options.allow; const disallow = typeof options.disallow === "string" ? [options.disallow] : options.disallow;
allow?.forEach((userAgent) => rules.push({ userAgent, allow: "/", }) );
disallow?.forEach((userAgent) => rules.push({ userAgent, disallow: "/", }) );
rules.push(...(options.rules ?? []));
// Create the robots.txt page const robots = await site.getOrCreatePage(options.filename); const existingContent = robots.content ? `${robots.content}\n` : "";
robots.content = existingContent + rules .map((rule) => Object.entries(rule) .sort(([keyA], [keyB]) => ruleSort.indexOf(keyA) - ruleSort.indexOf(keyB) ) .map(([key, value]) => `${key.charAt(0).toUpperCase()}${ key.slice(1).replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase() }: ${value}` ) .join("\n") ).join("\n\n"); }); };};
Version Info