1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import { merge } from "../core/utils/object.ts";
import type Site from "../core/site.ts";
type Rule = {  /** User-agent */  userAgent?: string;  /** Crawl-delay */  crawlDelay?: string;  /** Disallow */  disallow?: string;  /** Disavow */  disavow?: string;  /** Allow */  allow?: string;  /** Host */  host?: string;  /** Sitemap */  sitemap?: string;  /** Clean-param */  cleanParam?: string;};
const ruleSort = [  "userAgent",  "crawlDelay",  "disallow",  "disavow",  "allow",  "host",  "sitemap",  "cleanParam",];
export interface Options {  /** The robots.txt file name */  filename: string;  allow?: string[] | string;  disallow?: string[] | string;  rules?: Rule[];}
// Default optionsexport const defaults: Options = {  filename: "/robots.txt",  allow: "*",};
/** A plugin to generate a robots.txt after build */export default (userOptions?: Partial<Options>) => {  const options: Options = merge(defaults, userOptions);
  return (site: Site) => {    site.addEventListener("beforeSave", async () => {      const rules: Rule[] = [];      const allow = typeof options.allow === "string"        ? [options.allow]        : options.allow;      const disallow = typeof options.disallow === "string"        ? [options.disallow]        : options.disallow;
      allow?.forEach((userAgent) =>        rules.push({          userAgent,          allow: "/",        })      );
      disallow?.forEach((userAgent) =>        rules.push({          userAgent,          disallow: "/",        })      );
      rules.push(...(options.rules ?? []));
      // Create the robots.txt page      const robots = await site.getOrCreatePage(options.filename);      const existingContent = robots.content ? `${robots.content}\n` : "";
      robots.content = existingContent + rules        .map((rule) =>          Object.entries(rule)            .sort(([keyA], [keyB]) =>              ruleSort.indexOf(keyA) - ruleSort.indexOf(keyB)            )            .map(([key, value]) =>              `${key.charAt(0).toUpperCase()}${                key.slice(1).replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase()              }: ${value}`            )            .join("\n")        ).join("\n\n");    });  };};