GTM-130: Add llms*.txt

.github/workflows/update-flake-lock.yml

@@ -59,7 +59,7 @@ jobs:
           committer: github-actions[bot] github-actions[bot]@users.noreply.github.com
           assignees: lucperkins
           reviewers: lucperkins
-          labels:
+          labels: |
             automated
             templates
             update

.npmrc

@@ -1,2 +1 @@
-auto-install-peers = true
 shamefully-hoist = true

.prettierignore

@@ -7,3 +7,4 @@ pnpm-lock.yaml
 **/*.mdx
 src/components/Posthog.astro
 **/*.lock
+**/*.hbs

package.json

@@ -34,6 +34,7 @@
     "astro-expressive-code": "^0.38.3",
     "astro-icon": "^1.1.5",
     "astro-seo": "^0.8.4",
+    "handlebars": "^4.7.8",
     "marked": "^12.0.2",
     "posthog-js": "^1.266.0",
     "react": "^18.3.1",

pnpm-workspace.yaml

@@ -1,2 +1,2 @@
 overrides:
-  devalue@<5.3.2: '>=5.3.2'
+  devalue@<5.3.2: ">=5.3.2"

src/components/Footer.astro

@@ -7,15 +7,39 @@ const currentYear = new Date().getFullYear();
 
 <footer>
   <HorizontalContainer>
-    <div class="flex justify-end pb-16 pt-12">
-      <p
-        class="tracking-tight text-gray dark:text-gray md:text-base lg:text-lg"
-      >
-        Copyright &copy; {currentYear}
-        <Link href="https://determinate.systems" class="hover:text-primary">
-          Determinate Systems, Inc.
-        </Link>
-      </p>
+    <div class="pb-16 pt-12">
+      <div class="flex items-center justify-between">
+        <div class="flex items-center gap-4">
+          <strong>AI tools</strong>
+          <ul class="flex items-center gap-2">
+            {
+              ["llms.txt", "llms-small.txt", "llms-full.txt"].map((file) => (
+                <li>
+                  <a
+                    href={`/${file}`}
+                    class="text-sm text-gray duration-hover hover:text-nix-light-blue dark:hover:text-orange"
+                  >
+                    <code>{file}</code>
+                  </a>
+                </li>
+              ))
+            }
+          </ul>
+        </div>
+        <div>
+          <p
+            class="tracking-tight text-gray dark:text-gray md:text-base lg:text-lg"
+          >
+            Copyright &copy; {currentYear}
+            <Link
+              href="https://determinate.systems"
+              class="duration-hover hover:text-primary dark:hover:text-orange"
+            >
+              Determinate Systems, Inc.
+            </Link>
+          </p>
+        </div>
+      </div>
     </div>
   </HorizontalContainer>
 </footer>

src/components/FormattedDate.astro

@@ -8,7 +8,7 @@ const { date } = Astro.props;
 
 <time datetime={date.toISOString()}>
   {
-    date.toLocaleDateString("en-us", {
+    date.toLocaleDateString("en-US", {
       year: "numeric",
       month: "short",
       day: "numeric",

src/lib/utils.ts

@@ -15,3 +15,11 @@ export const startPagePath = (slug: string): string => {
 const pagePath = (collection: string, slug: string): string => {
   return `/${collection}/${slug}`;
 };
+
+export const formatDate = (date: Date): string => {
+  return date.toLocaleDateString("en-US", {
+    year: "numeric",
+    month: "long",
+    day: "numeric",
+  });
+};

src/pages/llms-full.txt.ts

@@ -0,0 +1,52 @@
+import { conceptPagePath, startPagePath } from "../lib/utils";
+import { site } from "../site";
+import { FORMATS } from "./llms.txt";
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import Handlebars from "handlebars";
+import fs from "node:fs";
+import path from "node:path";
+
+const { url: root, title, description: tagline, llms } = site;
+
+const templateFile = fs.readFileSync(
+  path.join(process.cwd(), "src/templates/llms-full.txt.hbs"),
+  "utf-8",
+);
+
+const template = Handlebars.compile(templateFile);
+
+export const GET: APIRoute = async () => {
+  const startPages = await getCollection("start");
+  const conceptPages = await getCollection("concepts");
+
+  const content = template({
+    root,
+    title,
+    system:
+      "This is the complete documentation for Zero to Nix, including the learning journey (start) and concept pages.",
+    instructions:
+      "Learning journey (start) and concept page items begin and end with a --- and all page titles are in bold.",
+    tagline,
+    description: llms.description,
+    startPages: startPages.map(({ data: { title }, slug, body }) => ({
+      title,
+      href: `${root}/${startPagePath(slug)}`,
+      content: body,
+    })),
+    conceptPages: conceptPages.map(({ data: { title }, slug, body }) => ({
+      title,
+      href: `${root}/${conceptPagePath(slug)}`,
+      content: body,
+    })),
+    otherFormats: [FORMATS.small, FORMATS.standard],
+  });
+
+  return new Response(content, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+    },
+  });
+};
+
+export const prerender = true;

src/pages/llms-small.txt.ts

@@ -0,0 +1,42 @@
+import { site } from "../site";
+import { FORMATS } from "./llms.txt";
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import Handlebars from "handlebars";
+import fs from "node:fs";
+import path from "node:path";
+
+const { url: root, title, llms } = site;
+
+const templateFile = fs.readFileSync(
+  path.join(process.cwd(), "src/templates/llms-small.txt.hbs"),
+  "utf-8",
+);
+
+const template = Handlebars.compile(templateFile);
+
+export const GET: APIRoute = async () => {
+  const startPages = await getCollection("start");
+  const conceptPages = await getCollection("concepts");
+
+  const content = template({
+    root,
+    title,
+    description: llms.description,
+    startPages: startPages.map(({ data: { title } }) => ({
+      title,
+    })),
+    conceptPages: conceptPages.map(({ data: { title } }) => ({
+      title,
+    })),
+    otherFormats: [FORMATS.standard, FORMATS.full],
+  });
+
+  return new Response(content, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+    },
+  });
+};
+
+export const prerender = true;

src/pages/llms.txt.ts

@@ -0,0 +1,66 @@
+import { conceptPagePath, startPagePath } from "../lib/utils";
+import { site } from "../site";
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import Handlebars from "handlebars";
+import fs from "node:fs";
+import path from "node:path";
+
+const { url: root, title, description: tagline, llms } = site;
+
+export const FORMATS: Record<
+  string,
+  { root: string; file: string; description: string }
+> = {
+  standard: {
+    root,
+    file: "llms.txt",
+    description: "the main version",
+  },
+  small: {
+    root,
+    file: "llms-small.txt",
+    description: "compact structure-only version",
+  },
+  full: {
+    root,
+    file: "llms-full.txt",
+    description: "complete content in one file",
+  },
+};
+
+const templateFile = fs.readFileSync(
+  path.join(process.cwd(), "src/templates/llms.txt.hbs"),
+  "utf-8",
+);
+
+const template = Handlebars.compile(templateFile);
+
+export const GET: APIRoute = async () => {
+  const startPages = await getCollection("start");
+  const conceptPages = await getCollection("concepts");
+
+  const content = template({
+    root,
+    title,
+    tagline,
+    description: llms.description,
+    startPages: startPages.map(({ data: { title }, slug }) => ({
+      title,
+      href: `${root}/${startPagePath(slug)}`,
+    })),
+    conceptPages: conceptPages.map(({ data: { title }, slug }) => ({
+      title,
+      href: `${root}/${conceptPagePath(slug)}`,
+    })),
+    otherFormats: [FORMATS.small, FORMATS.full],
+  });
+
+  return new Response(content, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+    },
+  });
+};
+
+export const prerender = true;