Add llms.txt (#8267)

Co-authored-by: artimath <ryanjhunter@gmail.com>

next.config.js

@@ -19,6 +19,14 @@ const nextConfig = {
     scrollRestoration: true,
     reactCompiler: true,
   },
+  async rewrites() {
+    return [
+      {
+        source: '/:path*.md',
+        destination: '/api/md/:path*',
+      },
+    ];
+  },
   env: {},
   webpack: (config, {dev, isServer, ...options}) => {
     if (process.env.ANALYZE) {

src/pages/api/md/[...path].ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {NextApiRequest, NextApiResponse} from 'next';
+import fs from 'fs';
+import path from 'path';
+
+export default function handler(req: NextApiRequest, res: NextApiResponse) {
+  const pathSegments = req.query.path;
+  if (!pathSegments) {
+    return res.status(404).send('Not found');
+  }
+
+  const filePath = Array.isArray(pathSegments)
+    ? pathSegments.join('/')
+    : pathSegments;
+
+  // Block /index.md URLs - use /foo.md instead of /foo/index.md
+  if (filePath.endsWith('/index') || filePath === 'index') {
+    return res.status(404).send('Not found');
+  }
+
+  // Try exact path first, then with /index
+  const candidates = [
+    path.join(process.cwd(), 'src/content', filePath + '.md'),
+    path.join(process.cwd(), 'src/content', filePath, 'index.md'),
+  ];
+
+  for (const fullPath of candidates) {
+    try {
+      const content = fs.readFileSync(fullPath, 'utf8');
+      res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+      res.setHeader('Cache-Control', 'public, max-age=3600');
+      return res.status(200).send(content);
+    } catch {
+      // Try next candidate
+    }
+  }
+
+  res.status(404).send('Not found');
+}

src/pages/llms.txt.tsx

@@ -0,0 +1,85 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {GetServerSideProps} from 'next';
+import {siteConfig} from '../siteConfig';
+import sidebarLearn from '../sidebarLearn.json';
+import sidebarReference from '../sidebarReference.json';
+import sidebarBlog from '../sidebarBlog.json';
+
+interface RouteItem {
+  title?: string;
+  path?: string;
+  routes?: RouteItem[];
+}
+
+interface Sidebar {
+  title: string;
+  routes: RouteItem[];
+}
+
+function extractRoutes(
+  routes: RouteItem[],
+  baseUrl: string
+): {title: string; url: string}[] {
+  const result: {title: string; url: string}[] = [];
+
+  for (const route of routes) {
+    if (route.title && route.path) {
+      result.push({
+        title: route.title,
+        url: `${baseUrl}${route.path}.md`,
+      });
+    }
+    if (route.routes) {
+      result.push(...extractRoutes(route.routes, baseUrl));
+    }
+  }
+
+  return result;
+}
+
+const sidebars: Sidebar[] = [
+  sidebarLearn as Sidebar,
+  sidebarReference as Sidebar,
+  sidebarBlog as Sidebar,
+];
+
+export const getServerSideProps: GetServerSideProps = async ({res}) => {
+  const subdomain =
+    siteConfig.languageCode === 'en' ? '' : siteConfig.languageCode + '.';
+  const baseUrl = 'https://' + subdomain + 'react.dev';
+
+  const lines = [
+    '# React Documentation',
+    '',
+    '> The library for web and native user interfaces.',
+  ];
+
+  for (const sidebar of sidebars) {
+    lines.push('');
+    lines.push(`## ${sidebar.title}`);
+    lines.push('');
+
+    const routes = extractRoutes(sidebar.routes, baseUrl);
+    for (const route of routes) {
+      lines.push(`- [${route.title}](${route.url})`);
+    }
+  }
+
+  const content = lines.join('\n');
+
+  res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+  res.write(content);
+  res.end();
+
+  return {props: {}};
+};
+
+export default function LlmsTxt() {
+  return null;
+}