fix: resolve symlinked files (#191)

* fix: resolve symlinked files

mcansh/remix-fastify root README is a symlink to the README in packages/remix-fastify and using the raw.github... doesnt follow the redirect

Signed-off-by: Logan McAnsh <logan@mcan.sh>

* chore: use `getReadme` to allow for any valid readme file name

omitting the ref will use the repos default branch

Signed-off-by: Logan McAnsh <logan@mcan.sh>

* chore: pass octokit around

Signed-off-by: Logan McAnsh <logan@mcan.sh>

* chore: re-use CacheContext type

Signed-off-by: Logan McAnsh <logan@mcan.sh>

* chore: use `format: raw`

Signed-off-by: Logan McAnsh <logan@mcan.sh>

---------

Signed-off-by: Logan McAnsh <logan@mcan.sh>

Author: mcansh

app/lib/gh-docs/branches.ts

@@ -1,7 +1,6 @@
 import { LRUCache } from "lru-cache";
-import type { Octokit } from "octokit";
+import type { CacheContext } from ".";
 
-type CacheContext = { octokit: Octokit };
 declare global {
   var branchesCache: LRUCache<string, string[], CacheContext>;
 }

app/lib/gh-docs/docs.ts

@@ -6,6 +6,7 @@ import { getRepoTarballStream } from "./repo-tarball";
 import { createTarFileProcessor } from "./tarball.server";
 import { load as $ } from "cheerio";
 import { env } from "~/env.server";
+import type { CacheContext } from ".";
 
 interface MenuDocAttributes {
   title: string;
@@ -29,7 +30,7 @@ export interface Doc extends Omit<MenuDoc, "hasContent"> {
 
 declare global {
   var menuCache: LRUCache<string, MenuDoc[]>;
-  var docCache: LRUCache<string, Doc>;
+  var docCache: LRUCache<string, Doc, CacheContext>;
 }
 
 let NO_CACHE = env.NO_CACHE;
@@ -81,19 +82,22 @@ function parseAttrs(
  * let's have simpler and faster deployments with just one origin server, but
  * still distribute the documents across the CDN.
  */
-global.docCache ??= new LRUCache<string, Doc>({
-  // let docCache = new LRUCache<string, Doc | undefined>({
+global.docCache ??= new LRUCache<string, Doc, CacheContext>({
   max: 300,
   ttl: NO_CACHE ? 1 : 1000 * 60 * 5, // 5 minutes
   allowStale: !NO_CACHE,
   noDeleteOnFetchRejection: true,
   fetchMethod: fetchDoc,
 });
 
-async function fetchDoc(key: string): Promise<Doc> {
+async function fetchDoc(
+  key: string,
+  _staleValue: Doc | undefined,
+  { context }: LRUCache.FetchOptionsWithContext<string, Doc, CacheContext>,
+): Promise<Doc> {
   let [repo, ref, slug] = key.split(":");
   let filename = `${slug}.md`;
-  let md = await getRepoContent(repo, ref, filename);
+  let md = await getRepoContent(repo, ref, filename, context);
   if (md === null) {
     throw Error(`Could not find ${filename} in ${repo}@${ref}`);
   }
@@ -124,9 +128,10 @@ export async function getDoc(
   repo: string,
   ref: string,
   slug: string,
+  context: CacheContext,
 ): Promise<Doc | undefined> {
   let key = `${repo}:${ref}:${slug}`;
-  let doc = await docCache.fetch(key);
+  let doc = await docCache.fetch(key, { context });
 
   return doc || undefined;
 }

app/lib/gh-docs/index.ts

@@ -15,6 +15,8 @@ export {
 
 export type { Doc } from "./docs";
 
+export type CacheContext = { octokit: Octokit };
+
 const REPO = env.SOURCE_REPO;
 const RELEASE_PACKAGE = env.RELEASE_PACKAGE;
 
@@ -48,8 +50,8 @@ export function getRepoDocsMenu(ref: string, lang: string) {
   return getMenu(REPO, fixupRefName(ref), lang);
 }
 
-export function getRepoDoc(ref: string, slug: string) {
-  return getDoc(REPO, fixupRefName(ref), slug);
+export function getRepoDoc(ref: string, slug: string, context: CacheContext) {
+  return getDoc(REPO, fixupRefName(ref), slug, context);
 }
 
 function fixupRefName(ref: string) {

app/lib/gh-docs/repo-content.ts

@@ -3,6 +3,8 @@ import path from "path";
 import invariant from "tiny-invariant";
 import { env } from "~/env.server";
 
+import type { CacheContext } from ".";
+
 /**
  * Fetches the contents of a file in a repository or from your local disk.
  *
@@ -14,16 +16,22 @@ export async function getRepoContent(
   repoPair: string,
   ref: string,
   filepath: string,
+  context: CacheContext,
 ): Promise<string | null> {
   if (ref === "local") return getLocalContent(filepath);
   let [owner, repo] = repoPair.split("/");
-  let pathname = `/${owner}/${repo}/${ref}/${filepath}`;
-  let response = await fetch(
-    new URL(pathname, "https://raw.githubusercontent.com/").href,
-    { headers: { "User-Agent": `docs:${owner}/${repo}` } },
-  );
-  if (!response.ok) return null;
-  return response.text();
+  let contents = await context.octokit.rest.repos.getContent({
+    owner,
+    repo,
+    ref,
+    path: filepath,
+    mediaType: { format: "raw" },
+  });
+
+  // when using `format: raw` the data property is the file contents
+  let md = contents.data as unknown;
+  if (md == null || typeof md !== "string") return null;
+  return md;
 }
 
 /**

app/lib/gh-docs/tags.ts

@@ -2,8 +2,9 @@ import { LRUCache } from "lru-cache";
 import parseLinkHeader from "parse-link-header";
 import semver from "semver";
 import type { Octokit } from "octokit";
+import type { CacheContext as BaseCacheContext } from ".";
 
-type CacheContext = { octokit: Octokit; releasePackage: string };
+type CacheContext = BaseCacheContext & { releasePackage: string };
 declare global {
   var tagsCache: LRUCache<string, string[], CacheContext>;
 }

app/lib/resources.server.ts

@@ -1,11 +1,10 @@
 import yaml from "yaml";
 import { LRUCache } from "lru-cache";
 import { env } from "~/env.server";
-import { getRepoContent } from "./gh-docs/repo-content";
 import { processMarkdown } from "./md.server";
-import type { Octokit } from "octokit";
 import resourcesYamlFileContents from "../../data/resources.yaml?raw";
 import { slugify } from "~/ui/primitives/utils";
+import type { CacheContext } from "./gh-docs";
 
 // TODO: parse this with zod
 let _resources: ResourceYamlData[] = yaml.parse(resourcesYamlFileContents);
@@ -36,7 +35,6 @@ type ResourceYamlKeys =
   | "featured";
 type ResourceYamlData = Pick<Resource, ResourceYamlKeys>;
 type ResourceGitHubData = Omit<Resource, ResourceYamlKeys>;
-type CacheContext = { octokit: Octokit };
 
 /**
  * Gets all of the resources, fetching and merging GitHub data for each one
@@ -73,7 +71,7 @@ export async function getResource(
 
   let [gitHubData, readmeHtml] = await Promise.all([
     getResourceGitHubData(resource.repoUrl, { octokit }),
-    getResourceReadme(resource.repoUrl),
+    getResourceReadme(resource.repoUrl, { octokit }),
   ]);
 
   if (!gitHubData || !readmeHtml) {
@@ -86,7 +84,7 @@ export async function getResource(
 //#region LRUCache and fetchers for GitHub data and READMEs
 
 declare global {
-  var resourceReadmeCache: LRUCache<string, string>;
+  var resourceReadmeCache: LRUCache<string, string, CacheContext>;
   var resourceGitHubDataCache: LRUCache<
     string,
     ResourceGitHubData,
@@ -96,26 +94,38 @@ declare global {
 
 let NO_CACHE = env.NO_CACHE;
 
-global.resourceReadmeCache ??= new LRUCache<string, string>({
+global.resourceReadmeCache ??= new LRUCache<string, string, CacheContext>({
   max: 300,
   ttl: NO_CACHE ? 1 : 1000 * 60 * 5, // 5 minutes
   allowStale: !NO_CACHE,
   noDeleteOnFetchRejection: true,
   fetchMethod: fetchReadme,
 });
 
-async function fetchReadme(repo: string): Promise<string> {
-  let md = await getRepoContent(repo, "main", "README.md");
-  if (md === null) {
-    throw Error(`Could not find README in ${repo}`);
+async function fetchReadme(
+  key: string,
+  _staleValue: string | undefined,
+  { context }: LRUCache.FetchOptionsWithContext<string, string, CacheContext>,
+): Promise<string> {
+  let [owner, repo] = key.split("/");
+  let contents = await context.octokit.rest.repos.getReadme({
+    owner,
+    repo,
+    mediaType: { format: "raw" },
+  });
+
+  // when using `format: raw` the data property is the file contents
+  let md = contents.data as unknown;
+  if (md == null || typeof md !== "string") {
+    throw Error(`Could not find README in ${key}`);
   }
   let { html } = await processMarkdown(md);
   return html;
 }
 
-async function getResourceReadme(repoUrl: string) {
+async function getResourceReadme(repoUrl: string, context: CacheContext) {
   let repo = repoUrl.replace("https://github.com/", "");
-  let doc = await resourceReadmeCache.fetch(repo);
+  let doc = await resourceReadmeCache.fetch(repo, { context });
 
   return doc || undefined;
 }

app/routes/$.tsx

@@ -2,6 +2,7 @@ import { handleRedirects } from "~/lib/http.server";
 import type { LoaderFunctionArgs } from "@remix-run/node";
 import { redirect, json } from "@remix-run/node";
 import { getRepoDoc } from "~/lib/gh-docs";
+import { octokit } from "~/lib/github.server";
 
 // We use the catch-all route to attempt to find a doc for the given path. If a
 // doc isn't found, we return a 404 as expected. However we also log those
@@ -55,7 +56,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   try {
     let ref = "main";
     let lang = "en";
-    let doc = await getRepoDoc(ref, `docs/${params["*"]}`);
+    let doc = await getRepoDoc(ref, `docs/${params["*"]}`, { octokit });
     if (!doc) throw null;
     // FIXME: This results in two fetches, as the loader for the docs page will
     // repeat the request cycle. This isn't a problem if the doc is in the LRU

app/routes/docs.$lang.$ref.$.tsx

@@ -24,6 +24,7 @@ import cx from "clsx";
 import { useDelegatedReactRouterLinks } from "~/ui/delegate-links";
 import type { loader as docsLayoutLoader } from "~/routes/docs.$lang.$ref";
 import type { loader as rootLoader } from "~/root";
+import { octokit } from "~/lib/github.server";
 
 export async function loader({ params, request }: LoaderFunctionArgs) {
   let url = new URL(request.url);
@@ -33,7 +34,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     let slug = params["*"]?.endsWith("/changelog")
       ? "CHANGELOG"
       : `docs/${params["*"] || "index"}`;
-    let doc = await getRepoDoc(params.ref, slug);
+    let doc = await getRepoDoc(params.ref, slug, { octokit });
     if (!doc) throw null;
     return json(
       { doc, pageUrl },