// The Ladder (git notes) against a real Artifacts repo -- the piece // flagged as harder than the changeset-commit path, now proven: the // native ARTIFACTS binding's read methods (log/readCommit/readTree) // only resolve the default branch, confirmed directly (a notes ref // returns an empty log no matter how it's spelled) -- notes need // isomorphic-git for both reads and writes. Unlike the main branch, // though, a notes ref's own history is small (one commit per past // Ladder write, not the project's full development history), so a // real, scoped `git.fetch` of just that one ref is cheap regardless // of how large or old the repo gets -- no reason to hold it to the // same no-clone constraint the changeset-commit path needed. // // Originally scoped to just what agent/scriptedWork.ts's own live // agent loop writes -- "file" rungs (anchored to a path's blob at // HEAD) and "changeset" rungs (anchored to the commit itself, with // "filechange" children embedded inline, never as separate notes -- // matching engine/ladderNotes.ts's own anchor rules). Now also covers // what cli/commands/ladder-build.ts's remote-targeting onboarding // needs: "directory"/"ask"/"contributor"/"dependencies" (none of // these are a real path in the tree, so -- same reasoning as "repo" -- // they anchor to the repo's own fixed root commit, not a blob) and // "release" (anchored to the real commit its tag points to, supplied // via `sources` the same way "changeset" already is). // // Unlike engine/ladderNotes.ts's local equivalent, which gives each // ask/contributor/dependency its own per-item notes ref, // "directory"/"ask"/"contributor"/"dependencies" all share the one // "code" ref here, same as "file"/"repo" -- a real round trip per // distinct ref is the documented cost on this path (see NotesCache's // own comment below), and a large onboarding run creating one new ref // per issue/contributor would multiply that cost for no real benefit // remotely. "release" gets its own ref, matching local. import * as git from "isomorphic-git"; import http from "isomorphic-git/http/web"; import { createMemFs } from "./memFs.js"; import type { ArtifactsRepoHandle } from "./cloudflareGitBackend.js"; export interface SimpleRung { path: string; kind: "repo" | "file" | "changeset" | "filechange" | "directory" | "release" | "ask" | "contributor" | "dependencies"; summary: string; sources: { kind: "commit"; ref: string; title: string; author: string; at: string }[]; children: SimpleRung[]; } export interface EmbeddedRef { ref: string; } export interface EmbeddedRung { path: string; kind: string; summary: string; sources: unknown[]; children: (EmbeddedRung | EmbeddedRef)[]; } function isRef(x: EmbeddedRung | EmbeddedRef): x is EmbeddedRef { return "ref" in x; } function notesRefFor(kind: SimpleRung["kind"]): string { if (kind === "changeset") return "refs/notes/zz/ladder/changeset"; if (kind === "release") return "refs/notes/zz/ladder/release"; return "refs/notes/zz/ladder/code"; } // None of these correspond to one real file blob -- same reasoning // "repo" already anchors to the fixed root commit for (a directory // listing/an issue thread/a contributor/a dependency all change // independently of any one commit, so anchoring to HEAD's own tree // would silently orphan the note on the very next unrelated commit). // "release" is anchored here too, by the same path-derivability // constraint changeset's own anchor sidesteps by embedding the sha // directly in its path ("changesets/") -- a release's path // ("releases/") doesn't carry a commit sha the read side could // recover without a real tag lookup, which the remote repo may not // even have pushed tags to resolve. The real commit a release maps to // is still recorded in its own `sources`, just not used as the note's // anchor. const ROOT_ANCHORED_KINDS = new Set(["repo", "directory", "ask", "contributor", "dependencies", "release"]); function embed(rung: SimpleRung): EmbeddedRung { return { path: rung.path, kind: rung.kind, summary: rung.summary, sources: rung.sources, children: rung.children.map(embed) }; } function tokenString(token: { plaintext: string } | string): string { return typeof token === "string" ? token.split("?")[0] : token.plaintext; } // repo.readTree only ever returns one level -- confirmed live (threw // "app/LICENSE not found" for a real file one directory down, even // though it exists) once onboarding started writing file rungs // outside a flat, single-directory repo. Walks one path segment at a // time rather than flattening the whole tree (cloudflareGitBackend.ts's // own flattenTree, built for a different caller that needs every path // at once): this only ever needs the one entry being anchored. async function anchorShaForFile(repo: ArtifactsRepoHandle, path: string): Promise { const [headCommit] = await repo.log({ ref: "main", limit: 1 }); if (!headCommit) throw new Error("repo has no commits yet"); const segments = path.split("/"); let treeHash = headCommit.treeHash; for (let i = 0; i < segments.length; i++) { const tree = await repo.readTree(treeHash); const entry = tree?.find((e) => e.name === segments[i]); if (!entry) throw new Error(`"${path}" not found in the repo at HEAD`); if (i === segments.length - 1) return entry.hash; if (entry.type !== "tree") throw new Error(`"${path}" not found in the repo at HEAD`); treeHash = entry.hash; } throw new Error(`"${path}" not found in the repo at HEAD`); } // The repo's first commit (no parents) -- a fixed point that never // changes for the life of the repo, the anchor "repo"-kind rungs use // (matching engine/ladderNotes.ts's anchorSha, which has the same // reason: HEAD's tree sha changes on every commit, which would // silently orphan _repo's note on the very next one). repo.log's // `limit` caps at 1000 per Cloudflare's docs; fine for a demo-scale // repo, a real limitation for anything with deeper history -- walking // further back would need repo.log's pagination, not added here since // nothing in the live agent loop has needed it yet. export async function rootCommitOid(repo: ArtifactsRepoHandle): Promise { const history = await repo.log({ ref: "main", limit: 1000 }); const root = history[history.length - 1]; if (!root) throw new Error("repo has no commits yet"); return root.hash; } async function anchorShaForRung(repo: ArtifactsRepoHandle, rung: Pick): Promise { if (ROOT_ANCHORED_KINDS.has(rung.kind)) return rootCommitOid(repo); if (rung.kind === "changeset" || rung.kind === "release") { const source = rung.sources.find((s) => s.kind === "commit"); if (!source) throw new Error(`${rung.path}: no commit source to anchor a note to`); return source.ref; } return anchorShaForFile(repo, rung.path); } /** A fresh, throwaway isomorphic-git dir with just one notes ref fetched into it (or empty, if that ref doesn't exist on the remote yet) -- never the main branch, never the rest of history. */ async function scopedNotesCheckout(remote: string, token: { plaintext: string } | string, ref: string): Promise<{ fs: ReturnType; dir: string; hadExisting: boolean }> { const fs = createMemFs(); const dir = "/notes"; await git.init({ fs, dir, defaultBranch: "main" }); await git.addRemote({ fs, dir, remote: "origin", url: remote }); const onAuth = () => ({ username: "x-token-auth", password: tokenString(token) }); try { const result = await git.fetch({ fs, http, dir, remote: "origin", onAuth, ref, remoteRef: ref, singleBranch: true, tags: false }); if (result.fetchHead) { await git.writeRef({ fs, dir, ref, value: result.fetchHead, force: true }); return { fs, dir, hadExisting: true }; } return { fs, dir, hadExisting: false }; } catch { // The ref genuinely doesn't exist yet on the remote (first write ever) -- start from nothing, same as a fresh repo's first note. return { fs, dir, hadExisting: false }; } } async function pushNote(remote: string, token: { plaintext: string } | string, fs: ReturnType, dir: string, ref: string, anchorSha: string, content: unknown): Promise { const person = { name: "zz-ladder", email: "zz-ladder@local", timestamp: Math.floor(Date.now() / 1000), timezoneOffset: 0 }; await git.addNote({ fs, dir, ref, oid: anchorSha, note: JSON.stringify(content), force: true, author: person, committer: person }); const onAuth = () => ({ username: "x-token-auth", password: tokenString(token) }); const pushResult = await git.push({ fs, http, dir, url: remote, onAuth, ref, remoteRef: ref }); // pushResult.ok is an aggregate flag; per-ref status also confirmed ok // live for this exact failure, so this check isn't the fix -- see the // KNOWN ISSUE note on readRung below for what's actually going on. const refStatus = pushResult.refs?.[ref]; if (!refStatus?.ok) throw new Error(`push of ${ref} rejected: ${refStatus?.error ?? JSON.stringify(pushResult)}`); } export async function writeRung(remote: string, token: { plaintext: string } | string, repo: ArtifactsRepoHandle, rung: SimpleRung): Promise { const ref = notesRefFor(rung.kind); const anchorSha = await anchorShaForRung(repo, rung); const { fs, dir } = await scopedNotesCheckout(remote, token, ref); await pushNote(remote, token, fs, dir, ref, anchorSha, embed(rung)); } // writeRung's own fetch+push cycle is cheap for one note at a time // (the live agent loop's own usage -- a single changeset/ask landing), // but every "file"/"directory"/"repo"/etc. rung shares the one "code" // ref, and a fresh scopedNotesCheckout re-fetches that ref's ENTIRE // accumulated history from scratch every time (no local cache carries // over between calls). Confirmed live: onboarding a real, non-trivial // repo through writeRung called once per rung projected to multiple // HOURS for ~240 rungs, individual calls getting slower over the run // as the "code" ref's own history grew with each note added to it -- // a real O(n^2) cost, not a fixed per-call latency. This groups by ref // instead: one checkout (one fetch) per ref, every rung sharing that // ref added to the SAME in-memory note commit, one push per ref at // the end -- the git-level cost of n notes on one ref drops from n // fetches of a growing history to 1. export async function writeRungsBatch(remote: string, token: { plaintext: string } | string, repo: ArtifactsRepoHandle, rungs: SimpleRung[]): Promise { const withAnchors = await Promise.all(rungs.map(async (rung) => ({ rung, ref: notesRefFor(rung.kind), anchorSha: await anchorShaForRung(repo, rung) }))); const byRef = new Map(); for (const item of withAnchors) { const group = byRef.get(item.ref); if (group) group.push(item); else byRef.set(item.ref, [item]); } const person = { name: "zz-ladder", email: "zz-ladder@local", timestamp: Math.floor(Date.now() / 1000), timezoneOffset: 0 }; for (const [ref, items] of byRef) { const { fs, dir } = await scopedNotesCheckout(remote, token, ref); for (const { rung, anchorSha } of items) { await git.addNote({ fs, dir, ref, oid: anchorSha, note: JSON.stringify(embed(rung)), force: true, author: person, committer: person }); } const onAuth = () => ({ username: "x-token-auth", password: tokenString(token) }); const pushResult = await git.push({ fs, http, dir, url: remote, onAuth, ref, remoteRef: ref }); const refStatus = pushResult.refs?.[ref]; if (!refStatus?.ok) throw new Error(`push of ${ref} rejected: ${refStatus?.error ?? JSON.stringify(pushResult)}`); } } // One repo-wide Ladder read (the "_repo" root plus each of its // top-level file/changeset children, resolved one DO call at a time // by webApp.ts's getLadderTree) used to re-run scopedNotesCheckout // from scratch for every single one of those calls -- even though // every "file"/"directory"/"function"/"lines" rung shares the exact // same notes ref ("code"), so three file children meant fetching that // identical ref three separate times. Confirmed live: this, not any // one slow call, was the real cost of the Ladder tab (~2.8s of a // ~3.4s fetch, for a repo with only 5 top-level children). A cache // keyed by ref name, shared across every readRung call that passes // the SAME NotesCache object, fixes that -- callers that want this // sharing pass one cache instance across a batch of reads (see // cloudflareBackend.ts's readAnchorFor); nothing requires it. export type NotesCache = Map; dir: string; hadExisting: boolean }>; // KNOWN ISSUE, not yet fixed: a brand-new repo's writes to this ref // (confirmed for "repo"/"file" kinds, which share one ref -- see // notesRefFor) report success -- isomorphic-git's push() reports // ok:true at both the aggregate AND per-ref level, and a same-request // re-fetch (the next sequential rung in the same writeLadder batch) // correctly sees the prior note via a brand-new scopedNotesCheckout -- // but a LATER, separate request's fetch of that exact ref fails with // isomorphic-git's own NotFoundError ("Could not find "), so // readRung below returns null forever after. Confirmed NOT a timing // issue (retried after 45s and 120s, same result) and NOT caused by // `singleBranch: true` on the fetch (tested without it, same result). // Looks like an Artifacts-side consistency gap specific to non-branch // refs (git notes) that's visible within the request that just wrote // but not from a fresh connection -- not something fixable from this // side without a different write/verify strategy (e.g. reading back // after push and retrying, or writing notes as part of the same // commit instead of a separate ref). Flagging for follow-up rather // than chasing further right now. export async function readRung(remote: string, token: { plaintext: string } | string, repo: ArtifactsRepoHandle, kind: SimpleRung["kind"], path: string, explicitAnchorSha?: string, cache?: NotesCache): Promise { const ref = notesRefFor(kind); const anchorSha = explicitAnchorSha ?? (await anchorShaForRung(repo, { kind, path, sources: [] })); let checkout = cache?.get(ref); if (!checkout) { checkout = await scopedNotesCheckout(remote, token, ref); cache?.set(ref, checkout); } const { fs, dir, hadExisting } = checkout; if (!hadExisting) return null; try { const raw = await git.readNote({ fs, dir, ref, oid: anchorSha }); return JSON.parse(new TextDecoder().decode(raw)) as EmbeddedRung; } catch { return null; } } /** * Adds references to the repo root's own note without touching any * existing child, written or not -- the Cloudflare twin of * engine/ladderNotes.ts's appendRepoChildren, same reason: writeRung * always overwrites a rung's whole note, which is right when writing * that rung itself but would re-derive and clobber the _repo root's * already-correct children list if every other write went through it * too. mergeLaneLadder calls this after every non-root rung write, so * this has to work for the Cloudflare path to be functionally * complete, not just for "repo"-kind writes specifically. */ export async function appendRepoChildren(remote: string, token: { plaintext: string } | string, repo: ArtifactsRepoHandle, newChildren: { path: string; kind: string }[]): Promise { const ref = notesRefFor("repo"); const anchorSha = await rootCommitOid(repo); const { fs, dir, hadExisting } = await scopedNotesCheckout(remote, token, ref); if (!hadExisting) throw new Error("repo root has no note yet; write the root rung first"); const current = JSON.parse(new TextDecoder().decode(await git.readNote({ fs, dir, ref, oid: anchorSha }))) as EmbeddedRung; const existingRefs = new Set(current.children.filter(isRef).map((c) => c.ref)); const toAdd = newChildren.filter((c) => !existingRefs.has(c.path)); if (toAdd.length === 0) return; current.children.push(...toAdd.map((c) => ({ ref: c.path }))); await pushNote(remote, token, fs, dir, ref, anchorSha, current); }