{ "cells": [ { "cell_type": "markdown", "id": "88ce7e3e", "metadata": {}, "source": [ "# MKT-005 — Field notes from an autonomous engineering fleet\n", "\n", "Reproduces every numerical claim in the paper from fixed snapshots of the\n", "GitHub record (PRs, issues, git history) of the privateos orchestrator and its\n", "four workspace repositories. Stdlib only; no pip dependencies; no network.\n", "\n", "**Snapshots** (see `data/README.md` for refetch commands): fetched 2026-06-11,\n", "privateos at commit `44a0a6484d677a8063d9a9a6669a915bb96bb77b`.\n", "\n", "Sections follow the NOTEBOOK-CONVENTIONS cell-group order:\n", "`setup` → `load:*` → `build:*` → `analyse:*` → `export`." ] }, { "cell_type": "code", "execution_count": 1, "id": "1bddcbb7", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.368005Z", "iopub.status.busy": "2026-06-11T16:42:12.367901Z", "iopub.status.idle": "2026-06-11T16:42:12.375670Z", "shell.execute_reply": "2026-06-11T16:42:12.374952Z" }, "tags": [ "setup" ] }, "outputs": [], "source": [ "# Tested on Python 3.12. Stdlib only — no pip dependencies required.\n", "import json, gzip, csv, math, hashlib\n", "from collections import Counter, defaultdict\n", "from datetime import datetime, date\n", "from pathlib import Path\n", "\n", "DATA = Path(\"data\"); RAW = DATA / \"raw\"; FIGURES = Path(\"figures\")\n", "\n", "# Fixed-snapshot provenance (data/README.md documents refetch commands).\n", "SNAPSHOT_DATE = \"2026-06-11\"\n", "PRIVATEOS_SHA = \"44a0a6484d677a8063d9a9a6669a915bb96bb77b\"\n", "ORG = \"Prestige-Worldwide-Algotrading\"\n", "WORKSPACES = [\"poly\", \"market-ambience\", \"atol-research\", \"privateos-sandbox\"]\n", "\n", "# The PR-body contract (P0 fix from docs/state-of-the-app-2026-05-05.md) landed\n", "# 2026-05-05; PRs created before 2026-05-06 form the before era and the\n", "# rest the after era.\n", "BODY_CONTRACT_CUTOFF = \"2026-05-06\"\n", "\n", "def jload(path):\n", " with gzip.open(path, \"rt\") as f:\n", " return json.load(f)\n", "\n", "def write_csv(path, rows, cols):\n", " with open(path, \"w\", newline=\"\") as f:\n", " w = csv.DictWriter(f, fieldnames=cols)\n", " w.writeheader()\n", " for r in rows:\n", " w.writerow({k: r[k] for k in cols})\n", " return path" ] }, { "cell_type": "code", "execution_count": 2, "id": "8f52821f", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.377619Z", "iopub.status.busy": "2026-06-11T16:42:12.377439Z", "iopub.status.idle": "2026-06-11T16:42:12.394891Z", "shell.execute_reply": "2026-06-11T16:42:12.394184Z" }, "tags": [ "load:privateos-prs" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "privateos PRs: 285\n" ] } ], "source": [ "privateos_prs = jload(RAW / \"privateos_prs.json.gz\")\n", "print(f\"privateos PRs: {len(privateos_prs)}\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "28ea9821", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.396483Z", "iopub.status.busy": "2026-06-11T16:42:12.396366Z", "iopub.status.idle": "2026-06-11T16:42:12.411994Z", "shell.execute_reply": "2026-06-11T16:42:12.411497Z" }, "tags": [ "load:privateos-issues" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "privateos issues: 310\n" ] } ], "source": [ "privateos_issues = jload(RAW / \"privateos_issues.json.gz\")\n", "print(f\"privateos issues: {len(privateos_issues)}\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "b199e1e1", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.413357Z", "iopub.status.busy": "2026-06-11T16:42:12.413247Z", "iopub.status.idle": "2026-06-11T16:42:12.420440Z", "shell.execute_reply": "2026-06-11T16:42:12.419986Z" }, "tags": [ "load:privateos-commits" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "privateos commits on main: 376\n", "range: 2026-04-29 -> 2026-05-27\n" ] } ], "source": [ "# git log --numstat dump: 'COMMIT\\t\\t\\t' header lines,\n", "# followed by '\\t\\t' numstat lines ('-' for binary).\n", "commits = [] # {sha, dt, subject, files: [(added, deleted, path)]}\n", "with gzip.open(RAW / \"privateos_git_numstat.tsv.gz\", \"rt\") as f:\n", " cur = None\n", " for line in f:\n", " line = line.rstrip(\"\\n\")\n", " if not line:\n", " continue\n", " if line.startswith(\"COMMIT\\t\"):\n", " _, sha, dt, subject = line.split(\"\\t\", 3)\n", " cur = {\"sha\": sha, \"dt\": dt, \"subject\": subject, \"files\": []}\n", " commits.append(cur)\n", " elif cur is not None:\n", " a, d, path = line.split(\"\\t\", 2)\n", " cur[\"files\"].append((None if a == \"-\" else int(a),\n", " None if d == \"-\" else int(d), path))\n", "print(f\"privateos commits on main: {len(commits)}\")\n", "print(f\"range: {min(c['dt'] for c in commits)[:10]} -> {max(c['dt'] for c in commits)[:10]}\")" ] }, { "cell_type": "code", "execution_count": 5, "id": "02cfde96", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.421589Z", "iopub.status.busy": "2026-06-11T16:42:12.421499Z", "iopub.status.idle": "2026-06-11T16:42:12.433022Z", "shell.execute_reply": "2026-06-11T16:42:12.431604Z" }, "tags": [ "load:workspace-prs" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "poly: 197 PRs total\n", "market-ambience: 41 PRs total\n", "atol-research: 49 PRs total\n", "privateos-sandbox: 1 PRs total\n" ] } ], "source": [ "workspace_prs = {ws: jload(RAW / f\"{ws}_prs.json.gz\") for ws in WORKSPACES}\n", "for ws, prs in workspace_prs.items():\n", " print(f\"{ws}: {len(prs)} PRs total\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "f0fe72ce", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.434616Z", "iopub.status.busy": "2026-06-11T16:42:12.434505Z", "iopub.status.idle": "2026-06-11T16:42:12.437677Z", "shell.execute_reply": "2026-06-11T16:42:12.437095Z" }, "tags": [ "load:learnings-files" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "learnings postmortem files: 29\n" ] } ], "source": [ "with gzip.open(RAW / \"privateos_learnings_files.txt.gz\", \"rt\") as f:\n", " learnings_files = [l.strip() for l in f if l.strip()]\n", "print(f\"learnings postmortem files: {len(learnings_files)}\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "fff6009a", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.439056Z", "iopub.status.busy": "2026-06-11T16:42:12.438971Z", "iopub.status.idle": "2026-06-11T16:42:12.441804Z", "shell.execute_reply": "2026-06-11T16:42:12.441497Z" }, "tags": [ "load:repo-tree" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tracked files in snapshot: 679\n" ] } ], "source": [ "# Per-file line counts for *.ts, *.tsx, and src/db/migrations/* at commit\n", "# 44a0a64 (wc -l over git ls-files; refetch command in data/README.md).\n", "with gzip.open(RAW / \"privateos_tree_loc.tsv.gz\", \"rt\") as f:\n", " tree_loc = [(int(n), p) for n, p in\n", " (line.split(\"\\t\") for line in f.read().splitlines() if line)]\n", "print(f\"tracked files in snapshot: {len(tree_loc)}\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "2780de68", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.442871Z", "iopub.status.busy": "2026-06-11T16:42:12.442800Z", "iopub.status.idle": "2026-06-11T16:42:12.450999Z", "shell.execute_reply": "2026-06-11T16:42:12.449169Z" }, "tags": [ "build:agent-prs" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "agent-authored PRs across 4 workspaces: 75\n", "Counter({'MERGED': 56, 'OPEN': 15, 'CLOSED': 4})\n", "privateos self-directed agent PRs (excluded from agent_prs.csv): 22 {'MERGED': 22} 2026-05-05 -> 2026-05-27\n" ] } ], "source": [ "# An agent-authored PR is identified by its head branch ('agent/...') or a\n", "# run-id stamp the runner writes into the PR body.\n", "def is_agent(p):\n", " return p[\"headRefName\"].startswith(\"agent/\") or \"run.id\" in (p[\"body\"] or \"\")\n", "\n", "def quality(p):\n", " b = p[\"body\"] or \"\"\n", " return {\n", " \"has_summary\": \"## Summary\" in b,\n", " \"has_linkage\": any(t in b for t in (\"Part of #\", \"Closes #\", \"Fixes #\")),\n", " \"has_scope\": \"## Scope\" in b,\n", " \"nodeid_branch\": p[\"headRefName\"].startswith(\"agent/I_\"),\n", " }\n", "\n", "agent_prs = []\n", "for ws in WORKSPACES:\n", " for p in workspace_prs[ws]:\n", " if not is_agent(p):\n", " continue\n", " created = p[\"createdAt\"][:10]\n", " row = {\"repo\": ws, \"number\": p[\"number\"], \"created\": created,\n", " \"state\": p[\"state\"], \"additions\": p[\"additions\"],\n", " \"deletions\": p[\"deletions\"], \"changed_files\": p[\"changedFiles\"],\n", " \"era\": \"before\" if created < BODY_CONTRACT_CUTOFF else \"after\"}\n", " row.update(quality(p))\n", " agent_prs.append(row)\n", "agent_prs.sort(key=lambda r: (r[\"created\"], r[\"repo\"], r[\"number\"]))\n", "\n", "cols = [\"repo\", \"number\", \"created\", \"state\", \"additions\", \"deletions\",\n", " \"changed_files\", \"era\", \"has_summary\", \"has_linkage\", \"has_scope\",\n", " \"nodeid_branch\"]\n", "write_csv(DATA / \"agent_prs.csv\", agent_prs, cols)\n", "print(f\"agent-authored PRs across {len(WORKSPACES)} workspaces: {len(agent_prs)}\")\n", "print(Counter(r[\"state\"] for r in agent_prs))\n", "\n", "# The fleet also worked on its own orchestrator. Agent-authored PRs inside\n", "# privateos are reported here but excluded from the operational record so the\n", "# construction and operation surfaces stay distinct (paper section 4.3).\n", "self_prs = [p for p in privateos_prs if is_agent(p)]\n", "print(f\"privateos self-directed agent PRs (excluded from agent_prs.csv): \"\n", " f\"{len(self_prs)}\", dict(Counter(p[\"state\"] for p in self_prs)),\n", " min(p[\"createdAt\"][:10] for p in self_prs), \"->\",\n", " max(p[\"createdAt\"][:10] for p in self_prs))" ] }, { "cell_type": "code", "execution_count": 9, "id": "fd942abc", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.475017Z", "iopub.status.busy": "2026-06-11T16:42:12.474738Z", "iopub.status.idle": "2026-06-11T16:42:12.506683Z", "shell.execute_reply": "2026-06-11T16:42:12.506076Z" }, "tags": [ "build:pr-quality-eras" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'era': 'before', 'n': 4, 'merged': 2, 'merge_rate': 0.5, 'median_changed_files': 9.0, 'has_summary': 1, 'has_linkage': 1, 'has_scope': 1, 'nodeid_branch': 3}\n", "{'era': 'after', 'n': 71, 'merged': 54, 'merge_rate': 0.761, 'median_changed_files': 3, 'has_summary': 71, 'has_linkage': 71, 'has_scope': 69, 'nodeid_branch': 0}\n" ] } ], "source": [ "def median(xs):\n", " xs = sorted(xs)\n", " n = len(xs)\n", " return xs[n // 2] if n % 2 else (xs[n // 2 - 1] + xs[n // 2]) / 2\n", "\n", "eras = []\n", "for era in (\"before\", \"after\"):\n", " sel = [r for r in agent_prs if r[\"era\"] == era]\n", " n = len(sel)\n", " eras.append({\n", " \"era\": era, \"n\": n,\n", " \"merged\": sum(1 for r in sel if r[\"state\"] == \"MERGED\"),\n", " \"merge_rate\": round(sum(1 for r in sel if r[\"state\"] == \"MERGED\") / n, 3),\n", " \"median_changed_files\": median([r[\"changed_files\"] for r in sel]),\n", " \"has_summary\": sum(1 for r in sel if r[\"has_summary\"]),\n", " \"has_linkage\": sum(1 for r in sel if r[\"has_linkage\"]),\n", " \"has_scope\": sum(1 for r in sel if r[\"has_scope\"]),\n", " \"nodeid_branch\": sum(1 for r in sel if r[\"nodeid_branch\"]),\n", " })\n", "cols = list(eras[0])\n", "write_csv(DATA / \"pr_quality_eras.csv\", eras, cols)\n", "for e in eras:\n", " print(e)" ] }, { "cell_type": "code", "execution_count": 10, "id": "1eb63f95", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.525330Z", "iopub.status.busy": "2026-06-11T16:42:12.524117Z", "iopub.status.idle": "2026-06-11T16:42:12.579438Z", "shell.execute_reply": "2026-06-11T16:42:12.576285Z" }, "tags": [ "build:construction-cadence" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "27 active days, 376 commits, 285 PRs, 310 issues\n" ] } ], "source": [ "by_day = defaultdict(lambda: {\"commits\": 0, \"prs_created\": 0, \"issues_opened\": 0})\n", "for c in commits:\n", " by_day[c[\"dt\"][:10]][\"commits\"] += 1\n", "for p in privateos_prs:\n", " by_day[p[\"createdAt\"][:10]][\"prs_created\"] += 1\n", "for i in privateos_issues:\n", " by_day[i[\"createdAt\"][:10]][\"issues_opened\"] += 1\n", "\n", "cadence = [{\"date\": d, **v} for d, v in sorted(by_day.items())]\n", "write_csv(DATA / \"construction_cadence.csv\", cadence,\n", " [\"date\", \"commits\", \"prs_created\", \"issues_opened\"])\n", "print(f\"{len(cadence)} active days, \"\n", " f\"{sum(r['commits'] for r in cadence)} commits, \"\n", " f\"{sum(r['prs_created'] for r in cadence)} PRs, \"\n", " f\"{sum(r['issues_opened'] for r in cadence)} issues\")" ] }, { "cell_type": "code", "execution_count": 11, "id": "1c4caca6", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.587604Z", "iopub.status.busy": "2026-06-11T16:42:12.587435Z", "iopub.status.idle": "2026-06-11T16:42:12.611507Z", "shell.execute_reply": "2026-06-11T16:42:12.611033Z" }, "tags": [ "build:loc-by-subsystem" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "top subsystems by lines added: [('tests', 91382), ('src/server', 62902), ('src/frontend', 27025), ('src/app-installer', 12173), ('pnpm-lock.yaml', 9320), ('src/db', 8293), ('docs', 6967), ('src', 3296)]\n" ] } ], "source": [ "# Lines added per ISO week per subsystem (top-level dir; src/ split one level deeper).\n", "def subsystem(path):\n", " parts = path.split(\"/\")\n", " if parts[0] == \"src\" and len(parts) > 2:\n", " return f\"src/{parts[1]}\"\n", " return parts[0]\n", "\n", "week_sub = defaultdict(int)\n", "for c in commits:\n", " wk = datetime.fromisoformat(c[\"dt\"]).strftime(\"%G-W%V\")\n", " for a, d, path in c[\"files\"]:\n", " if a is not None:\n", " week_sub[(wk, subsystem(path))] += a\n", "\n", "totals = Counter()\n", "for (wk, sub), a in week_sub.items():\n", " totals[sub] += a\n", "keep = {s for s, _ in totals.most_common(8)}\n", "\n", "rows = defaultdict(lambda: defaultdict(int))\n", "for (wk, sub), a in week_sub.items():\n", " rows[wk][sub if sub in keep else \"other\"] += a\n", "loc_rows = [{\"week\": wk, \"subsystem\": sub, \"lines_added\": n}\n", " for wk in sorted(rows) for sub, n in sorted(rows[wk].items())]\n", "write_csv(DATA / \"loc_by_subsystem.csv\", loc_rows, [\"week\", \"subsystem\", \"lines_added\"])\n", "print(f\"top subsystems by lines added: {totals.most_common(8)}\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "0470ed02", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.612810Z", "iopub.status.busy": "2026-06-11T16:42:12.612711Z", "iopub.status.idle": "2026-06-11T16:42:12.628504Z", "shell.execute_reply": "2026-06-11T16:42:12.625637Z" }, "tags": [ "build:incident-registry" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "verified 11 incident artifacts + 9 claimRun rollout PRs (all merged, epic #196)\n", " scope-indiscipline poly#191 2026-05-05 scope-explosion\n", " self-trigger-loop privateos#249 2026-05-06 fixer-self-loop\n", " silent-capability-loss privateos#193 2026-05-05 tools-never-registered\n", " silent-capability-loss privateos#220 2026-05-05 sdk-success-is-error\n", " control-ineffective privateos#362 2026-05-12 kill-is-row-only\n", " liveness-misjudgment privateos#610 2026-05-24 janitor-reaps-live-runs\n", " liveness-misjudgment privateos#613 2026-05-26 janitor-threshold-fix\n", " idempotency-gap privateos#594 2026-05-20 duplicate-deferred-rows\n", " isolation-failure privateos#595 2026-05-20 worktree-gone\n", " observability-gap privateos#221 2026-05-05 observability-gaps\n", " control-built privateos#398 2026-05-15 claimrun-primitive\n" ] } ], "source": [ "# Verified incident registry. Every war story cited in the paper maps to a\n", "# concrete GitHub artifact in the snapshots; each entry is asserted against the\n", "# dump (number exists, expected title substring present). A failed assertion\n", "# means the paper's claim no longer matches the record.\n", "REGISTRY = [\n", " # slug, family, repo, kind, number, must-contain (title)\n", " (\"scope-explosion\", \"scope-indiscipline\", \"poly\", \"pr\", 191,\n", " \"drop BuilderConfig\"),\n", " (\"fixer-self-loop\", \"self-trigger-loop\", \"privateos\", \"pr\", 249,\n", " \"Fixer triggers exclusively review-driven\"),\n", " (\"tools-never-registered\", \"silent-capability-loss\", \"privateos\", \"issue\", 193,\n", " \"defined but never registered\"),\n", " (\"sdk-success-is-error\", \"silent-capability-loss\", \"privateos\", \"pr\", 220,\n", " \"success + is_error\"),\n", " (\"kill-is-row-only\", \"control-ineffective\", \"privateos\", \"issue\", 362,\n", " \"reap is row-only\"),\n", " (\"janitor-reaps-live-runs\", \"liveness-misjudgment\", \"privateos\", \"issue\", 610,\n", " \"still reaping live in-process runs\"),\n", " (\"janitor-threshold-fix\", \"liveness-misjudgment\", \"privateos\", \"pr\", 613,\n", " \"widen janitor stale threshold\"),\n", " (\"duplicate-deferred-rows\", \"idempotency-gap\", \"privateos\", \"issue\", 594,\n", " \"duplicate deferred rows\"),\n", " (\"worktree-gone\", \"isolation-failure\", \"privateos\", \"pr\", 595,\n", " \"WorktreeGoneError\"),\n", " (\"observability-gaps\", \"observability-gap\", \"privateos\", \"issue\", 221,\n", " \"no mid-run cost telemetry\"),\n", " (\"claimrun-primitive\", \"control-built\", \"privateos\", \"pr\", 398,\n", " \"claimRun primitive\"),\n", "]\n", "\n", "privateos_by_num = {\"pr\": {p[\"number\"]: p for p in privateos_prs},\n", " \"issue\": {i[\"number\"]: i for i in privateos_issues}}\n", "poly_by_num = {\"pr\": {p[\"number\"]: p for p in workspace_prs[\"poly\"]}}\n", "\n", "registry_rows = []\n", "for slug, family, repo, kind, number, expect in REGISTRY:\n", " pool = poly_by_num if repo == \"poly\" else privateos_by_num\n", " art = pool[kind].get(number)\n", " assert art is not None, f\"{slug}: {repo} {kind} #{number} not found in snapshot\"\n", " assert expect in art[\"title\"], (\n", " f\"{slug}: title mismatch for {repo}#{number}: {art['title']!r}\")\n", " registry_rows.append({\n", " \"slug\": slug, \"family\": family, \"repo\": repo, \"kind\": kind,\n", " \"number\": number, \"date\": art[\"createdAt\"][:10],\n", " \"state\": art[\"state\"], \"title\": art[\"title\"]})\n", "\n", "# The claimRun rollout under epic #196 landed as nine merged PRs (titled\n", "# 1/10 through 10/10; PRs 6 and 7 were bundled in #407).\n", "CLAIMRUN_ROLLOUT = [398, 401, 403, 404, 406, 407, 408, 409, 410]\n", "for n in CLAIMRUN_ROLLOUT:\n", " pr = privateos_by_num[\"pr\"][n]\n", " assert pr[\"state\"] == \"MERGED\" and \"#196\" in pr[\"title\"], f\"rollout PR #{n}\"\n", "\n", "write_csv(DATA / \"incident_registry.csv\", registry_rows,\n", " [\"slug\", \"family\", \"repo\", \"kind\", \"number\", \"date\", \"state\", \"title\"])\n", "print(f\"verified {len(registry_rows)} incident artifacts \"\n", " f\"+ {len(CLAIMRUN_ROLLOUT)} claimRun rollout PRs (all merged, epic #196)\")\n", "for r in registry_rows:\n", " print(f\" {r['family']:<22} {r['repo']}#{r['number']:<4} {r['date']} {r['slug']}\")" ] }, { "cell_type": "code", "execution_count": 13, "id": "cd665b26", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.630226Z", "iopub.status.busy": "2026-06-11T16:42:12.630131Z", "iopub.status.idle": "2026-06-11T16:42:12.634283Z", "shell.execute_reply": "2026-06-11T16:42:12.633918Z" }, "tags": [ "build:learnings-catalog" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "29 postmortems in 8 categories\n", " architecture-patterns 9\n", " conventions 7\n", " db-issues 2\n", " prompt-issues 2\n", " prompt-patterns 2\n", " runtime-errors 5\n", " test-failures 1\n", " tooling-decisions 1\n" ] } ], "source": [ "# docs/learnings/ in privateos: one markdown postmortem per learned bug class.\n", "cat_rows = sorted(Counter(f.split(\"/\")[2] for f in learnings_files).items())\n", "learn_rows = [{\"category\": c, \"postmortems\": n} for c, n in cat_rows]\n", "write_csv(DATA / \"learnings_catalog.csv\", learn_rows, [\"category\", \"postmortems\"])\n", "print(f\"{len(learnings_files)} postmortems in {len(learn_rows)} categories\")\n", "for r in learn_rows:\n", " print(f\" {r['category']:<24} {r['postmortems']}\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "610378ba", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.635738Z", "iopub.status.busy": "2026-06-11T16:42:12.635652Z", "iopub.status.idle": "2026-06-11T16:42:12.639770Z", "shell.execute_reply": "2026-06-11T16:42:12.639036Z" }, "tags": [ "build:workspace-summary" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'repo': 'poly', 'agent_prs': 39, 'merged': 36, 'merge_rate': 0.923, 'first': '2026-05-04', 'last': '2026-05-26', 'median_changed_files': 3}\n", "{'repo': 'market-ambience', 'agent_prs': 12, 'merged': 12, 'merge_rate': 1.0, 'first': '2026-05-14', 'last': '2026-05-20', 'median_changed_files': 3.0}\n", "{'repo': 'atol-research', 'agent_prs': 23, 'merged': 8, 'merge_rate': 0.348, 'first': '2026-05-15', 'last': '2026-05-20', 'median_changed_files': 4}\n", "{'repo': 'privateos-sandbox', 'agent_prs': 1, 'merged': 0, 'merge_rate': 0.0, 'first': '2026-04-29', 'last': '2026-04-29', 'median_changed_files': 1}\n" ] } ], "source": [ "ws_rows = []\n", "for ws in WORKSPACES:\n", " sel = [r for r in agent_prs if r[\"repo\"] == ws]\n", " if not sel:\n", " continue\n", " ws_rows.append({\n", " \"repo\": ws, \"agent_prs\": len(sel),\n", " \"merged\": sum(1 for r in sel if r[\"state\"] == \"MERGED\"),\n", " \"merge_rate\": round(sum(1 for r in sel if r[\"state\"] == \"MERGED\") / len(sel), 3),\n", " \"first\": min(r[\"created\"] for r in sel),\n", " \"last\": max(r[\"created\"] for r in sel),\n", " \"median_changed_files\": median([r[\"changed_files\"] for r in sel])})\n", "write_csv(DATA / \"workspace_summary.csv\", ws_rows,\n", " [\"repo\", \"agent_prs\", \"merged\", \"merge_rate\", \"first\", \"last\",\n", " \"median_changed_files\"])\n", "for r in ws_rows:\n", " print(r)" ] }, { "cell_type": "code", "execution_count": 15, "id": "7f2dd6b9", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.641842Z", "iopub.status.busy": "2026-06-11T16:42:12.641759Z", "iopub.status.idle": "2026-06-11T16:42:12.645605Z", "shell.execute_reply": "2026-06-11T16:42:12.645080Z" }, "tags": [ "analyse:headline" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "construction: 376 commits, 285 PRs (284 merged), 310 issues, 2026-04-29 -> 2026-05-27 (29 days)\n", "operation: 75 agent-authored PRs across 4 workspaces, 56 merged (75%)\n" ] } ], "source": [ "# Headline numbers cited in the paper (construction + operational record).\n", "n_commits = len(commits)\n", "n_prs = len(privateos_prs)\n", "n_merged = sum(1 for p in privateos_prs if p[\"state\"] == \"MERGED\")\n", "n_issues = len(privateos_issues)\n", "d0 = min(c[\"dt\"] for c in commits)[:10]\n", "d1 = max(c[\"dt\"] for c in commits)[:10]\n", "span_days = (date.fromisoformat(d1) - date.fromisoformat(d0)).days + 1\n", "n_agent = len(agent_prs)\n", "n_agent_merged = sum(1 for r in agent_prs if r[\"state\"] == \"MERGED\")\n", "\n", "assert n_merged <= n_prs and n_agent_merged <= n_agent\n", "print(f\"construction: {n_commits} commits, {n_prs} PRs ({n_merged} merged), \"\n", " f\"{n_issues} issues, {d0} -> {d1} ({span_days} days)\")\n", "print(f\"operation: {n_agent} agent-authored PRs across {len(WORKSPACES)} \"\n", " f\"workspaces, {n_agent_merged} merged \"\n", " f\"({100 * n_agent_merged / n_agent:.0f}%)\")" ] }, { "cell_type": "code", "execution_count": 16, "id": "f1bf9d8f", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.646940Z", "iopub.status.busy": "2026-06-11T16:42:12.646846Z", "iopub.status.idle": "2026-06-11T16:42:12.651391Z", "shell.execute_reply": "2026-06-11T16:42:12.650330Z" }, "tags": [ "analyse:quality-shift" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "has_summary: before 1/4, after 71/71 (one-sided Fisher p = 5.92e-05)\n", "has_linkage: before 1/4, after 71/71 (one-sided Fisher p = 5.92e-05)\n", "merge rate: before 2/4, after 54/71 (one-sided Fisher p = 0.26)\n", "\n", "Caveat for the paper: the before era is four PRs. The structure metrics\n", "(summary/linkage present in 71/71 after, ~never before) carry the claim;\n", "the merge-rate shift alone does not reach significance at this n.\n" ] } ], "source": [ "# Did the 2026-05-05 PR-body contract change agent PR quality?\n", "# One-sided Fisher exact test (hypergeometric tail, stdlib only):\n", "# probability of seeing at most s1 successes in the before-era group if\n", "# before/after shared one success rate.\n", "def fisher_one_sided(s1, n1, s2, n2):\n", " N, K = n1 + n2, s1 + s2\n", " return sum(math.comb(K, k) * math.comb(N - K, n1 - k) / math.comb(N, n1)\n", " for k in range(0, s1 + 1) if n1 - k <= N - K)\n", "\n", "before = [r for r in agent_prs if r[\"era\"] == \"before\"]\n", "after = [r for r in agent_prs if r[\"era\"] == \"after\"]\n", "n1, n2 = len(before), len(after)\n", "\n", "for metric in (\"has_summary\", \"has_linkage\"):\n", " s1 = sum(1 for r in before if r[metric])\n", " s2 = sum(1 for r in after if r[metric])\n", " p = fisher_one_sided(s1, n1, s2, n2)\n", " print(f\"{metric}: before {s1}/{n1}, after {s2}/{n2} (one-sided Fisher p = {p:.2e})\")\n", "\n", "m1 = sum(1 for r in before if r[\"state\"] == \"MERGED\")\n", "m2 = sum(1 for r in after if r[\"state\"] == \"MERGED\")\n", "p = fisher_one_sided(m1, n1, m2, n2)\n", "print(f\"merge rate: before {m1}/{n1}, after {m2}/{n2} (one-sided Fisher p = {p:.2f})\")\n", "print()\n", "print(\"Caveat for the paper: the before era is four PRs. The structure metrics\")\n", "print(\"(summary/linkage present in 71/71 after, ~never before) carry the claim;\")\n", "print(\"the merge-rate shift alone does not reach significance at this n.\")" ] }, { "cell_type": "code", "execution_count": 17, "id": "1497f7e4", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.652852Z", "iopub.status.busy": "2026-06-11T16:42:12.652761Z", "iopub.status.idle": "2026-06-11T16:42:12.655986Z", "shell.execute_reply": "2026-06-11T16:42:12.655277Z" }, "tags": [ "analyse:blast-radius" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "before: n=4 median=9.0 max=26 files\n", "after: n=71 median=3 max=30 files\n", "poly#191 (the scope-explosion case study): 15 files, +53/-565, state=CLOSED\n" ] } ], "source": [ "# Scope discipline: changed-files distribution by era, with poly#191 called out.\n", "for era in (\"before\", \"after\"):\n", " xs = [r[\"changed_files\"] for r in agent_prs if r[\"era\"] == era]\n", " print(f\"{era}: n={len(xs)} median={median(xs)} max={max(xs)} files\")\n", "p191 = next(r for r in agent_prs if r[\"repo\"] == \"poly\" and r[\"number\"] == 191)\n", "print(f\"poly#191 (the scope-explosion case study): {p191['changed_files']} files, \"\n", " f\"+{p191['additions']}/-{p191['deletions']}, state={p191['state']}\")\n", "assert p191[\"state\"] == \"CLOSED\" and p191[\"deletions\"] > 500" ] }, { "cell_type": "code", "execution_count": 18, "id": "9dab78bc", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.657570Z", "iopub.status.busy": "2026-06-11T16:42:12.657400Z", "iopub.status.idle": "2026-06-11T16:42:12.661458Z", "shell.execute_reply": "2026-06-11T16:42:12.660851Z" }, "tags": [ "analyse:janitor-saga" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "#362 (2026-05-12): fix(runs-janitor): in-process persona runners have no pid → reap is row-only, cannot actually stop a stuck runner\n", "#610 (2026-05-24): runs-janitor: KILL_REASON_PID_UNKNOWN still reaping live in-process runs after #362 fix (PR #393)\n", "#613 (2026-05-26): fix(#610): widen janitor stale threshold 30s→90s, tighten heartbeat 5s→3s\n", "liveness saga span: 14 days from first kill bug to threshold fix\n" ] } ], "source": [ "# Liveness tuning is hard: #362 (kill was row-only) -> #610 (janitor reaped\n", "# live runs) -> #613 (stale threshold 30s->90s, heartbeat 5s->3s).\n", "i362 = next(i for i in privateos_issues if i[\"number\"] == 362)\n", "i610 = next(i for i in privateos_issues if i[\"number\"] == 610)\n", "p613 = next(p for p in privateos_prs if p[\"number\"] == 613)\n", "assert \"30s\" in p613[\"title\"] and \"90s\" in p613[\"title\"] and \"3s\" in p613[\"title\"]\n", "for art in (i362, i610, p613):\n", " print(f\"#{art['number']} ({art['createdAt'][:10]}): {art['title']}\")\n", "days = (date.fromisoformat(p613[\"createdAt\"][:10])\n", " - date.fromisoformat(i362[\"createdAt\"][:10])).days\n", "print(f\"liveness saga span: {days} days from first kill bug to threshold fix\")" ] }, { "cell_type": "code", "execution_count": 19, "id": "6a0d8c5f", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.663120Z", "iopub.status.busy": "2026-06-11T16:42:12.662770Z", "iopub.status.idle": "2026-06-11T16:42:12.667048Z", "shell.execute_reply": "2026-06-11T16:42:12.666559Z" }, "tags": [ "analyse:repo-snapshot" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TypeScript LOC under src/: 100,863\n", "TypeScript LOC under tests/: 88,454\n", "SQL migrations: 66\n" ] } ], "source": [ "# Repository shape at commit 44a0a64, cited in paper section 4.1.\n", "src_loc = sum(n for n, p in tree_loc\n", " if p.startswith(\"src/\") and p.endswith((\".ts\", \".tsx\")))\n", "tests_loc = sum(n for n, p in tree_loc\n", " if p.startswith(\"tests/\") and p.endswith((\".ts\", \".tsx\")))\n", "migrations = sum(1 for n, p in tree_loc\n", " if p.startswith(\"src/db/migrations/\") and p.endswith(\".sql\"))\n", "print(f\"TypeScript LOC under src/: {src_loc:,}\")\n", "print(f\"TypeScript LOC under tests/: {tests_loc:,}\")\n", "print(f\"SQL migrations: {migrations}\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "46c6df69", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.668329Z", "iopub.status.busy": "2026-06-11T16:42:12.668240Z", "iopub.status.idle": "2026-06-11T16:42:12.671503Z", "shell.execute_reply": "2026-06-11T16:42:12.671089Z" }, "tags": [ "figure:01-cadence-and-incidents" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "wrote figures/01-cadence-and-incidents.vega.json\n" ] } ], "source": [ "# Figure 1: construction cadence with verified incident dates overlaid.\n", "incident_marks = [{\"date\": r[\"date\"], \"slug\": r[\"slug\"], \"family\": r[\"family\"]}\n", " for r in registry_rows]\n", "spec = {\n", " \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.json\",\n", " \"description\": \"Commits per day to the privateos main branch over the \"\n", " \"29-day build window, with the dates of the eleven verified \"\n", " \"incident artifacts overlaid as dashed rules.\",\n", " \"layer\": [\n", " {\"data\": {\"url\": \"data/construction_cadence.csv\"},\n", " \"mark\": {\"type\": \"bar\"},\n", " \"encoding\": {\n", " \"x\": {\"field\": \"date\", \"type\": \"temporal\", \"title\": \"2026\"},\n", " \"y\": {\"field\": \"commits\", \"type\": \"quantitative\",\n", " \"title\": \"Commits per day\"}}},\n", " {\"data\": {\"values\": incident_marks},\n", " \"mark\": {\"type\": \"rule\", \"strokeDash\": [3, 3], \"opacity\": 0.6},\n", " \"encoding\": {\n", " \"x\": {\"field\": \"date\", \"type\": \"temporal\"},\n", " \"tooltip\": [{\"field\": \"slug\", \"type\": \"nominal\"},\n", " {\"field\": \"family\", \"type\": \"nominal\"},\n", " {\"field\": \"date\", \"type\": \"temporal\"}]}},\n", " ],\n", "}\n", "out = FIGURES / \"01-cadence-and-incidents.vega.json\"\n", "out.write_text(json.dumps(spec, indent=2) + \"\\n\")\n", "print(\"wrote\", out)" ] }, { "cell_type": "code", "execution_count": 21, "id": "7519c9cc", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.672702Z", "iopub.status.busy": "2026-06-11T16:42:12.672632Z", "iopub.status.idle": "2026-06-11T16:42:12.676084Z", "shell.execute_reply": "2026-06-11T16:42:12.675644Z" }, "tags": [ "figure:03-quality-before-after" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "wrote figures/03-quality-before-after.vega.json\n" ] } ], "source": [ "# Figure 3: PR-structure metrics before/after the 2026-05-05 body contract.\n", "METRIC_LABELS = {\"has_summary\": \"Summary section\",\n", " \"has_linkage\": \"Issue linkage\",\n", " \"has_scope\": \"Scope section\"}\n", "rows = []\n", "for e in eras:\n", " for key, label in METRIC_LABELS.items():\n", " rows.append({\"metric\": label, \"era\": e[\"era\"],\n", " \"share\": round(100 * e[key] / e[\"n\"], 1)})\n", "spec = {\n", " \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.json\",\n", " \"description\": \"Share of agent-authored PRs carrying a Summary section, \"\n", " \"an issue linkage line, and a Scope section, before \"\n", " \"(n=4) and after (n=71) the PR-body contract.\",\n", " \"data\": {\"values\": rows},\n", " \"mark\": {\"type\": \"bar\"},\n", " \"encoding\": {\n", " \"x\": {\"field\": \"metric\", \"type\": \"nominal\", \"title\": None,\n", " \"sort\": list(METRIC_LABELS.values())},\n", " \"xOffset\": {\"field\": \"era\", \"sort\": [\"before\", \"after\"]},\n", " \"y\": {\"field\": \"share\", \"type\": \"quantitative\",\n", " \"title\": \"Share of agent PRs (%)\",\n", " \"scale\": {\"domain\": [0, 100]}},\n", " \"color\": {\"field\": \"era\", \"type\": \"nominal\",\n", " \"sort\": [\"before\", \"after\"], \"title\": \"Era\"},\n", " \"tooltip\": [{\"field\": \"metric\", \"type\": \"nominal\"},\n", " {\"field\": \"era\", \"type\": \"nominal\"},\n", " {\"field\": \"share\", \"type\": \"quantitative\"}],\n", " },\n", "}\n", "out = FIGURES / \"03-quality-before-after.vega.json\"\n", "out.write_text(json.dumps(spec, indent=2) + \"\\n\")\n", "print(\"wrote\", out)" ] }, { "cell_type": "code", "execution_count": 22, "id": "e5e9e41f", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.677572Z", "iopub.status.busy": "2026-06-11T16:42:12.677495Z", "iopub.status.idle": "2026-06-11T16:42:12.680545Z", "shell.execute_reply": "2026-06-11T16:42:12.680133Z" }, "tags": [ "figure:04-blast-radius" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "wrote figures/04-blast-radius.vega.json\n" ] } ], "source": [ "# Figure 4: blast radius (files changed) of every agent PR over time, with\n", "# the body-contract date as a dashed rule. poly#191 is the point at\n", "# (2026-05-05, 15 files); the caption identifies it.\n", "spec = {\n", " \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.json\",\n", " \"description\": \"Files changed per agent-authored PR by creation date \"\n", " \"(n=75, four workspaces). The dashed rule marks the \"\n", " \"2026-05-05 PR-body contract.\",\n", " \"layer\": [\n", " {\"data\": {\"url\": \"data/agent_prs.csv\"},\n", " \"mark\": {\"type\": \"point\"},\n", " \"encoding\": {\n", " \"x\": {\"field\": \"created\", \"type\": \"temporal\", \"title\": \"2026\"},\n", " \"y\": {\"field\": \"changed_files\", \"type\": \"quantitative\",\n", " \"title\": \"Files changed\"},\n", " \"tooltip\": [{\"field\": \"repo\", \"type\": \"nominal\"},\n", " {\"field\": \"number\", \"type\": \"quantitative\"},\n", " {\"field\": \"state\", \"type\": \"nominal\"},\n", " {\"field\": \"changed_files\", \"type\": \"quantitative\"}]}},\n", " {\"data\": {\"values\": [{\"cutoff\": \"2026-05-05\"}]},\n", " \"mark\": {\"type\": \"rule\", \"strokeDash\": [3, 3], \"opacity\": 0.6},\n", " \"encoding\": {\"x\": {\"field\": \"cutoff\", \"type\": \"temporal\"}}},\n", " ],\n", "}\n", "out = FIGURES / \"04-blast-radius.vega.json\"\n", "out.write_text(json.dumps(spec, indent=2) + \"\\n\")\n", "print(\"wrote\", out)" ] }, { "cell_type": "code", "execution_count": 23, "id": "70f1e069", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.681559Z", "iopub.status.busy": "2026-06-11T16:42:12.681493Z", "iopub.status.idle": "2026-06-11T16:42:12.684242Z", "shell.execute_reply": "2026-06-11T16:42:12.683823Z" }, "tags": [ "figure:02-workspace-merge-rates" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "wrote figures/02-workspace-merge-rates.vega.json\n" ] } ], "source": [ "# Figure 2: agent-PR merge rate by workspace (the domain-dependence finding).\n", "spec = {\n", " \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.json\",\n", " \"description\": \"Merge rate of agent-authored PRs by workspace repository. \"\n", " \"poly 36/39, market-ambience 12/12, atol-research 8/23, \"\n", " \"privateos-sandbox 0/1.\",\n", " \"data\": {\"url\": \"data/workspace_summary.csv\"},\n", " \"mark\": {\"type\": \"bar\"},\n", " \"encoding\": {\n", " \"x\": {\"field\": \"repo\", \"type\": \"nominal\", \"title\": None,\n", " \"sort\": \"-y\"},\n", " \"y\": {\"field\": \"merge_rate\", \"type\": \"quantitative\",\n", " \"title\": \"Merge rate\", \"axis\": {\"format\": \".0%\"},\n", " \"scale\": {\"domain\": [0, 1]}},\n", " \"tooltip\": [{\"field\": \"repo\", \"type\": \"nominal\"},\n", " {\"field\": \"merged\", \"type\": \"quantitative\"},\n", " {\"field\": \"agent_prs\", \"type\": \"quantitative\"}],\n", " },\n", "}\n", "out = FIGURES / \"02-workspace-merge-rates.vega.json\"\n", "out.write_text(json.dumps(spec, indent=2) + \"\\n\")\n", "print(\"wrote\", out)" ] }, { "cell_type": "code", "execution_count": 24, "id": "9d2973f0", "metadata": { "execution": { "iopub.execute_input": "2026-06-11T16:42:12.685286Z", "iopub.status.busy": "2026-06-11T16:42:12.685219Z", "iopub.status.idle": "2026-06-11T16:42:12.688423Z", "shell.execute_reply": "2026-06-11T16:42:12.688037Z" }, "tags": [ "export" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "agent_prs.csv rows=75 sha256=a91da3bb37b3cfd5c02bf75ee9a218a2da395a9591603d107a2bb024bde82351\n", "pr_quality_eras.csv rows=2 sha256=f2a00859b1bd87845cba4afdb1ad99397baa438c6a6c92c340c11f8ba111b79e\n", "construction_cadence.csv rows=27 sha256=4d7ba368990a502b8b2247a23b68d7f24696d2e7952e4b809232ded636ac8059\n", "loc_by_subsystem.csv rows=38 sha256=9850cc01928a8dfa99bf31e675e3bd314ba9099573e640139efe261c65ccb9d6\n", "incident_registry.csv rows=11 sha256=3b14ad7cf2ea34773ba934f22365dab587095f09617818583c3d33e6654a80bc\n", "learnings_catalog.csv rows=8 sha256=5bf7b455ceab0e03faa529f22ed4506efc320d09f7286f62340973fef409dbd3\n", "workspace_summary.csv rows=4 sha256=af2a5b9303707e6fa3d989f6cb09199a56bb9a37d30396a8bfeca3f79081cbdd\n", "\n", "figure: 01-cadence-and-incidents.vega.json\n", "figure: 02-workspace-merge-rates.vega.json\n", "figure: 03-quality-before-after.vega.json\n", "figure: 04-blast-radius.vega.json\n", "\n", "all checks pass\n" ] } ], "source": [ "# Derived datasets written by the build cells; sha256 digests for data/manifest.yaml.\n", "derived = [\"agent_prs.csv\", \"pr_quality_eras.csv\", \"construction_cadence.csv\",\n", " \"loc_by_subsystem.csv\", \"incident_registry.csv\",\n", " \"learnings_catalog.csv\", \"workspace_summary.csv\"]\n", "for name in derived:\n", " path = DATA / name\n", " rows = sum(1 for _ in open(path)) - 1\n", " digest = hashlib.sha256(path.read_bytes()).hexdigest()\n", " print(f\"{name:<28} rows={rows:<5} sha256={digest}\")\n", "print()\n", "for f in sorted(FIGURES.glob(\"*.vega.json\")):\n", " print(\"figure:\", f.name)\n", "print()\n", "print(\"all checks pass\")" ] }, { "cell_type": "markdown", "id": "c4b05175", "metadata": {}, "source": [ "Motion figures (the kanban dispatch loop, the fixer self-loop, the janitor\n", "saga) are mechanism illustrations rather than data figures; they are produced\n", "in the site's animation pipeline, not by this notebook." ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.13" } }, "nbformat": 4, "nbformat_minor": 5 }