ckb/.agents/skills/ckb-export-starlight/scripts/export_starlight.py
2026-07-17 14:45:15 +02:00

586 lines
20 KiB
Python

#!/usr/bin/env python3
# Copyright 2026 Michał Kopeć
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Export wiki/ into an Astro + Starlight-consumable form at outputs/starlight/.
Usage:
python3 export_starlight.py --mode {full,content-only} [--include-meta] [--today YYYY-MM-DD]
Run from the repository root (the directory containing wiki/ and outputs/).
--mode full regenerate the whole outputs/starlight/ project: scaffold
(package.json, astro.config.mjs, tsconfig.json,
src/content.config.ts, splash index.mdx) plus content.
--mode content-only regenerate only src/content/docs/** and public/assets/**
(and a SIDEBAR-SNIPPET.md), leaving any existing scaffold
files untouched. Use this if dropping the output into a
Starlight project you already have.
--include-meta also publish wiki/log.md (as Change Log) and
wiki/error-book.md (as Error Book). Omit to keep those
internal-only.
--today override "today" for staleness checks (default: real
today). Mainly useful for reproducible test runs.
"""
import argparse
import json
import re
import shutil
import sys
from datetime import date
from pathlib import Path
REPO_ROOT = Path.cwd()
WIKI = REPO_ROOT / "wiki"
OUT = REPO_ROOT / "outputs" / "starlight"
RESERVED_ROOT_ROUTES = {
"index.md": "routing-table",
"overview.md": "overview",
"log.md": "changelog",
"error-book.md": "error-book",
}
def yaml_quote(s):
s = str(s).replace("\\", "\\\\").replace('"', '\\"')
return f'"{s}"'
def parse_frontmatter(text):
"""Minimal parser for this repo's flat key: value frontmatter (no lists/nesting)."""
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---", 4)
if end == -1:
return {}, text
fm_block = text[4:end]
body = text[end + 4 :].lstrip("\n")
fm = {}
for line in fm_block.splitlines():
line = line.strip()
if not line or ":" not in line:
continue
key, _, value = line.partition(":")
key = key.strip()
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
value = value[1:-1]
fm[key] = value
return fm, body
def extract_h1(body):
lines = body.splitlines()
for i, line in enumerate(lines):
stripped = line.strip()
if stripped.startswith("# "):
title = stripped[2:].strip()
remaining = lines[:i] + lines[i + 1 :]
# drop one immediately-following blank line so we don't leave a gap
if i < len(remaining) and remaining[i].strip() == "":
remaining = remaining[:i] + remaining[i + 1 :]
return title, "\n".join(remaining).lstrip("\n")
if stripped:
break # first non-blank line isn't an H1 - stop looking
return None, body
def wiki_relpath_to_route(relpath):
"""relpath is a path string relative to wiki/, e.g. 'entities/adam-betts.md'."""
relpath = relpath.lstrip("/")
if relpath in RESERVED_ROOT_ROUTES:
return "/" + RESERVED_ROOT_ROUTES[relpath] + "/"
p = Path(relpath)
parts = list(p.parts)
if parts and parts[-1].endswith(".md"):
parts[-1] = parts[-1][:-3]
if parts and parts[-1] == "index":
parts = parts[:-1]
if not parts:
return "/"
return "/" + "/".join(parts) + "/"
def out_path_for(relpath):
"""Where a wiki/<relpath> file's exported page lives under src/content/docs/."""
if relpath in RESERVED_ROOT_ROUTES:
return Path(RESERVED_ROOT_ROUTES[relpath] + ".md")
return Path(relpath)
def collect_wiki_pages():
pages = [] # list of (relpath_str, absolute_path)
for path in sorted(WIKI.rglob("*.md")):
relpath = path.relative_to(WIKI).as_posix()
pages.append((relpath, path))
return pages
def build_title_index(pages):
"""title -> route, for resolving bare [[Wikilinks]]."""
index = {}
for relpath, path in pages:
text = path.read_text(encoding="utf-8")
_, body = parse_frontmatter(text)
title, _ = extract_h1(body)
route = wiki_relpath_to_route(relpath)
if title:
index.setdefault(title, route)
index.setdefault(title.lower(), route)
slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
if slug:
index.setdefault(slug, route)
rel_no_ext = relpath[:-3] if relpath.endswith(".md") else relpath
index.setdefault(rel_no_ext, route)
index.setdefault(rel_no_ext.lower(), route)
stem = Path(relpath).stem
if stem != "index":
index.setdefault(stem, route)
index.setdefault(stem.lower(), route)
return index
WIKILINK_ADJACENT_RE = re.compile(r"(\[[^\]]+\]\([^)]+\))\s*\(\[\[[^\]]+\]\]\)")
WIKILINK_BEFORE_LINK_RE = re.compile(r"\[\[[^\]]+\]\]\s*/\s*(\[[^\]]+\]\([^)]+\))")
BARE_WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
WIKI_ABS_NON_MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(/wiki/([^)#]+)(#[^)]*)?\)")
WIKI_ABS_LINK_RE = re.compile(r"\]\(/wiki/([^)#]+)(#[^)]*)?\)")
CASCADE_LINK_RE = re.compile(r"\[([^\]]+)\]\((?:linked|libs)/[^)]+\)")
IMAGE_REL_RE = re.compile(r"!\[([^\]]*)\]\(\.\./assets/([^)]+)\)")
def rewrite_body(body, title_index, unresolved_wikilinks, copied_assets):
# 1. strip wikilinks that ride alongside a markdown link (either order:
# "[text](path) ([[Wikilink]])" or the legacy "[[Wikilink]] / [text](path)")
body = WIKILINK_ADJACENT_RE.sub(r"\1", body)
body = WIKILINK_BEFORE_LINK_RE.sub(r"\1", body)
# 2. resolve (or strip) any remaining bare [[Wikilinks]]
def _bare(m):
text = m.group(1)
route = title_index.get(text) or title_index.get(text.lower())
if route:
return f"[{text}]({route})"
unresolved_wikilinks.append(text)
return text
body = BARE_WIKILINK_RE.sub(_bare, body)
# 3. defuse /wiki/... links to non-Markdown source files not exported as pages
def _abs_non_md_link(m):
text = m.group(1)
relpath = m.group(2)
if relpath.endswith(".md"):
return m.group(0)
if relpath == "graph/edges.json":
return f"`{text}` (rendered below)"
return f"{text} (source file: `/wiki/{relpath}`)"
body = WIKI_ABS_NON_MD_LINK_RE.sub(_abs_non_md_link, body)
# 4. rewrite /wiki/... absolute links to site routes
def _abs_link(m):
relpath = m.group(1)
frag = m.group(2) or ""
route = wiki_relpath_to_route(relpath)
if frag:
route = route.rstrip("/") + "/" + frag
return f"]({route})"
body = WIKI_ABS_LINK_RE.sub(_abs_link, body)
# 5. defuse linked/ and libs/ cascade references (not part of this export)
def _cascade(m):
text, target = m.group(1), m.group(0)
path = re.search(r"\((?:linked|libs)/([^)]+)\)", target).group(0)[1:-1]
return f"{text} (upstream reference: `{path}`, not included in this export)"
body = CASCADE_LINK_RE.sub(_cascade, body)
# 6. copy relative person/asset images and rewrite to /assets/...
def _image(m):
alt, rel = m.group(1), m.group(2)
src = WIKI / "assets" / rel
if src.is_file():
dest = OUT / "public" / "assets" / rel
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(src, dest)
copied_assets.append(rel)
return f"![{alt}](/assets/{rel})"
body = IMAGE_REL_RE.sub(_image, body)
return body
def build_metadata_block(fm, today):
lines = []
meta_bits = []
for label, key in (("Type", "type"), ("Confidence", "confidence"),
("Quality", "quality"), ("Last updated", "last_updated"),
("Retention", "retention")):
if fm.get(key):
meta_bits.append(f"**{label}:** {fm[key]}")
if meta_bits:
lines.append(":::note[Knowledge base metadata]")
lines.append(" · ".join(meta_bits))
lines.append(":::")
lines.append("")
if fm.get("resource"):
resource = fm["resource"]
if resource.startswith("http://") or resource.startswith("https://"):
lines.append(f":::note[Source]\n[{resource}]({resource})\n:::\n")
else:
lines.append(f":::note[Source]\n`{resource}`\n:::\n")
if fm.get("superseded_by"):
lines.append(
f":::caution[Superseded]\nThis page is superseded by "
f"`{fm['superseded_by']}`.\n:::\n"
)
if fm.get("supersedes"):
lines.append(
f":::tip[Supersedes]\nThis page supersedes `{fm['supersedes']}`.\n:::\n"
)
return "\n".join(lines)
def compute_badge(fm, today):
if fm.get("superseded_by"):
return {"text": "Superseded", "variant": "danger"}
try:
if fm.get("last_updated") and fm.get("freshness_window_days"):
last_updated = date.fromisoformat(fm["last_updated"])
window = int(fm["freshness_window_days"])
if (today - last_updated).days > window:
return {"text": "Stale", "variant": "caution"}
except ValueError:
pass
try:
if fm.get("confidence") and float(fm["confidence"]) < 0.5:
return {"text": "Low confidence", "variant": "note"}
except ValueError:
pass
return None
def render_page(relpath, path, title_index, today, report):
text = path.read_text(encoding="utf-8")
fm, body = parse_frontmatter(text)
h1_title, body = extract_h1(body)
title = h1_title or Path(relpath).stem.replace("-", " ").title()
unresolved = []
body = rewrite_body(body, title_index, unresolved, report["copied_assets"])
report["unresolved_wikilinks"].extend(
f"{relpath}: [[{t}]]" for t in unresolved
)
meta_block = build_metadata_block(fm, today)
badge = compute_badge(fm, today)
fm_lines = ["---", f"title: {yaml_quote(title)}"]
if fm.get("tldr"):
fm_lines.append(f"description: {yaml_quote(fm['tldr'])}")
if badge:
fm_lines.append("sidebar:")
fm_lines.append(" badge:")
fm_lines.append(f" text: {yaml_quote(badge['text'])}")
fm_lines.append(f" variant: {badge['variant']}")
fm_lines.append("---")
out_body = (meta_block + "\n" + body) if meta_block else body
return "\n".join(fm_lines) + "\n\n" + out_body.strip() + "\n"
def normalize_edges(raw_edges):
if isinstance(raw_edges, dict):
raw_edges = raw_edges.get("edges", [])
if not isinstance(raw_edges, list):
return []
normalized = []
for edge in raw_edges:
if not isinstance(edge, dict):
continue
normalized.append(
{
"from": edge.get("from") or edge.get("source") or "",
"type": edge.get("type") or edge.get("relationship") or "",
"to": edge.get("to") or edge.get("target") or "",
"note": edge.get("note") or edge.get("evidence") or "",
"source_files": edge.get("source_files") or [],
}
)
return normalized
def table_cell(value):
return str(value).replace("|", "\\|").replace("\n", " ")
def render_edges_table(edges):
edges = normalize_edges(edges)
if not edges:
return "\n*No edges recorded yet.*\n"
rows = sorted(edges, key=lambda e: (e.get("from", ""), e.get("type", "")))
lines = ["", "## All relationships", "",
"| From | Relationship | To | Evidence | Sources |", "|---|---|---|---|---|"]
for e in rows:
sources = ", ".join(f"`{table_cell(s)}`" for s in e.get("source_files", []))
lines.append(
f"| `{table_cell(e.get('from', ''))}` | {table_cell(e.get('type', ''))} | "
f"`{table_cell(e.get('to', ''))}` | {table_cell(e.get('note', ''))} | {sources} |"
)
return "\n".join(lines) + "\n"
def derive_site_title():
overview = WIKI / "overview.md"
if overview.is_file():
_, body = parse_frontmatter(overview.read_text(encoding="utf-8"))
title, _ = extract_h1(body)
if title:
return title
return REPO_ROOT.name
SPLASH_TEMPLATE = """---
title: {title}
description: {description}
template: splash
hero:
tagline: {tagline}
actions:
- text: Browse entities
link: /entities/
icon: right-arrow
- text: Browse sources
link: /sources/
icon: right-arrow
- text: Knowledge base index
link: /routing-table/
icon: document
variant: minimal
---
import {{ CardGrid, Card }} from '@astrojs/starlight/components';
<CardGrid>
\t<Card title="Entities" icon="document">
\t\tPeople, organisations, products and concepts tracked in this knowledge base.
\t</Card>
\t<Card title="Sources" icon="open-book">
\t\tOne-page summaries of ingested raw material, with provenance back to the original source.
\t</Card>
\t<Card title="Graph" icon="random">
\t\tTyped relationships between entities and sources - who works for whom, what depends on what, what contradicts what.
\t</Card>
</CardGrid>
"""
PACKAGE_JSON_TEMPLATE = """{{
"name": "{name}",
"type": "module",
"version": "0.0.1",
"scripts": {{
"dev": "astro dev",
"start": "astro dev",
"build": "astro build",
"preview": "astro preview"
}},
"dependencies": {{
"@astrojs/starlight": "^0.41.3",
"astro": "^7.0.9"
}}
}}
"""
TSCONFIG_TEMPLATE = """{
"extends": "astro/tsconfigs/strict"
}
"""
CONTENT_CONFIG_TEMPLATE = """import { defineCollection } from 'astro:content';
import { docsLoader } from '@astrojs/starlight/loaders';
import { docsSchema } from '@astrojs/starlight/schema';
export const collections = {
docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }),
};
"""
def build_sidebar_js(top_dirs, include_meta):
items = []
items.append(" { label: 'Knowledge Base Index', slug: 'routing-table' },")
items.append(" { label: 'Overview', slug: 'overview' },")
if include_meta:
items.append(" { label: 'Error Book', slug: 'error-book' },")
items.append(" { label: 'Change Log', slug: 'changelog' },")
for d in sorted(top_dirs):
label = d.replace("-", " ").title()
items.append(
f" {{ label: '{label}', items: [{{ autogenerate: {{ directory: '{d}' }} }}] }},"
)
return "[\n" + "\n".join(items) + "\n ]"
def astro_config_template(site_title, top_dirs, include_meta):
sidebar_js = build_sidebar_js(top_dirs, include_meta)
return f"""import {{ defineConfig }} from 'astro/config';
import starlight from '@astrojs/starlight';
export default defineConfig({{
integrations: [
starlight({{
title: {yaml_quote(site_title)},
sidebar: {sidebar_js},
}}),
],
}});
"""
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", choices=["full", "content-only"], required=True)
parser.add_argument("--include-meta", action="store_true")
parser.add_argument("--today", default=None)
parser.add_argument(
"--site-title", default=None,
help="Override the site title (default: derived from wiki/overview.md's H1)",
)
args = parser.parse_args()
if not WIKI.is_dir():
sys.exit(f"No wiki/ directory found under {REPO_ROOT} - run this from the repo root.")
today = date.fromisoformat(args.today) if args.today else date.today()
pages = collect_wiki_pages()
title_index = build_title_index(pages)
docs_dir = OUT / "src" / "content" / "docs"
assets_dir = OUT / "public" / "assets"
if args.mode == "full" and OUT.exists():
shutil.rmtree(OUT)
else:
if docs_dir.exists():
shutil.rmtree(docs_dir)
if assets_dir.exists():
shutil.rmtree(assets_dir)
docs_dir.mkdir(parents=True, exist_ok=True)
report = {"copied_assets": [], "unresolved_wikilinks": [], "pages": 0, "skipped_meta": []}
top_dirs = set()
edges_path = WIKI / "graph" / "edges.json"
edges = json.loads(edges_path.read_text(encoding="utf-8")) if edges_path.is_file() else []
for relpath, path in pages:
if relpath in ("log.md", "error-book.md") and not args.include_meta:
report["skipped_meta"].append(relpath)
continue
rendered = render_page(relpath, path, title_index, today, report)
if relpath == "graph/index.md":
rendered = rendered.rstrip("\n") + "\n" + render_edges_table(edges)
dest = docs_dir / out_path_for(relpath)
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(rendered, encoding="utf-8")
report["pages"] += 1
parts = Path(relpath).parts
if len(parts) > 1:
top_dirs.add(parts[0])
site_title = args.site_title or derive_site_title()
if args.mode == "full":
(OUT / "package.json").write_text(
PACKAGE_JSON_TEMPLATE.format(name=re.sub(r"[^a-z0-9-]", "-", site_title.lower())),
encoding="utf-8",
)
(OUT / "tsconfig.json").write_text(TSCONFIG_TEMPLATE, encoding="utf-8")
(OUT / "src" / "content.config.ts").write_text(CONTENT_CONFIG_TEMPLATE, encoding="utf-8")
(OUT / "astro.config.mjs").write_text(
astro_config_template(site_title, top_dirs, args.include_meta), encoding="utf-8"
)
overview_fm, overview_body = ({}, "")
overview_path = WIKI / "overview.md"
if overview_path.is_file():
overview_fm, overview_body = parse_frontmatter(overview_path.read_text(encoding="utf-8"))
tagline = overview_fm.get("tldr", "A cascade knowledge base.")
(docs_dir / "index.mdx").write_text(
SPLASH_TEMPLATE.format(
title=yaml_quote(site_title),
description=yaml_quote(tagline),
tagline=yaml_quote(tagline),
),
encoding="utf-8",
)
else:
snippet = (
"# Sidebar snippet\n\n"
"Paste this into your own `astro.config.mjs`, inside "
"`starlight({ ... })`:\n\n"
"```js\n"
f"sidebar: {build_sidebar_js(top_dirs, args.include_meta)},\n"
"```\n\n"
"Copy `src/content/docs/**` into your project's equivalent "
"directory, and `public/assets/**` into your project's "
"`public/assets/`.\n"
)
(OUT / "SIDEBAR-SNIPPET.md").write_text(snippet, encoding="utf-8")
# Validate internal links resolve to a generated file
broken = []
for relpath, _ in pages:
if relpath in ("log.md", "error-book.md") and not args.include_meta:
continue
dest = docs_dir / out_path_for(relpath)
text = dest.read_text(encoding="utf-8")
for m in re.finditer(r"\]\((/[a-zA-Z0-9_\-./]+/)\)", text):
route = m.group(1)
candidate_dir = docs_dir / route.strip("/")
candidate_file = docs_dir / (route.strip("/") + ".md")
candidate_index = candidate_dir / "index.md"
candidate_mdx = docs_dir / (route.strip("/") + ".mdx") if route.strip("/") else docs_dir / "index.mdx"
if not (candidate_file.is_file() or candidate_index.is_file() or candidate_mdx.is_file()):
broken.append(f"{relpath} -> {route}")
print(f"mode: {args.mode}")
print(f"pages exported: {report['pages']}")
print(f"assets copied: {len(report['copied_assets'])}")
if report["skipped_meta"]:
print(f"meta pages skipped (--include-meta not set): {', '.join(report['skipped_meta'])}")
if report["unresolved_wikilinks"]:
print(f"unresolved [[wikilinks]] (stripped to plain text): {len(report['unresolved_wikilinks'])}")
for w in report["unresolved_wikilinks"]:
print(f" - {w}")
if broken:
print(f"WARNING: {len(broken)} internal link(s) do not resolve to a generated page:")
for b in broken:
print(f" - {b}")
print(f"output: {OUT}")
if __name__ == "__main__":
main()