#!/usr/bin/env python3
"""
regenerate_surfaces.py — bring every derived surface into agreement with data/registry.json.
═══════════════════════════════════════════════════════════════════════════════
THE PROBLEM THIS SCRIPT SOLVES
═══════════════════════════════════════════════════════════════════════════════
data/registry.json is the canonical source of truth for all deposits.
But Alexanarch serves the archive through MANY derived surfaces, and if any
of them goes stale relative to registry.json the archive is internally
inconsistent — a deposit can exist in the registry but be invisible to anyone
browsing the site.
The derived surfaces are:
1. s/browse/index.html — static browse page (every deposit, no JS)
2. data/browse-index.json — compact JSON of all deposits (used by tools)
3. data/chunks/registry/*.json — 1MB-targeted chunks for streaming
4. data/chunks/registry/_index.json — chunk catalog metadata
5. sitemap.xml — XML sitemap for crawlers
6. SHA256SUMS.txt — content-addressable checksums
The mint-axn.yml workflow handles registry + record pages + deposits/*.md,
but does NOT touch ANY of the six derived surfaces above. The result is
that any deposit minted through the auto-flow leaves the archive in an
inconsistent state until someone runs this script.
THE FIX: run this script after EVERY change to data/registry.json.
═══════════════════════════════════════════════════════════════════════════════
USAGE
═══════════════════════════════════════════════════════════════════════════════
python3 scripts/regenerate_surfaces.py # regenerate all surfaces
python3 scripts/regenerate_surfaces.py --dry-run # show what would change
python3 scripts/regenerate_surfaces.py --only browse # only browse page
python3 scripts/regenerate_surfaces.py --only browse,chunks # subset
Available surfaces: browse, browse-index, chunks, sitemap, sha256sums
After running this script the archive is internally consistent.
The script is idempotent — running it twice produces the same result.
═══════════════════════════════════════════════════════════════════════════════
"""
import argparse
import hashlib
import json
import os
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
# Import canonical navbar renderer
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.render_navbar import render_navbar
REPO_ROOT = Path(__file__).resolve().parent.parent
REGISTRY_PATH = REPO_ROOT / "data" / "registry.json"
# Pre-overwrite receipt mechanism (workplan item 8.13).
# Each regenerator write logs an auto-receipt to data/pre-overwrite-receipts.log
# so the audit trail captures EVERY surface-overwrite event, regardless of
# whether the write came from an instance or from the regenerator. The
# receipt log is the structural complement to the discipline of running
# scripts/pre_overwrite.py before any ad-hoc edit.
sys.path.insert(0, str(Path(__file__).resolve().parent))
try:
from overwrite_guard import issue_auto_receipt
_OVERWRITE_GUARD_AVAILABLE = True
except ImportError:
_OVERWRITE_GUARD_AVAILABLE = False
ALL_SURFACES = ["state", "browse", "browse-index", "hex-to-deposit", "chunks", "sitemap", "sha256sums", "wiki", "graph", "homepage-noscript", "api-index", "search-index", "search-static", "dynamic-counts", "semantic-addresses"]
def _receipt(path, reason: str = "regenerate_surfaces write"):
"""Issue an auto-receipt for a regenerator write. No-op if guard module
isn't importable (defensive — the script should still work even if the
guard module is missing or temporarily broken)."""
if not _OVERWRITE_GUARD_AVAILABLE:
return
try:
issue_auto_receipt(path, actor="regenerate_surfaces", reason=reason,
instance_id=os.environ.get("ALEXANARCH_INSTANCE_ID", "regenerator"))
except Exception as e:
# Don't let receipt failure block a regenerator run; log and continue.
print(f"[regenerate_surfaces] warning: receipt failed for {path}: {e}", file=sys.stderr)
# ──────────────────────────────────────────────────────────────────────────────
# Helpers
# ──────────────────────────────────────────────────────────────────────────────
def load_registry():
with open(REGISTRY_PATH) as f:
return json.load(f)
def esc_html(s):
if s is None:
return ""
return (str(s)
.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace('"', """))
def esc_xml(s):
if s is None:
return ""
return (str(s)
.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace("'", "'")
.replace('"', """))
# ──────────────────────────────────────────────────────────────────────────────
# Surface 1: s/browse/index.html
# ──────────────────────────────────────────────────────────────────────────────
BROWSE_HEADER = """
Browse — Alexanarch ({total} deposits)
Complete Deposit Registry
{total} deposits · sorted by deposit number, oldest first · for newest see home page
\n'
f'\n'
)
# JS pagination — clean raw triple-string, no Python-string escape gymnastics
js = r""""""
html = _WIKI_HTML_HEAD + body_pre + js + _WIKI_HTML_TAIL
json_path = REPO_ROOT / "data" / "wiki-entries.json"
html_path = WIKI_PATH
if dry_run:
print(f" [dry-run] data/wiki-entries.json would be {len(json_str):,} bytes ({len(flat)} entries)")
print(f" [dry-run] s/wiki/index.html would be {len(html):,} bytes (JS-paginated, 25 per page)")
chunk_dirs = list((REPO_ROOT / "s" / "wiki").glob("chunk-*"))
if chunk_dirs:
print(f" [dry-run] would delete {len(chunk_dirs)} chunk dir(s)")
return
json_path.parent.mkdir(parents=True, exist_ok=True)
_receipt(json_path)
with open(json_path, "w", encoding="utf-8") as f:
f.write(json_str + "\n")
html_path.parent.mkdir(parents=True, exist_ok=True)
_receipt(html_path)
html_path.write_text(html, encoding="utf-8")
# Remove obsolete chunk directories
import shutil
chunk_dirs = list((REPO_ROOT / "s" / "wiki").glob("chunk-*"))
for cdir in chunk_dirs:
if cdir.is_dir():
shutil.rmtree(cdir)
print(f" ✓ data/wiki-entries.json ({len(json_str):,} bytes, {len(flat)} entries)")
print(f" ✓ s/wiki/index.html ({len(html):,} bytes, JS-paginated 25 per page)")
if chunk_dirs:
print(f" ✓ removed {len(chunk_dirs)} obsolete chunk director{'y' if len(chunk_dirs)==1 else 'ies'}")
# Task 2 of EA-RETRIEVAL-DENSITY-01: after the dataset is written, delegate
# to publish_wiki_entries.py which generates one HTML page per entry at
# /s/wiki/{n}/ and OVERWRITES s/wiki/index.html with a static alphabetical
# listing (retiring the JS-only pagination that hid 97% of entries from
# non-JS crawlers). Overwrite is intentional: the paginated version above
# is written first as a defensive fallback in case publish_wiki_entries
# fails, then replaced by the crawler-friendly static list.
try:
import subprocess
result = subprocess.run(
["python3", str(REPO_ROOT / "scripts" / "publish_wiki_entries.py")],
capture_output=True, text=True, cwd=str(REPO_ROOT),
)
if result.returncode == 0:
for line in result.stdout.strip().split("\n"):
print(f" {line}")
else:
print(f" ! publish_wiki_entries failed (returncode={result.returncode}):")
print(f" stderr: {result.stderr[:400]}")
except Exception as e:
print(f" ! publish_wiki_entries invocation error: {e}")
CITATION_GRAPH_PATH = REPO_ROOT / "data" / "citation-graph.json"
GRAPH_PATH = REPO_ROOT / "s" / "graph" / "index.html"
def _load_citation_graph():
if not CITATION_GRAPH_PATH.exists():
return None
with open(CITATION_GRAPH_PATH) as f:
return json.load(f)
def regenerate_graph(reg, dry_run=False):
"""Project the graph tab from entity_triples + citation_graph.
Renders subject→predicate→object triples in the same visual format the
hand-curated graph used to use ([observed]/[inferred]/[performative]).
Sourced from:
- entity-index.json: each concept's entity_triples[] array
- citation-graph.json: deposit→deposit edges become (deposit,
cites, deposit) triples
"""
eidx = _load_entity_index()
citation_graph = _load_citation_graph()
# Gather triples from entity_triples first
concept_triples = []
if eidx:
for term, c in eidx.get("concepts", {}).items():
for tr in (c.get("entity_triples") or [])[:3]: # top 3 per concept to avoid explosion
s = tr.get("subject") or term
p = tr.get("predicate")
o = tr.get("object")
ev = tr.get("evidence_status") or "observed"
if s and p and o:
concept_triples.append((s, p, o, ev))
# Cap concept triples to keep page size reasonable
concept_triples = concept_triples[:1500]
# Gather deposit-to-deposit citation edges (cap to 500 for page size)
deposit_triples = []
if citation_graph:
for e in (citation_graph.get("edges") or [])[:500]:
src = e.get("source_axn") or f"#{e.get('source_deposit')}"
tgt = e.get("target_axn") or f"#{e.get('target_deposit')}"
via = e.get("via") or "cites"
deposit_triples.append((src, via, tgt, "observed"))
total_concept_edges = sum(len((c.get("entity_triples") or [])) for c in (eidx or {}).get("concepts", {}).values()) if eidx else 0
total_citation_edges = len((citation_graph or {}).get("edges") or [])
parts = []
parts.append(_GRAPH_HTML_HEAD)
parts.append(
f'
'
f'{total_concept_edges + total_citation_edges:,} total edges '
f'projected from data/entity-index.json '
f'and data/citation-graph.json. '
f'[observed] '
f'[inferred] '
f'[performative]
\n'
)
if concept_triples:
parts.append('
'
f'Concept Relations '
f'(showing first {len(concept_triples):,} of {total_concept_edges:,})
\n')
for s, p, o, ev in concept_triples:
cls = "evo" if ev == "observed" else "evi" if ev == "inferred" else "evp"
parts.append(
f'
'
f'{esc_html(s)}'
f'{esc_html(p)}'
f'{esc_html(o)} [{esc_html(ev)}]'
f'
\n'
)
if deposit_triples:
parts.append('
'
f'Deposit Citation Edges '
f'(showing first {len(deposit_triples):,} of {total_citation_edges:,})
\n')
for s, p, o, ev in deposit_triples:
parts.append(
f'
'
f'{esc_html(s)}'
f'{esc_html(p)}'
f'{esc_html(o)} [observed]'
f'
\n'
)
parts.append(_GRAPH_HTML_TAIL)
html = "".join(parts)
size = len(html.encode("utf-8"))
if dry_run:
print(f" [dry-run] {GRAPH_PATH} would be {size:,} bytes, {len(concept_triples)+len(deposit_triples)} edges shown")
return
GRAPH_PATH.parent.mkdir(parents=True, exist_ok=True)
_receipt(GRAPH_PATH)
GRAPH_PATH.write_text(html, encoding="utf-8")
print(f" ✓ s/graph/index.html ({size:,} bytes, {len(concept_triples)+len(deposit_triples)} edges)")
# HTML templates for wiki and graph — keep the existing visual style/classes
_WIKI_STYLE = """"""
_WIKI_HTML_HEAD = (
''
''
'Wiki — Alexanarch'
+ _WIKI_STYLE +
'
'
''
'\n'
)
# ──────────────────────────────────────────────────────────────────────────────
# Driver
# ──────────────────────────────────────────────────────────────────────────────
# ──────────────────────────────────────────────────────────────────────────────
# state.json + homepage noscript fallback + api/index.json — single-source-of-truth
# ──────────────────────────────────────────────────────────────────────────────
def regenerate_state(reg, dry_run=False):
"""Wraps scripts/generate_state.py — produces data/state.json as the
canonical generated source for all displayed counts.
"""
# Defer to the standalone module so it can also be run directly
sys.path.insert(0, str(REPO_ROOT / 'scripts'))
import generate_state as _gs
if dry_run:
st = _gs.build_state()
print(f" [dry-run] data/state.json would be regenerated "
f"(deposits={st['deposits']['total']}, captures={st['corpus']['captures']})")
return
_gs.main()
def regenerate_homepage_noscript(reg, dry_run=False):
"""Rewrite the block in index.html so it shows
the same N latest deposits the JavaScript renders. Previously hand-maintained
with 4 hardcoded records; now generated from registry on every regen so
no-JS readers and crawlers see actual current state.
The contract: the static fallback must equal what JS would render for the
'Recent Deposits' section. Both pull from data/registry.json — JS at
runtime, this generator at build time.
"""
index_path = REPO_ROOT / 'index.html'
if not index_path.exists():
print(f" ⚠ {index_path} not found — skipping")
return
# Match the JS slice: filter SUPERSEDED out (canonical reference for an
# older version is the Version history on the current version's page,
# not a separate card here), then take the last 5 by deposit_number,
# reverse for newest-first. Keeps the static fallback in parity with
# the runtime JS view.
deposits = sorted(reg['deposits'], key=lambda d: d.get('deposit_number', 0))
active_deposits = [d for d in deposits if d.get('status', 'ACTIVE') != 'SUPERSEDED']
recent = list(reversed(active_deposits[-5:]))
cards = []
for d in recent:
n = d.get('deposit_number', 0)
axn = esc_html(d.get('axn', ''))
title = esc_html(d.get('title', '(untitled)'))
creator = esc_html(d.get('creator', ''))
date = esc_html(d.get('date', ''))
content_type_raw = d.get('content_type', '') or ''
# Card-hardening (MANUS, 2026-07-08): content_type is a category label
# (like "Philological erratum"), not a specification paragraph. Clip
# defensively so an accidentally-long entry cannot balloon the card
# vertically. The full content_type remains in the deposit record page.
if len(content_type_raw) > 140:
content_type = esc_html(content_type_raw[:140].rsplit(' ', 1)[0] + '…')
else:
content_type = esc_html(content_type_raw)
desc = esc_html((d.get('description', '') or '')[:250])
if len(d.get('description', '') or '') > 250:
desc = desc + '...'
# Version chip + status badge (mirrors browse-card logic)
version = d.get('version', '')
status = d.get('status', 'ACTIVE')
superseded_by_n = d.get('superseded_by_deposit_number')
in_real_series = bool(d.get('version_series_id'))
version_chip = ''
if version and (version != 'v1.0' or in_real_series):
version_chip = (f' {esc_html(version)}')
status_banner = ''
opacity = '1'
if status == 'SUPERSEDED' and superseded_by_n:
opacity = '0.65'
status_banner = (f'
'
f''
)
cards.append(card)
new_noscript = (
''
)
html = index_path.read_text()
# Replace the existing noscript block
import re
noscript_pattern = re.compile(r'', re.DOTALL)
if not noscript_pattern.search(html):
print(f" ⚠ index.html has no