"""
Coding of Hacker News thread 49696125 ("How to write an effective software
design document", posted 2026-09-14) for the guide at
https://livemark.ai/guides/design-docs-nobody-reads.

Run from this directory. It reads hn-49696125.json if present (the thread as
fetched from https://hn.algolia.com/api/v1/items/49696125 on 2026-09-17T13:51Z)
and otherwise fetches it, walks the comment tree in order, applies the hand
codes below (one primary theme per comment, optional secondary), and prints
every count the guide quotes. The codes are one person's reading on
2026-09-17; the 14 indices in `ambiguous` were judgement calls.
"""
import json, html, re, csv, sys, os, urllib.request
from collections import Counter

HERE = os.path.dirname(os.path.abspath(__file__))
JSON_PATH = os.path.join(HERE, "hn-49696125.json")
if not os.path.exists(JSON_PATH):
    with urllib.request.urlopen("https://hn.algolia.com/api/v1/items/49696125") as r:
        open(JSON_PATH, "wb").write(r.read())
d = json.load(open(JSON_PATH))

THEMES = {
    "A": "unread: getting anyone to read the doc is the problem",
    "B": "length-detail: too long, wrong level of detail",
    "C": "ai-slop: generated design docs are slop",
    "D": "ai-calculus: does AI change whether or how docs are needed",
    "E": "not-worth-it: build or prototype instead",
    "F": "worth-it: defence of docs",
    "G": "review-process-adoption: sign-off, reviewers, blame, getting a team to write them",
    "H": "scope-lifetime: design doc vs spec vs requirements; living vs frozen",
    "I": "praise-tips-structure: praise for the article, template suggestions, author replies",
    "J": "offtopic",
}

code = {
1:'J',2:'J',3:'I',4:'D',5:'G',6:'G',7:'G/E',8:'G',9:'G',10:'G',11:'G',12:'B',13:'I/B',14:'I/B',15:'H',16:'A',17:'A/D',18:'A/B',19:'A/G',20:'C',
21:'C',22:'C',23:'C/G',24:'I',25:'E',26:'F',27:'E',28:'F',29:'F',30:'E',31:'F/E',32:'H',33:'F',34:'F',35:'J',36:'E/G',37:'E',38:'J',39:'F',40:'F',
41:'F',42:'D',43:'D',44:'D',45:'D',46:'D/F',47:'D',48:'D',49:'D',50:'D',51:'D',52:'D',53:'D',54:'D/H',55:'D',56:'D',57:'D',58:'D',59:'D',60:'C/B',
61:'D',62:'A/C',63:'B',64:'G',65:'E/G',66:'G',67:'G',68:'G',69:'E/F',70:'F',71:'B/I',72:'E',73:'E/B',74:'E',75:'F/H',76:'E',77:'F/E',78:'F',79:'F',80:'D',
81:'D/F',82:'D',83:'D/F',84:'D',85:'D/E',86:'F',87:'F',88:'J/D',89:'F',90:'H',91:'H',92:'H',93:'F',94:'F/D',95:'F',96:'F',97:'F',98:'F',99:'I/B',100:'J',
101:'H/B',102:'H',103:'H',104:'H',105:'B/H',106:'H',107:'G/H',108:'G',109:'D',110:'D',111:'D',112:'F',113:'D',114:'J',115:'J',116:'J',117:'F/H',118:'H/B',119:'G/I',120:'F',
121:'H/D',122:'H',123:'H',124:'G',125:'F',126:'F/C',127:'D',128:'F/H',129:'I',130:'E',131:'F',132:'F',133:'F',134:'E/F',135:'I/G',136:'J',137:'A/B',138:'H',139:'J',140:'B/C',141:'J'}
ambiguous={7,13,31,60,62,65,69,77,83,88,105,118,128,134}

def strip(t):
    t = re.sub(r"<p>", "\n", t or "")
    t = re.sub(r"<[^>]+>", "", t)
    return html.unescape(t)

rows = []
def walk(n, depth):
    for c in n.get("children", []):
        rows.append({"id": c["id"], "author": c.get("author"), "depth": depth,
                     "created": c.get("created_at"), "text": strip(c.get("text"))})
        walk(c, depth + 1)
walk(d, 0)
assert len(rows) == len(code), (len(rows), len(code))
for k, r in enumerate(rows, 1):
    p, _, s = code[k].partition("/")
    r["idx"] = k; r["primary"] = p; r["secondary"] = s; r["ambiguous"] = k in ambiguous

n = len(rows); top = [r for r in rows if r["depth"] == 0]
print(f"comments {n}, top-level {len(top)}, authors {len({r['author'] for r in rows})}, "
      f"deleted/empty {sum(1 for r in rows if not r['text'].strip())}")
author = d.get("author"); by_author = sum(1 for r in rows if r["author"] == "mtlynch")
print(f"comments by the article author (mtlynch): {by_author}")

print("\nprimary theme, all comments | top-level")
ca = Counter(r["primary"] for r in rows); ct = Counter(r["primary"] for r in top)
for t, c in ca.most_common():
    print(f"  {t} {THEMES[t][:40]:40} {c:3} ({100*c/n:4.1f}%) | {ct.get(t,0):2} ({100*ct.get(t,0)/len(top):4.1f}%)")
print(f"  E+F worth-it fight {ca['E']+ca['F']} ({100*(ca['E']+ca['F'])/n:.1f}%), F:E {ca['F']}:{ca['E']}")
print(f"  A+B+C reader friction {ca['A']+ca['B']+ca['C']} ({100*(ca['A']+ca['B']+ca['C'])/n:.1f}%)")
print(f"  C+D about AI {ca['C']+ca['D']} ({100*(ca['C']+ca['D'])/n:.1f}%)")

def hits(pat, pool=rows):
    return [r for r in pool if re.search(pat, r["text"], re.I)]

unread_re = r"(nobody|no one|no-one|don'?t|doesn'?t|never|lazy|ignore|tl;?dr|hard to|challenge|problem|fatigue|get(ting)? (people|anyone|them)|have people)[^.!?]{0,60}\b(read|skim)|\b(read|skim)\w*[^.!?]{0,40}\b(nobody|no one|lazy|ignore|tl;?dr|fatigue|big tradeoff)"
print(f"\nsecond derivation of A by keyword: {len(hits(unread_re))} (coded A: {ca['A']})")
ai_re = r"\b(AI|LLMs?|agents?|Claude|generated|prompt)\b"
print(f"AI keyword (AI|LLM|agent|Claude|generated|prompt): {len(hits(ai_re))}")

print("\nreader's-contract vocabulary across all comments:")
for name, pat in [("deadline", r"deadline"), ("silence/silent", r"\bsilen"), ("sign off / signed off", r"sign(ed)?[- ]?off"),
                  ("approve/approval", r"approv"), ("decider / who decides", r"decider|who decides"), ("reviewer", r"reviewer")]:
    print(f"  {name}: {len(hits(pat))}")

F = [r for r in rows if r["primary"] == "F"]
think_re = r"\bthink|\bthought"
print(f"\nof {len(F)} worth-it comments: think/thought keyword {len(hits(think_re, F))}, review keyword {len(hits(r'review', F))}")

# Hand sub-code of the 32 worth-it comments (2026-09-17): what each says the
# doc is FOR. think = writing forces the author to think the problem through;
# agree = it is how several people or teams come to agree; review = a design
# is easier to review than finished code. Comments not listed give another
# reason or none.
F_SUBCODE = {70: "think", 77: "think", 96: "think", 117: "think", 125: "think", 128: "think", 131: "think",
             86: "agree", 87: "agree", 89: "agree", 93: "agree", 95: "agree", 133: "agree",
             98: "review", 78: "review"}
print("hand sub-code of worth-it comments:", dict(Counter(F_SUBCODE.values())), f"of {len(F)}")

G = [r for r in rows if r["primary"] == "G"]
print(f"review-process comments (G): {len(G)} -> ids {[r['id'] for r in G]}")
sub = {}
def size(c):
    return 1 + sum(size(x) for x in c.get("children", []))
def find(n, wanted):
    for c in n.get("children", []):
        if c["id"] == wanted: return c
        f = find(c, wanted)
        if f: return f
big = find(d, 49698166)
def authors(c, acc):
    acc.add(c.get("author")); [authors(x, acc) for x in c.get("children", [])]; return acc
print(f"subtree under 49698166: {size(big)} comments including the root, {len(authors(big, set()) - {None})} people")

with open(os.path.join(HERE, "hn-49696125-coded.csv"), "w", newline="") as f:
    w = csv.writer(f); w.writerow(["idx", "id", "depth", "created", "primary", "secondary", "ambiguous"])
    for r in rows: w.writerow([r["idx"], r["id"], r["depth"], r["created"], r["primary"], r["secondary"], r["ambiguous"]])
print("\nwrote hn-49696125-coded.csv (no author column; the thread is public at the URL above)")