#!/usr/bin/env -S uv run --script# /// script# dependencies = ["requests", "beautifulsoup4", "html5lib"]# ///"""concatenate every crime and punishment section in the local zim into one page.
usage: ./crime-page.py [--base http://127.0.0.1:8090] [--link-base ''] [-o out.html]
"""importargparseimportreimportsysimportrequestsfrombs4importBeautifulSoupfromurllib.parseimporturljoinHEADING_ID=re.compile(r"^Crime_and_[Pp]unishment$")SEARCH="/wiki/search?books.name={book}&pattern=crime+and+punishment&pageLength=50"SKIP_PREFIXES=("Talk:","User:","Research:")deffind_pages(base,book):html=requests.get(base+SEARCH.format(book=book),timeout=10).textsoup=BeautifulSoup(html,"html5lib")urls=[]forainsoup.select(".results a[href]"):href=str(a["href"])title=href.rsplit("/",1)[-1]if"/wiki/"notinhreforany(title.startswith(p.replace(":","%3A"))forpinSKIP_PREFIXES):continueurls.append(href)returnsorted(set(urls))defextract_section(base,page_url):"""return (title, section html) or None if the page has no crime heading."""full=base+page_urlhtml=requests.get(full,timeout=10).textsoup=BeautifulSoup(html,"html5lib")span=soup.find(id=HEADING_ID)ifnotspan:returnNoneheading=span.find_parent(re.compile(r"^h[1-6]$"))ifnotheading:returnNonelevel=int(heading.name[1])parts=[]forsibinheading.find_next_siblings():ifsib.nameandre.fullmatch(r"h[1-6]",sib.name)andint(sib.name[1])<=level:breakparts.append(sib)section=BeautifulSoup("<div></div>","html5lib").divassertsectionisnotNoneforpinparts:section.append(p)# absolute-ify links and images so they resolve against kiwix from anywherefortag,attrin(("a","href"),("img","src")):fortinsection.find_all(tag):ift.has_attr(attr):t[attr]=urljoin(full,str(t[attr]))# MediaWiki's JS collapses these on the wiki; without it they spill out inlinefortinsection.select(".mw-collapsed .mw-collapsible-content"):t.decompose()# drop the wiki's inline presentation so the page's own styles applyfortinsection.find_all(True):forattrin("style","bgcolor","border","cellspacing","cellpadding","width","align"):ift.has_attr(attr):delt[attr]title=soup.find("h1")name=title.get_text(strip=True).replace(" - Discworld MUD Wiki","")iftitleelsepage_urlreturnname,str(section),fullPAGE="""<!doctype html>
<meta charset="utf-8">
<title>crime and punishment, everywhere</title>
<style>
html {{ scroll-behavior: smooth; }}
body {{ max-width: 60em; margin: 2em auto; padding: 0 1em;
background: white; color: black; font-family: monospace; }}
h2 {{ border-bottom: 1px solid #ccc; padding-top: 1em; }}
h2 a {{ font-size: 0.6em; margin-left: 1em; }}
nav {{ position: fixed; top: 2em; left: 1.5em; font-size: 0.9em; }}
nav a {{ display: block; padding: 0.15em 0; text-decoration: none; }}
nav a:hover {{ text-decoration: underline; }}
@media (max-width: 60em) {{ nav {{ display: none; }} }}
table {{ border-collapse: collapse; margin: 1em 0; }}
td, th {{ border: 1px solid #999; padding: 0.4em 0.6em; text-align: left; }}
pre {{ overflow-x: auto; border: 1px solid #ccc; padding: 0.6em; }}
img {{ max-width: 100%; }}
</style>
<nav>{toc}</nav>
<h1>crime and punishment, everywhere</h1>
<p>a concatenation of every crime and punishment section in the wiki snapshot</p>
<p>generated with <a href="https://no.dungeon.red/crime-page/py">https://no.dungeon.red/crime-page/py</a></p>
{body}"""defslug(name):returnre.sub(r"[^a-z0-9]+","-",name.lower()).strip("-")defmain():ap=argparse.ArgumentParser()ap.add_argument("--base",default="http://127.0.0.1:8090")ap.add_argument("--link-base",default=None,help="prefix for links in the output (default: --base; '' for root-relative)")ap.add_argument("--book",default="dwwiki_2026-08")ap.add_argument("-o","--out",default="crime-and-punishment.html")args=ap.parse_args()chunks=[]toc=[]forurlinfind_pages(args.base,args.book):got=extract_section(args.base,url)ifnotgot:continuename,section,full=gotprint(f" {name}",file=sys.stderr)anchor=slug(name)toc.append(f'<a href="#{anchor}">{name}</a>')chunks.append(f'<h2 id="{anchor}">{name} <a href="{full}">full page</a></h2>\n{section}')out=PAGE.format(toc="\n".join(toc),body="\n".join(chunks))ifargs.link_baseisnotNone:out=out.replace(args.base+"/",args.link_base+"/")withopen(args.out,"w")asf:f.write(out)print(f"wrote {args.out} ({len(chunks)} sections)",file=sys.stderr)if__name__=="__main__":main()