"""Extract structured content from the source PDF of "วิศวกรรมชลประทานเบื้องต้น". The book is typeset with legacy Thai fonts (AngsanaNew / EucrosiaUPC). Two of its chapters reflow text so that a heading shares its line with the prose that follows it, so headings are cut on *span* boundaries (bold run at line start), not on lines. Equations, rules-drawn tables and pictures cannot be linearised faithfully, so each of those graphic bands is rendered as a JPEG crop and put back into the reading order where it belongs. Outputs extracted/pages.json one record per page, items in reading order assets/figures/*.jpg cropped figures / tables / equations Usage python tools/extract.py [--pdf PATH] """ from __future__ import annotations import argparse import json import re import statistics import unicodedata from pathlib import Path import pymupdf as fitz ROOT = Path(__file__).resolve().parent.parent DEFAULT_PDF = Path(r"C:\Users\BANK\Downloads\Documents\Intro-Irri-Eng.pdf") FIG_DIR = ROOT / "assets" / "figures" TOP_MARGIN = 62.0 # above this line = running header BOTTOM_MARGIN = 808.0 # below this line = footer CONTENT_X0, CONTENT_X1 = 72.0, 526.0 CONTENT_W = CONTENT_X1 - CONTENT_X0 MATH_CHARS = set("=×÷∴π√∑∫≥≤±²³·∝≠≈γλΔ∂") HEADING_RE = re.compile(r"^(บทที่\s*\d+|\d+(?:\.\d+)*\.?)\s*\S") CHAPTER_RE = re.compile(r"^บทที่\s*\d+") CAPTION_RE = re.compile(r"^(รูปที่|ตารางที่|ภาพที่)\s*\d") THAI_RE = re.compile(r"[\u0e00-\u0e7f]") THAI_RUN = re.compile(r"[\u0e00-\u0e7f]{20,}") PUA_MAP = { "\uf70a": "\u0e48", # ่ "\uf70b": "\u0e49", # ้ "\uf706": "\u0e49", # ้ (bold subset of the same face) "\uf714": "\u0e49", # ้ "\uf715": "\u0e53", # ๊ "\uf708": "\u0e4b", # ๋ "\uf712": "\u0e47", # ็ "\uf70e": "\u0e4c", # ์ "\uf701": "\u0e34", # ิ "\uf702": "\u0e35", # ี "\uf710": "\u0e31", # ั "\uf0a7": "\u2022", # bullet } def norm(text: str) -> str: """Repair legacy-Thai artefacts (subset fonts use PUA code points).""" text = unicodedata.normalize("NFC", text) for src, dst in PUA_MAP.items(): text = text.replace(src, dst) text = text.replace("\u0e4d\u0e32", "\u0e33").replace("\u200b", "") text = re.sub(r"[ \t\u00a0]+", " ", text) return text.strip() # ------------------------------------------------------------------ primitives def read_lines(page): """Text lines of the content area, tagged with their source block.""" lines = [] for bno, block in enumerate(page.get_text("dict")["blocks"]): if block["type"] != 0: continue bb = block["bbox"] if bb[3] < TOP_MARGIN or bb[1] > BOTTOM_MARGIN: continue for line in block["lines"]: spans = [] for s in line["spans"]: if not s["text"]: continue spans.append({ "text": s["text"], "size": round(s["size"], 1), "bold": bool(s["flags"] & 16) or "bold" in s["font"].lower(), }) if not "".join(s["text"] for s in spans).strip(): continue sizes = [s["size"] for s in spans] lines.append({ "block": bno, "spans": spans, "text": norm("".join(s["text"] for s in spans)), "size": max(sizes), "min_size": min(sizes), "bold": any(s["bold"] for s in spans), "x0": round(line["bbox"][0], 1), "y0": round(line["bbox"][1], 1), "x1": round(line["bbox"][2], 1), "y1": round(line["bbox"][3], 1), }) lines.sort(key=lambda l: (round(l["y0"] / 4), l["x0"])) return merge_fragments(lines) def merge_fragments(lines): """LaTeX breaks a line around wide spaces — glue the pieces back.""" merged, cur = [], None for l in lines: if cur and cur["block"] == l["block"] and abs(l["y0"] - cur["y0"]) < 3 and l["x0"] - cur["x1"] < 40: gap = " " if (cur["spans"][-1]["text"].endswith(" ") or l["spans"][0]["text"].startswith(" ")) else "" joined = cur["text"] + gap + l["text"] cur["spans"] = cur["spans"] + l["spans"] cur["text"] = norm(joined) cur["x1"] = max(cur["x1"], l["x1"]) cur["y1"] = max(cur["y1"], l["y1"]) cur["min_size"] = min(cur["min_size"], l["min_size"]) cur["bold"] = cur["bold"] or l["bold"] else: if cur: merged.append(cur) cur = dict(l) if cur: merged.append(cur) return merged def body_size(lines): """The text size that carries most of the characters on the page.""" counter = {} for l in lines: key = round(l["size"], 1) counter[key] = counter.get(key, 0) + len(l["text"]) return max(counter.items(), key=lambda kv: kv[1])[0] if counter else 16.0 def split_bold_head(line): """Return (head_text, rest_text) where head is the leading bold run.""" head, i = [], 0 spans = line["spans"] while i < len(spans) and (spans[i]["bold"] or not spans[i]["text"].strip()): head.append(spans[i]) i += 1 if i == 0: return "", line["text"] head_text = norm("".join(s["text"] for s in head)) rest_text = norm("".join(s["text"] for s in spans[i:])) return head_text, rest_text def heading_of(line, body): """(level, heading_text, rest_text) when the line opens with a heading.""" head, rest = split_bold_head(line) if not head: # unfussy chapters: the heading is not bold, it is a short numbered line m = HEADING_RE.match(line["text"]) if m: num = m.group(1).rstrip(".") parts = num.split(".") if ("." in num and len(parts) >= 2 and parts[0].isdigit() and 1 <= int(parts[0]) <= 15 and parts[1].isdigit() and int(parts[1]) >= 1 and len(line["text"]) <= 45 and line["text"].count(" ") <= 3 and line["size"] >= body * 0.98): rest_text = line["text"][m.end(1):].strip() if len(rest_text) >= 12 and THAI_RE.search(rest_text): return (3 if num.count(".") >= 2 else 2), line["text"], "" return None if len(head) > 140: return None if CAPTION_RE.match(head) or "=" in head: return None full_bold = not rest if CHAPTER_RE.match(head) and (line["size"] >= body * 1.2 or full_bold): return 1, head, rest m = HEADING_RE.match(head) if not m: return None num = m.group(1).rstrip(".") if "." not in num: return None if not (full_bold or line["size"] >= body * 1.02): return None level = 3 if num.count(".") >= 2 else 2 return level, head, rest # ------------------------------------------------------------------- graphics def graphic_rects(page): rects = [] for block in page.get_text("dict")["blocks"]: if block["type"] != 1: continue r = fitz.Rect(block["bbox"]) if r.y1 >= TOP_MARGIN and r.y0 <= BOTTOM_MARGIN and r.width > 6 and r.height > 6: rects.append(r) for dr in page.get_drawings(): r = fitz.Rect(dr["rect"]) if r.y1 < TOP_MARGIN or r.y0 > BOTTOM_MARGIN: continue if r.width < 6 and r.height < 6: continue rects.append(r) return rects def merge_rects(rects, vgap=20.0, hgap=40.0): items = [{"rect": fitz.Rect(r)} for r in rects] changed = True while changed: changed = False out = [] while items: cur = items.pop(0) again = True while again: again = False for i, other in enumerate(items): a, b = cur["rect"], other["rect"] vy = not (b.y0 > a.y1 + vgap or b.y1 < a.y0 - vgap) vx = not (b.x0 > a.x1 + hgap or b.x1 < a.x0 - hgap) if vy and vx: cur["rect"] = a | b items.pop(i) again = changed = True break out.append(cur) items = out return [it["rect"] for it in items] def formula_block_rects(page): """Display equations: their own block, maths glyphs, no running prose.""" out = [] for block in page.get_text("dict")["blocks"]: if block["type"] != 0: continue bb = block["bbox"] if bb[3] < TOP_MARGIN or bb[1] > BOTTOM_MARGIN: continue text = norm("".join(s["text"] for l in block["lines"] for s in l["spans"])) if not text: continue text_len = len(text) size = min((s["size"] for l in block["lines"] for s in l["spans"]), default=16) maths = sum(1 for ch in text if ch in MATH_CHARS) prose = len(THAI_RUN.findall(text)) >= 1 or text_len > 170 if prose: continue if maths >= 1 or size < 13.0 or (bb[3] - bb[1] < 40 and text_len < 45): out.append(fitz.Rect(bb)) return out def absorb_into_bands(bands, lines, used, body): """Pull narrow text lines (table cells, equation operands, sub-labels) into bands.""" for band in bands: grown = fitz.Rect(band.x0 - 10, band.y0 - 14, band.x1 + 10, band.y1 + 12) for i, l in enumerate(lines): if i in used: continue if (l["x1"] - l["x0"]) > CONTENT_W * 0.86: continue # full justified prose line if CAPTION_RE.match(l["text"]) or heading_of(l, body): continue cx, cy = (l["x0"] + l["x1"]) / 2, (l["y0"] + l["y1"]) / 2 if grown.contains(fitz.Point(cx, cy)): used.add(i) band |= fitz.Rect(l["x0"], l["y0"], l["x1"], l["y1"]) def render_band(page, rect, path): zoom = min(2.4, max(1.6, 1100.0 / max(rect.width, 1))) pix = page.get_pixmap(clip=fitz.Rect(rect), matrix=fitz.Matrix(zoom, zoom)) pix.save(str(path), jpg_quality=90) return pix.width, pix.height # --------------------------------------------------------------------- per page def extract_page(page, pno): lines = read_lines(page) body = body_size(lines) used = set() rects = graphic_rects(page) + formula_block_rects(page) bands = merge_rects(rects) bands = [b for b in bands if b.height >= 10 and b.width >= 24] absorb_into_bands(bands, lines, used, body) margin = statistics.mode([round(l["x0"]) for l in lines]) if lines else CONTENT_X0 wide = [l["x1"] for l in lines if (l["x1"] - l["x0"]) > CONTENT_W * 0.5] right = statistics.mode([round(v) for v in wide]) if wide else CONTENT_X1 items, para, para_y = [], [], [] para_block = [None] prev_y1 = [0.0] def flush(): if not para: return text = join_lines(para) if text: items.append({"type": "p", "text": text, "y": para_y[0]}) para.clear() para_y.clear() para_block[0] = None for i, l in enumerate(lines): if i in used: continue head = heading_of(l, body) if head: level, head_text, rest = head flush() items.append({"type": f"h{level}", "text": head_text, "y": l["y0"], "yb": l["y1"]}) if rest: para.append(rest) para_y.append(l["y0"]) para_block[0] = l["block"] prev_y1[0] = l["y1"] continue if CAPTION_RE.match(l["text"]): flush() items.append({"type": "caption", "text": l["text"], "y": l["y0"], "yb": l["y1"]}) continue if para and para_block[0] != l["block"]: flush() if para and (l["y0"] - prev_y1[0] > 9 or l["x0"] > margin + 12): flush() if para and re.match(r"^[•\-–]\s", l["text"]): flush() # list item starts its own block para.append(l["text"]) para_y.append(l["y0"]) para_block[0] = l["block"] prev_y1[0] = l["y1"] flush() for b in bands: items.append({"type": "figure", "rect": b, "y": b.y0}) items.sort(key=lambda it: (it["y"], 0 if it["type"] == "figure" else 1)) return finish_items(page, pno, items) def finish_items(page, pno, items): # merge a caption into the figure it describes out = [] i = 0 while i < len(items): it = items[i] if it["type"] == "caption": prev_fig = out[-1] if out and out[-1]["type"] == "figure" else None next_fig = items[i + 1] if i + 1 < len(items) and items[i + 1]["type"] == "figure" else None if prev_fig and it["y"] - prev_fig["rect"].y1 < 70: prev_fig.setdefault("caption", it["text"]) i += 1 continue if next_fig and next_fig["rect"].y0 - it["y"] < 70: next_fig.setdefault("caption", it["text"]) i += 1 continue out.append(it) i += 1 final, fig_no = [], 0 for it in out: if it["type"] == "figure": fig_no += 1 name = f"p{pno + 1:03d}-{fig_no}.jpg" w, h = render_band(page, it["rect"], FIG_DIR / name) final.append({ "type": "figure", "src": f"assets/figures/{name}", "width": w, "height": h, "caption": it.get("caption", ""), "page": pno + 1, }) continue if it["type"] == "h1" and final and final[-1]["type"] == "h1": final[-1]["text"] = f'{final[-1]["text"]} {it["text"]}'.strip() continue final.append({"type": it["type"], "text": it.get("text", ""), "page": pno + 1}) return final def join_lines(parts): buf = "" for t in parts: t = t.strip() if not t: continue if not buf: buf = t elif re.search(r"[A-Za-z0-9)\]]$", buf) and re.match(r"^[A-Za-z0-9(]", t): buf = f"{buf} {t}" else: buf += t return re.sub(r"\s+", " ", buf).strip() def main(): ap = argparse.ArgumentParser() ap.add_argument("--pdf", default=str(DEFAULT_PDF)) args = ap.parse_args() if FIG_DIR.exists(): for f in FIG_DIR.glob("*"): f.unlink() FIG_DIR.mkdir(parents=True, exist_ok=True) doc = fitz.open(args.pdf) pages = [] for pno in range(doc.page_count): pages.append({"page": pno + 1, "items": extract_page(doc[pno], pno)}) if (pno + 1) % 50 == 0: print(f" … {pno + 1}/{doc.page_count}") toc = [{"level": lvl, "title": norm(t), "page": pg} for lvl, t, pg in doc.get_toc()] payload = {"source": Path(args.pdf).name, "page_count": doc.page_count, "toc": toc, "pages": pages} out = ROOT / "extracted" / "pages.json" out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8") kinds = {} for p in pages: for it in p["items"]: kinds[it["type"]] = kinds.get(it["type"], 0) + 1 print(f"pages={doc.page_count} items={kinds} -> {out}") if __name__ == "__main__": main()