Files
Bkk1234 4f74a509d5 intro-irre: แปลงตำราวิศวกรรมชลประทานเบื้องต้นเป็นเว็บเอกสาร 10 บท
- tools/extract.py: PDF 300 หน้า -> extracted/pages.json + crop ภาพ/ตาราง/สมการ 423 รูป
  (ถอด PUA วรรณยุกต์ไทย, รวมสระอำ, แยกหัวข้อที่ระดับ span)
- tools/build.py: generator หน้า docs แบบ static (สารบัญซ้าย/เนื้อหา/ในหน้านี้)
- index.html + chapters/ch01..10.html + css/docs.css + js/docs.js
  (ธีม K2D, ค้นหาเต็มเล่มผ่าน data/search.json, dark mode, drawer มือถือ)
- README.md: วิธี build และข้อจำกัดที่รู้อยู่
2026-11-19 01:34:49 +07:00

440 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Extract structured content from the source PDF of "วิศวกรรมชลประทานเบื้องต้น".
The book is typeset with legacy Thai fonts (AngsanaNew / EucrosiaUPC). Two of
its chapters reflow text so that a heading shares its line with the prose that
follows it, so headings are cut on *span* boundaries (bold run at line start),
not on lines.
Equations, rules-drawn tables and pictures cannot be linearised faithfully, so
each of those graphic bands is rendered as a JPEG crop and put back into the
reading order where it belongs.
Outputs
extracted/pages.json one record per page, items in reading order
assets/figures/*.jpg cropped figures / tables / equations
Usage
python tools/extract.py [--pdf PATH]
"""
from __future__ import annotations
import argparse
import json
import re
import statistics
import unicodedata
from pathlib import Path
import pymupdf as fitz
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_PDF = Path(r"C:\Users\BANK\Downloads\Documents\Intro-Irri-Eng.pdf")
FIG_DIR = ROOT / "assets" / "figures"
TOP_MARGIN = 62.0 # above this line = running header
BOTTOM_MARGIN = 808.0 # below this line = footer
CONTENT_X0, CONTENT_X1 = 72.0, 526.0
CONTENT_W = CONTENT_X1 - CONTENT_X0
MATH_CHARS = set("=×÷∴π√∑∫≥≤±²³·∝≠≈γλΔ∂")
HEADING_RE = re.compile(r"^(บทที่\s*\d+|\d+(?:\.\d+)*\.?)\s*\S")
CHAPTER_RE = re.compile(r"^บทที่\s*\d+")
CAPTION_RE = re.compile(r"^(รูปที่|ตารางที่|ภาพที่)\s*\d")
THAI_RE = re.compile(r"[\u0e00-\u0e7f]")
THAI_RUN = re.compile(r"[\u0e00-\u0e7f]{20,}")
PUA_MAP = {
"\uf70a": "\u0e48", # ่
"\uf70b": "\u0e49", # ้
"\uf706": "\u0e49", # ้ (bold subset of the same face)
"\uf714": "\u0e49", # ้
"\uf715": "\u0e53", # ๊
"\uf708": "\u0e4b", # ๋
"\uf712": "\u0e47", # ็
"\uf70e": "\u0e4c", # ์
"\uf701": "\u0e34", # ิ
"\uf702": "\u0e35", # ี
"\uf710": "\u0e31", # ั
"\uf0a7": "\u2022", # bullet
}
def norm(text: str) -> str:
"""Repair legacy-Thai artefacts (subset fonts use PUA code points)."""
text = unicodedata.normalize("NFC", text)
for src, dst in PUA_MAP.items():
text = text.replace(src, dst)
text = text.replace("\u0e4d\u0e32", "\u0e33").replace("\u200b", "")
text = re.sub(r"[ \t\u00a0]+", " ", text)
return text.strip()
# ------------------------------------------------------------------ primitives
def read_lines(page):
"""Text lines of the content area, tagged with their source block."""
lines = []
for bno, block in enumerate(page.get_text("dict")["blocks"]):
if block["type"] != 0:
continue
bb = block["bbox"]
if bb[3] < TOP_MARGIN or bb[1] > BOTTOM_MARGIN:
continue
for line in block["lines"]:
spans = []
for s in line["spans"]:
if not s["text"]:
continue
spans.append({
"text": s["text"],
"size": round(s["size"], 1),
"bold": bool(s["flags"] & 16) or "bold" in s["font"].lower(),
})
if not "".join(s["text"] for s in spans).strip():
continue
sizes = [s["size"] for s in spans]
lines.append({
"block": bno,
"spans": spans,
"text": norm("".join(s["text"] for s in spans)),
"size": max(sizes),
"min_size": min(sizes),
"bold": any(s["bold"] for s in spans),
"x0": round(line["bbox"][0], 1),
"y0": round(line["bbox"][1], 1),
"x1": round(line["bbox"][2], 1),
"y1": round(line["bbox"][3], 1),
})
lines.sort(key=lambda l: (round(l["y0"] / 4), l["x0"]))
return merge_fragments(lines)
def merge_fragments(lines):
"""LaTeX breaks a line around wide spaces — glue the pieces back."""
merged, cur = [], None
for l in lines:
if cur and cur["block"] == l["block"] and abs(l["y0"] - cur["y0"]) < 3 and l["x0"] - cur["x1"] < 40:
gap = " " if (cur["spans"][-1]["text"].endswith(" ") or l["spans"][0]["text"].startswith(" ")) else ""
joined = cur["text"] + gap + l["text"]
cur["spans"] = cur["spans"] + l["spans"]
cur["text"] = norm(joined)
cur["x1"] = max(cur["x1"], l["x1"])
cur["y1"] = max(cur["y1"], l["y1"])
cur["min_size"] = min(cur["min_size"], l["min_size"])
cur["bold"] = cur["bold"] or l["bold"]
else:
if cur:
merged.append(cur)
cur = dict(l)
if cur:
merged.append(cur)
return merged
def body_size(lines):
"""The text size that carries most of the characters on the page."""
counter = {}
for l in lines:
key = round(l["size"], 1)
counter[key] = counter.get(key, 0) + len(l["text"])
return max(counter.items(), key=lambda kv: kv[1])[0] if counter else 16.0
def split_bold_head(line):
"""Return (head_text, rest_text) where head is the leading bold run."""
head, i = [], 0
spans = line["spans"]
while i < len(spans) and (spans[i]["bold"] or not spans[i]["text"].strip()):
head.append(spans[i])
i += 1
if i == 0:
return "", line["text"]
head_text = norm("".join(s["text"] for s in head))
rest_text = norm("".join(s["text"] for s in spans[i:]))
return head_text, rest_text
def heading_of(line, body):
"""(level, heading_text, rest_text) when the line opens with a heading."""
head, rest = split_bold_head(line)
if not head:
# unfussy chapters: the heading is not bold, it is a short numbered line
m = HEADING_RE.match(line["text"])
if m:
num = m.group(1).rstrip(".")
parts = num.split(".")
if ("." in num and len(parts) >= 2
and parts[0].isdigit() and 1 <= int(parts[0]) <= 15
and parts[1].isdigit() and int(parts[1]) >= 1
and len(line["text"]) <= 45 and line["text"].count(" ") <= 3
and line["size"] >= body * 0.98):
rest_text = line["text"][m.end(1):].strip()
if len(rest_text) >= 12 and THAI_RE.search(rest_text):
return (3 if num.count(".") >= 2 else 2), line["text"], ""
return None
if len(head) > 140:
return None
if CAPTION_RE.match(head) or "=" in head:
return None
full_bold = not rest
if CHAPTER_RE.match(head) and (line["size"] >= body * 1.2 or full_bold):
return 1, head, rest
m = HEADING_RE.match(head)
if not m:
return None
num = m.group(1).rstrip(".")
if "." not in num:
return None
if not (full_bold or line["size"] >= body * 1.02):
return None
level = 3 if num.count(".") >= 2 else 2
return level, head, rest
# ------------------------------------------------------------------- graphics
def graphic_rects(page):
rects = []
for block in page.get_text("dict")["blocks"]:
if block["type"] != 1:
continue
r = fitz.Rect(block["bbox"])
if r.y1 >= TOP_MARGIN and r.y0 <= BOTTOM_MARGIN and r.width > 6 and r.height > 6:
rects.append(r)
for dr in page.get_drawings():
r = fitz.Rect(dr["rect"])
if r.y1 < TOP_MARGIN or r.y0 > BOTTOM_MARGIN:
continue
if r.width < 6 and r.height < 6:
continue
rects.append(r)
return rects
def merge_rects(rects, vgap=20.0, hgap=40.0):
items = [{"rect": fitz.Rect(r)} for r in rects]
changed = True
while changed:
changed = False
out = []
while items:
cur = items.pop(0)
again = True
while again:
again = False
for i, other in enumerate(items):
a, b = cur["rect"], other["rect"]
vy = not (b.y0 > a.y1 + vgap or b.y1 < a.y0 - vgap)
vx = not (b.x0 > a.x1 + hgap or b.x1 < a.x0 - hgap)
if vy and vx:
cur["rect"] = a | b
items.pop(i)
again = changed = True
break
out.append(cur)
items = out
return [it["rect"] for it in items]
def formula_block_rects(page):
"""Display equations: their own block, maths glyphs, no running prose."""
out = []
for block in page.get_text("dict")["blocks"]:
if block["type"] != 0:
continue
bb = block["bbox"]
if bb[3] < TOP_MARGIN or bb[1] > BOTTOM_MARGIN:
continue
text = norm("".join(s["text"] for l in block["lines"] for s in l["spans"]))
if not text:
continue
text_len = len(text)
size = min((s["size"] for l in block["lines"] for s in l["spans"]), default=16)
maths = sum(1 for ch in text if ch in MATH_CHARS)
prose = len(THAI_RUN.findall(text)) >= 1 or text_len > 170
if prose:
continue
if maths >= 1 or size < 13.0 or (bb[3] - bb[1] < 40 and text_len < 45):
out.append(fitz.Rect(bb))
return out
def absorb_into_bands(bands, lines, used, body):
"""Pull narrow text lines (table cells, equation operands, sub-labels) into bands."""
for band in bands:
grown = fitz.Rect(band.x0 - 10, band.y0 - 14, band.x1 + 10, band.y1 + 12)
for i, l in enumerate(lines):
if i in used:
continue
if (l["x1"] - l["x0"]) > CONTENT_W * 0.86:
continue # full justified prose line
if CAPTION_RE.match(l["text"]) or heading_of(l, body):
continue
cx, cy = (l["x0"] + l["x1"]) / 2, (l["y0"] + l["y1"]) / 2
if grown.contains(fitz.Point(cx, cy)):
used.add(i)
band |= fitz.Rect(l["x0"], l["y0"], l["x1"], l["y1"])
def render_band(page, rect, path):
zoom = min(2.4, max(1.6, 1100.0 / max(rect.width, 1)))
pix = page.get_pixmap(clip=fitz.Rect(rect), matrix=fitz.Matrix(zoom, zoom))
pix.save(str(path), jpg_quality=90)
return pix.width, pix.height
# --------------------------------------------------------------------- per page
def extract_page(page, pno):
lines = read_lines(page)
body = body_size(lines)
used = set()
rects = graphic_rects(page) + formula_block_rects(page)
bands = merge_rects(rects)
bands = [b for b in bands if b.height >= 10 and b.width >= 24]
absorb_into_bands(bands, lines, used, body)
margin = statistics.mode([round(l["x0"]) for l in lines]) if lines else CONTENT_X0
wide = [l["x1"] for l in lines if (l["x1"] - l["x0"]) > CONTENT_W * 0.5]
right = statistics.mode([round(v) for v in wide]) if wide else CONTENT_X1
items, para, para_y = [], [], []
para_block = [None]
prev_y1 = [0.0]
def flush():
if not para:
return
text = join_lines(para)
if text:
items.append({"type": "p", "text": text, "y": para_y[0]})
para.clear()
para_y.clear()
para_block[0] = None
for i, l in enumerate(lines):
if i in used:
continue
head = heading_of(l, body)
if head:
level, head_text, rest = head
flush()
items.append({"type": f"h{level}", "text": head_text, "y": l["y0"], "yb": l["y1"]})
if rest:
para.append(rest)
para_y.append(l["y0"])
para_block[0] = l["block"]
prev_y1[0] = l["y1"]
continue
if CAPTION_RE.match(l["text"]):
flush()
items.append({"type": "caption", "text": l["text"], "y": l["y0"], "yb": l["y1"]})
continue
if para and para_block[0] != l["block"]:
flush()
if para and (l["y0"] - prev_y1[0] > 9 or l["x0"] > margin + 12):
flush()
if para and re.match(r"^[•\-–]\s", l["text"]):
flush() # list item starts its own block
para.append(l["text"])
para_y.append(l["y0"])
para_block[0] = l["block"]
prev_y1[0] = l["y1"]
flush()
for b in bands:
items.append({"type": "figure", "rect": b, "y": b.y0})
items.sort(key=lambda it: (it["y"], 0 if it["type"] == "figure" else 1))
return finish_items(page, pno, items)
def finish_items(page, pno, items):
# merge a caption into the figure it describes
out = []
i = 0
while i < len(items):
it = items[i]
if it["type"] == "caption":
prev_fig = out[-1] if out and out[-1]["type"] == "figure" else None
next_fig = items[i + 1] if i + 1 < len(items) and items[i + 1]["type"] == "figure" else None
if prev_fig and it["y"] - prev_fig["rect"].y1 < 70:
prev_fig.setdefault("caption", it["text"])
i += 1
continue
if next_fig and next_fig["rect"].y0 - it["y"] < 70:
next_fig.setdefault("caption", it["text"])
i += 1
continue
out.append(it)
i += 1
final, fig_no = [], 0
for it in out:
if it["type"] == "figure":
fig_no += 1
name = f"p{pno + 1:03d}-{fig_no}.jpg"
w, h = render_band(page, it["rect"], FIG_DIR / name)
final.append({
"type": "figure",
"src": f"assets/figures/{name}",
"width": w,
"height": h,
"caption": it.get("caption", ""),
"page": pno + 1,
})
continue
if it["type"] == "h1" and final and final[-1]["type"] == "h1":
final[-1]["text"] = f'{final[-1]["text"]} {it["text"]}'.strip()
continue
final.append({"type": it["type"], "text": it.get("text", ""), "page": pno + 1})
return final
def join_lines(parts):
buf = ""
for t in parts:
t = t.strip()
if not t:
continue
if not buf:
buf = t
elif re.search(r"[A-Za-z0-9)\]]$", buf) and re.match(r"^[A-Za-z0-9(]", t):
buf = f"{buf} {t}"
else:
buf += t
return re.sub(r"\s+", " ", buf).strip()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--pdf", default=str(DEFAULT_PDF))
args = ap.parse_args()
if FIG_DIR.exists():
for f in FIG_DIR.glob("*"):
f.unlink()
FIG_DIR.mkdir(parents=True, exist_ok=True)
doc = fitz.open(args.pdf)
pages = []
for pno in range(doc.page_count):
pages.append({"page": pno + 1, "items": extract_page(doc[pno], pno)})
if (pno + 1) % 50 == 0:
print(f" … {pno + 1}/{doc.page_count}")
toc = [{"level": lvl, "title": norm(t), "page": pg} for lvl, t, pg in doc.get_toc()]
payload = {"source": Path(args.pdf).name, "page_count": doc.page_count, "toc": toc, "pages": pages}
out = ROOT / "extracted" / "pages.json"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8")
kinds = {}
for p in pages:
for it in p["items"]:
kinds[it["type"]] = kinds.get(it["type"], 0) + 1
print(f"pages={doc.page_count} items={kinds} -> {out}")
if __name__ == "__main__":
main()