- tools/extract.py: PDF 300 หน้า -> extracted/pages.json + crop ภาพ/ตาราง/สมการ 423 รูป (ถอด PUA วรรณยุกต์ไทย, รวมสระอำ, แยกหัวข้อที่ระดับ span) - tools/build.py: generator หน้า docs แบบ static (สารบัญซ้าย/เนื้อหา/ในหน้านี้) - index.html + chapters/ch01..10.html + css/docs.css + js/docs.js (ธีม K2D, ค้นหาเต็มเล่มผ่าน data/search.json, dark mode, drawer มือถือ) - README.md: วิธี build และข้อจำกัดที่รู้อยู่
440 lines
15 KiB
Python
440 lines
15 KiB
Python
"""Extract structured content from the source PDF of "วิศวกรรมชลประทานเบื้องต้น".
|
||
|
||
The book is typeset with legacy Thai fonts (AngsanaNew / EucrosiaUPC). Two of
|
||
its chapters reflow text so that a heading shares its line with the prose that
|
||
follows it, so headings are cut on *span* boundaries (bold run at line start),
|
||
not on lines.
|
||
|
||
Equations, rules-drawn tables and pictures cannot be linearised faithfully, so
|
||
each of those graphic bands is rendered as a JPEG crop and put back into the
|
||
reading order where it belongs.
|
||
|
||
Outputs
|
||
extracted/pages.json one record per page, items in reading order
|
||
assets/figures/*.jpg cropped figures / tables / equations
|
||
|
||
Usage
|
||
python tools/extract.py [--pdf PATH]
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import statistics
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
import pymupdf as fitz
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
DEFAULT_PDF = Path(r"C:\Users\BANK\Downloads\Documents\Intro-Irri-Eng.pdf")
|
||
FIG_DIR = ROOT / "assets" / "figures"
|
||
|
||
TOP_MARGIN = 62.0 # above this line = running header
|
||
BOTTOM_MARGIN = 808.0 # below this line = footer
|
||
CONTENT_X0, CONTENT_X1 = 72.0, 526.0
|
||
CONTENT_W = CONTENT_X1 - CONTENT_X0
|
||
|
||
MATH_CHARS = set("=×÷∴π√∑∫≥≤±²³·∝≠≈γλΔ∂")
|
||
HEADING_RE = re.compile(r"^(บทที่\s*\d+|\d+(?:\.\d+)*\.?)\s*\S")
|
||
CHAPTER_RE = re.compile(r"^บทที่\s*\d+")
|
||
CAPTION_RE = re.compile(r"^(รูปที่|ตารางที่|ภาพที่)\s*\d")
|
||
THAI_RE = re.compile(r"[\u0e00-\u0e7f]")
|
||
THAI_RUN = re.compile(r"[\u0e00-\u0e7f]{20,}")
|
||
|
||
PUA_MAP = {
|
||
"\uf70a": "\u0e48", # ่
|
||
"\uf70b": "\u0e49", # ้
|
||
"\uf706": "\u0e49", # ้ (bold subset of the same face)
|
||
"\uf714": "\u0e49", # ้
|
||
"\uf715": "\u0e53", # ๊
|
||
"\uf708": "\u0e4b", # ๋
|
||
"\uf712": "\u0e47", # ็
|
||
"\uf70e": "\u0e4c", # ์
|
||
"\uf701": "\u0e34", # ิ
|
||
"\uf702": "\u0e35", # ี
|
||
"\uf710": "\u0e31", # ั
|
||
"\uf0a7": "\u2022", # bullet
|
||
}
|
||
|
||
|
||
def norm(text: str) -> str:
|
||
"""Repair legacy-Thai artefacts (subset fonts use PUA code points)."""
|
||
text = unicodedata.normalize("NFC", text)
|
||
for src, dst in PUA_MAP.items():
|
||
text = text.replace(src, dst)
|
||
text = text.replace("\u0e4d\u0e32", "\u0e33").replace("\u200b", "")
|
||
text = re.sub(r"[ \t\u00a0]+", " ", text)
|
||
return text.strip()
|
||
|
||
|
||
# ------------------------------------------------------------------ primitives
|
||
def read_lines(page):
|
||
"""Text lines of the content area, tagged with their source block."""
|
||
lines = []
|
||
for bno, block in enumerate(page.get_text("dict")["blocks"]):
|
||
if block["type"] != 0:
|
||
continue
|
||
bb = block["bbox"]
|
||
if bb[3] < TOP_MARGIN or bb[1] > BOTTOM_MARGIN:
|
||
continue
|
||
for line in block["lines"]:
|
||
spans = []
|
||
for s in line["spans"]:
|
||
if not s["text"]:
|
||
continue
|
||
spans.append({
|
||
"text": s["text"],
|
||
"size": round(s["size"], 1),
|
||
"bold": bool(s["flags"] & 16) or "bold" in s["font"].lower(),
|
||
})
|
||
if not "".join(s["text"] for s in spans).strip():
|
||
continue
|
||
sizes = [s["size"] for s in spans]
|
||
lines.append({
|
||
"block": bno,
|
||
"spans": spans,
|
||
"text": norm("".join(s["text"] for s in spans)),
|
||
"size": max(sizes),
|
||
"min_size": min(sizes),
|
||
"bold": any(s["bold"] for s in spans),
|
||
"x0": round(line["bbox"][0], 1),
|
||
"y0": round(line["bbox"][1], 1),
|
||
"x1": round(line["bbox"][2], 1),
|
||
"y1": round(line["bbox"][3], 1),
|
||
})
|
||
lines.sort(key=lambda l: (round(l["y0"] / 4), l["x0"]))
|
||
return merge_fragments(lines)
|
||
|
||
|
||
def merge_fragments(lines):
|
||
"""LaTeX breaks a line around wide spaces — glue the pieces back."""
|
||
merged, cur = [], None
|
||
for l in lines:
|
||
if cur and cur["block"] == l["block"] and abs(l["y0"] - cur["y0"]) < 3 and l["x0"] - cur["x1"] < 40:
|
||
gap = " " if (cur["spans"][-1]["text"].endswith(" ") or l["spans"][0]["text"].startswith(" ")) else ""
|
||
joined = cur["text"] + gap + l["text"]
|
||
cur["spans"] = cur["spans"] + l["spans"]
|
||
cur["text"] = norm(joined)
|
||
cur["x1"] = max(cur["x1"], l["x1"])
|
||
cur["y1"] = max(cur["y1"], l["y1"])
|
||
cur["min_size"] = min(cur["min_size"], l["min_size"])
|
||
cur["bold"] = cur["bold"] or l["bold"]
|
||
else:
|
||
if cur:
|
||
merged.append(cur)
|
||
cur = dict(l)
|
||
if cur:
|
||
merged.append(cur)
|
||
return merged
|
||
|
||
|
||
def body_size(lines):
|
||
"""The text size that carries most of the characters on the page."""
|
||
counter = {}
|
||
for l in lines:
|
||
key = round(l["size"], 1)
|
||
counter[key] = counter.get(key, 0) + len(l["text"])
|
||
return max(counter.items(), key=lambda kv: kv[1])[0] if counter else 16.0
|
||
|
||
|
||
def split_bold_head(line):
|
||
"""Return (head_text, rest_text) where head is the leading bold run."""
|
||
head, i = [], 0
|
||
spans = line["spans"]
|
||
while i < len(spans) and (spans[i]["bold"] or not spans[i]["text"].strip()):
|
||
head.append(spans[i])
|
||
i += 1
|
||
if i == 0:
|
||
return "", line["text"]
|
||
head_text = norm("".join(s["text"] for s in head))
|
||
rest_text = norm("".join(s["text"] for s in spans[i:]))
|
||
return head_text, rest_text
|
||
|
||
|
||
def heading_of(line, body):
|
||
"""(level, heading_text, rest_text) when the line opens with a heading."""
|
||
head, rest = split_bold_head(line)
|
||
if not head:
|
||
# unfussy chapters: the heading is not bold, it is a short numbered line
|
||
m = HEADING_RE.match(line["text"])
|
||
if m:
|
||
num = m.group(1).rstrip(".")
|
||
parts = num.split(".")
|
||
if ("." in num and len(parts) >= 2
|
||
and parts[0].isdigit() and 1 <= int(parts[0]) <= 15
|
||
and parts[1].isdigit() and int(parts[1]) >= 1
|
||
and len(line["text"]) <= 45 and line["text"].count(" ") <= 3
|
||
and line["size"] >= body * 0.98):
|
||
rest_text = line["text"][m.end(1):].strip()
|
||
if len(rest_text) >= 12 and THAI_RE.search(rest_text):
|
||
return (3 if num.count(".") >= 2 else 2), line["text"], ""
|
||
return None
|
||
if len(head) > 140:
|
||
return None
|
||
if CAPTION_RE.match(head) or "=" in head:
|
||
return None
|
||
full_bold = not rest
|
||
if CHAPTER_RE.match(head) and (line["size"] >= body * 1.2 or full_bold):
|
||
return 1, head, rest
|
||
m = HEADING_RE.match(head)
|
||
if not m:
|
||
return None
|
||
num = m.group(1).rstrip(".")
|
||
if "." not in num:
|
||
return None
|
||
if not (full_bold or line["size"] >= body * 1.02):
|
||
return None
|
||
level = 3 if num.count(".") >= 2 else 2
|
||
return level, head, rest
|
||
|
||
|
||
# ------------------------------------------------------------------- graphics
|
||
def graphic_rects(page):
|
||
rects = []
|
||
for block in page.get_text("dict")["blocks"]:
|
||
if block["type"] != 1:
|
||
continue
|
||
r = fitz.Rect(block["bbox"])
|
||
if r.y1 >= TOP_MARGIN and r.y0 <= BOTTOM_MARGIN and r.width > 6 and r.height > 6:
|
||
rects.append(r)
|
||
for dr in page.get_drawings():
|
||
r = fitz.Rect(dr["rect"])
|
||
if r.y1 < TOP_MARGIN or r.y0 > BOTTOM_MARGIN:
|
||
continue
|
||
if r.width < 6 and r.height < 6:
|
||
continue
|
||
rects.append(r)
|
||
return rects
|
||
|
||
|
||
def merge_rects(rects, vgap=20.0, hgap=40.0):
|
||
items = [{"rect": fitz.Rect(r)} for r in rects]
|
||
changed = True
|
||
while changed:
|
||
changed = False
|
||
out = []
|
||
while items:
|
||
cur = items.pop(0)
|
||
again = True
|
||
while again:
|
||
again = False
|
||
for i, other in enumerate(items):
|
||
a, b = cur["rect"], other["rect"]
|
||
vy = not (b.y0 > a.y1 + vgap or b.y1 < a.y0 - vgap)
|
||
vx = not (b.x0 > a.x1 + hgap or b.x1 < a.x0 - hgap)
|
||
if vy and vx:
|
||
cur["rect"] = a | b
|
||
items.pop(i)
|
||
again = changed = True
|
||
break
|
||
out.append(cur)
|
||
items = out
|
||
return [it["rect"] for it in items]
|
||
|
||
|
||
def formula_block_rects(page):
|
||
"""Display equations: their own block, maths glyphs, no running prose."""
|
||
out = []
|
||
for block in page.get_text("dict")["blocks"]:
|
||
if block["type"] != 0:
|
||
continue
|
||
bb = block["bbox"]
|
||
if bb[3] < TOP_MARGIN or bb[1] > BOTTOM_MARGIN:
|
||
continue
|
||
text = norm("".join(s["text"] for l in block["lines"] for s in l["spans"]))
|
||
if not text:
|
||
continue
|
||
text_len = len(text)
|
||
size = min((s["size"] for l in block["lines"] for s in l["spans"]), default=16)
|
||
maths = sum(1 for ch in text if ch in MATH_CHARS)
|
||
prose = len(THAI_RUN.findall(text)) >= 1 or text_len > 170
|
||
if prose:
|
||
continue
|
||
if maths >= 1 or size < 13.0 or (bb[3] - bb[1] < 40 and text_len < 45):
|
||
out.append(fitz.Rect(bb))
|
||
return out
|
||
|
||
|
||
def absorb_into_bands(bands, lines, used, body):
|
||
"""Pull narrow text lines (table cells, equation operands, sub-labels) into bands."""
|
||
for band in bands:
|
||
grown = fitz.Rect(band.x0 - 10, band.y0 - 14, band.x1 + 10, band.y1 + 12)
|
||
for i, l in enumerate(lines):
|
||
if i in used:
|
||
continue
|
||
if (l["x1"] - l["x0"]) > CONTENT_W * 0.86:
|
||
continue # full justified prose line
|
||
if CAPTION_RE.match(l["text"]) or heading_of(l, body):
|
||
continue
|
||
cx, cy = (l["x0"] + l["x1"]) / 2, (l["y0"] + l["y1"]) / 2
|
||
if grown.contains(fitz.Point(cx, cy)):
|
||
used.add(i)
|
||
band |= fitz.Rect(l["x0"], l["y0"], l["x1"], l["y1"])
|
||
|
||
|
||
def render_band(page, rect, path):
|
||
zoom = min(2.4, max(1.6, 1100.0 / max(rect.width, 1)))
|
||
pix = page.get_pixmap(clip=fitz.Rect(rect), matrix=fitz.Matrix(zoom, zoom))
|
||
pix.save(str(path), jpg_quality=90)
|
||
return pix.width, pix.height
|
||
|
||
|
||
# --------------------------------------------------------------------- per page
|
||
def extract_page(page, pno):
|
||
lines = read_lines(page)
|
||
body = body_size(lines)
|
||
used = set()
|
||
|
||
rects = graphic_rects(page) + formula_block_rects(page)
|
||
bands = merge_rects(rects)
|
||
bands = [b for b in bands if b.height >= 10 and b.width >= 24]
|
||
absorb_into_bands(bands, lines, used, body)
|
||
|
||
margin = statistics.mode([round(l["x0"]) for l in lines]) if lines else CONTENT_X0
|
||
wide = [l["x1"] for l in lines if (l["x1"] - l["x0"]) > CONTENT_W * 0.5]
|
||
right = statistics.mode([round(v) for v in wide]) if wide else CONTENT_X1
|
||
|
||
items, para, para_y = [], [], []
|
||
para_block = [None]
|
||
prev_y1 = [0.0]
|
||
|
||
def flush():
|
||
if not para:
|
||
return
|
||
text = join_lines(para)
|
||
if text:
|
||
items.append({"type": "p", "text": text, "y": para_y[0]})
|
||
para.clear()
|
||
para_y.clear()
|
||
para_block[0] = None
|
||
|
||
for i, l in enumerate(lines):
|
||
if i in used:
|
||
continue
|
||
head = heading_of(l, body)
|
||
if head:
|
||
level, head_text, rest = head
|
||
flush()
|
||
items.append({"type": f"h{level}", "text": head_text, "y": l["y0"], "yb": l["y1"]})
|
||
if rest:
|
||
para.append(rest)
|
||
para_y.append(l["y0"])
|
||
para_block[0] = l["block"]
|
||
prev_y1[0] = l["y1"]
|
||
continue
|
||
if CAPTION_RE.match(l["text"]):
|
||
flush()
|
||
items.append({"type": "caption", "text": l["text"], "y": l["y0"], "yb": l["y1"]})
|
||
continue
|
||
if para and para_block[0] != l["block"]:
|
||
flush()
|
||
if para and (l["y0"] - prev_y1[0] > 9 or l["x0"] > margin + 12):
|
||
flush()
|
||
if para and re.match(r"^[•\-–]\s", l["text"]):
|
||
flush() # list item starts its own block
|
||
para.append(l["text"])
|
||
para_y.append(l["y0"])
|
||
para_block[0] = l["block"]
|
||
prev_y1[0] = l["y1"]
|
||
flush()
|
||
|
||
for b in bands:
|
||
items.append({"type": "figure", "rect": b, "y": b.y0})
|
||
|
||
items.sort(key=lambda it: (it["y"], 0 if it["type"] == "figure" else 1))
|
||
return finish_items(page, pno, items)
|
||
|
||
|
||
def finish_items(page, pno, items):
|
||
# merge a caption into the figure it describes
|
||
out = []
|
||
i = 0
|
||
while i < len(items):
|
||
it = items[i]
|
||
if it["type"] == "caption":
|
||
prev_fig = out[-1] if out and out[-1]["type"] == "figure" else None
|
||
next_fig = items[i + 1] if i + 1 < len(items) and items[i + 1]["type"] == "figure" else None
|
||
if prev_fig and it["y"] - prev_fig["rect"].y1 < 70:
|
||
prev_fig.setdefault("caption", it["text"])
|
||
i += 1
|
||
continue
|
||
if next_fig and next_fig["rect"].y0 - it["y"] < 70:
|
||
next_fig.setdefault("caption", it["text"])
|
||
i += 1
|
||
continue
|
||
out.append(it)
|
||
i += 1
|
||
|
||
final, fig_no = [], 0
|
||
for it in out:
|
||
if it["type"] == "figure":
|
||
fig_no += 1
|
||
name = f"p{pno + 1:03d}-{fig_no}.jpg"
|
||
w, h = render_band(page, it["rect"], FIG_DIR / name)
|
||
final.append({
|
||
"type": "figure",
|
||
"src": f"assets/figures/{name}",
|
||
"width": w,
|
||
"height": h,
|
||
"caption": it.get("caption", ""),
|
||
"page": pno + 1,
|
||
})
|
||
continue
|
||
if it["type"] == "h1" and final and final[-1]["type"] == "h1":
|
||
final[-1]["text"] = f'{final[-1]["text"]} {it["text"]}'.strip()
|
||
continue
|
||
final.append({"type": it["type"], "text": it.get("text", ""), "page": pno + 1})
|
||
return final
|
||
|
||
|
||
def join_lines(parts):
|
||
buf = ""
|
||
for t in parts:
|
||
t = t.strip()
|
||
if not t:
|
||
continue
|
||
if not buf:
|
||
buf = t
|
||
elif re.search(r"[A-Za-z0-9)\]]$", buf) and re.match(r"^[A-Za-z0-9(]", t):
|
||
buf = f"{buf} {t}"
|
||
else:
|
||
buf += t
|
||
return re.sub(r"\s+", " ", buf).strip()
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--pdf", default=str(DEFAULT_PDF))
|
||
args = ap.parse_args()
|
||
|
||
if FIG_DIR.exists():
|
||
for f in FIG_DIR.glob("*"):
|
||
f.unlink()
|
||
FIG_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
doc = fitz.open(args.pdf)
|
||
pages = []
|
||
for pno in range(doc.page_count):
|
||
pages.append({"page": pno + 1, "items": extract_page(doc[pno], pno)})
|
||
if (pno + 1) % 50 == 0:
|
||
print(f" … {pno + 1}/{doc.page_count}")
|
||
|
||
toc = [{"level": lvl, "title": norm(t), "page": pg} for lvl, t, pg in doc.get_toc()]
|
||
payload = {"source": Path(args.pdf).name, "page_count": doc.page_count, "toc": toc, "pages": pages}
|
||
out = ROOT / "extracted" / "pages.json"
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
|
||
kinds = {}
|
||
for p in pages:
|
||
for it in p["items"]:
|
||
kinds[it["type"]] = kinds.get(it["type"], 0) + 1
|
||
print(f"pages={doc.page_count} items={kinds} -> {out}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|