wahyd4.github.com/scripts/extract_posts.py

#!/usr/bin/env python3
"""Extract posts from Hexo-generated static HTML back into markdown.

Source: wahyd4.github.com (GitHub Pages deploy artifact, Hexo 6.3.0 + NexT 8.25.0)
Target: Hexo source/_posts/*.md with front-matter (title/date/tags).
"""
import re
import glob
import html as html_mod
import os
import sys

from markdownify import markdownify as md

SRC = os.path.expanduser('~/code/blog/wahyd4.github.com')
OUT = os.path.expanduser('~/code/blog/hexo-blog/source/_posts')
os.makedirs(OUT, exist_ok=True)


def extract_code_blocks(body: str) -> tuple[str, list[tuple[str, str]]]:
    """Extract <figure class="highlight"> and <pre> blocks, replace with placeholders.
    Returns (body, [(placeholder, fenced_code_block), ...])"""
    blocks = []

    def repl_figure(m):
        lang = m.group(1) or ''
        inner = m.group(2)
        # get code column text
        cm = re.search(r'<td class="code">\s*<pre>(.*?)</pre>', inner, re.S)
        code_raw = cm.group(1) if cm else inner
        # lines are <span class="line">..</span> separated by <br>
        lines = re.findall(r'<span class="line">(.*?)</span>', code_raw, re.S)
        if not lines:
            lines = code_raw.split('<br>')
        code = ''.join(html_mod.unescape(l) for l in lines)
        code = re.sub(r'<[^>]+>', '', code)  # safety: strip any stray tags
        code = code.rstrip('\n')
        placeholder = f'%%CODEBLOCK{len(blocks)}%%'
        blocks.append((placeholder, f'```{lang}\n{code}\n```'))
        return placeholder

    def repl_brushtype_pre(m):
        lang = (m.group(1) or '').strip()
        if lang.startswith('brush:'):
            lang = lang.split(':', 1)[1].split(';')[0].strip()
        code = m.group(2)
        code = html_mod.unescape(code)
        code = code.replace('<br>', '\n')
        code = re.sub(r'<[^>]+>', '', code)
        code = code.rstrip('\n')
        placeholder = f'%%CODEBLOCK{len(blocks)}%%'
        blocks.append((placeholder, f'```{lang}\n{code}\n```'))
        return placeholder

    def repl_plain_pre(m):
        code = m.group(2) or m.group(1)
        code = html_mod.unescape(code)
        code = code.replace('<br>', '\n')
        code = re.sub(r'<[^>]+>', '', code)
        code = code.rstrip('\n')
        placeholder = f'%%CODEBLOCK{len(blocks)}%%'
        blocks.append((placeholder, f'```\n{code}\n```'))
        return placeholder

    # 1) hexo highlight figure
    body = re.sub(
        r'<figure class="highlight\s+([^"]*?)">(.*?)</figure>',
        repl_figure, body, flags=re.S)
    # 2) <pre class="brush: lang; ...">
    body = re.sub(
        r'<pre class="brush:([^;"]*)[^"]*"(?: title="[^"]*")?>(.*?)</pre>',
        repl_brushtype_pre, body, flags=re.S)
    # 3) <pre class="...">...</pre> (no code child)
    body = re.sub(
        r'<pre(?![^>]*>.*?</code>)([^>]*)>(.*?)</pre>',
        repl_plain_pre, body, flags=re.S)
    # 4) <pre><code>...</code></pre>
    body = re.sub(
        r'<pre><code[^>]*>(.*?)</code></pre>',
        repl_plain_pre, body, flags=re.S)
    return body, blocks


def restore_code_blocks(md_text: str, blocks: list[tuple[str, str]]) -> str:
    for placeholder, fenced in blocks:
        md_text = md_text.replace(placeholder, f'\n\n{fenced}\n\n')
    return md_text


def extract_post(path: str) -> dict | None:
    html = open(path, encoding='utf-8', errors='replace').read()

    # title from <h1 class="post-title">
    m = re.search(r'<h1 class="post-title[^>]*>(.*?)</h1>', html, re.S)
    if not m:
        return None
    title = re.sub(r'<[^>]+>', '', m.group(1)).strip()

    # date from <time ... datetime="...">
    m = re.search(r'datetime="(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}):\d{2}', html)
    date = None
    if m:
        date = f'{m.group(1)} {m.group(2)}:00'

    # tags
    tags = []
    m = re.search(r'<div class="post-tags">(.*?)</div>', html, re.S)
    if m:
        tags = re.findall(r'<a href="[^"]*"[^>]*>\s*#\s*(.*?)\s*</a>', m.group(1))
        tags = [t.strip() for t in tags if t.strip()]

    # body
    m = re.search(r'<div class="post-body[^>]*>(.*?)<footer class="post-footer">', html, re.S)
    if not m:
        return None
    body = m.group(1)

    body, blocks = extract_code_blocks(body)

    # markdownify
    md_text = md(
        body,
        heading_style='ATX',
        bullets='-',
        strong_em_symbol='*',
        strip=['headerlink'],
        code_language='',
    )

    md_text = restore_code_blocks(md_text, blocks)

    # clean up
    md_text = re.sub(r'\n{3,}', '\n\n', md_text)
    md_text = md_text.strip() + '\n'

    return {'title': title, 'date': date, 'tags': tags, 'body': md_text}


def slug_from_path(path: str) -> str:
    return os.path.basename(os.path.dirname(path))


def main():
    posts = sorted(glob.glob(os.path.join(SRC, '[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]/*/index.html')))
    print(f'found {len(posts)} posts')
    ok = fail = 0
    for p in posts:
        data = extract_post(p)
        if not data:
            print('FAIL:', p)
            fail += 1
            continue
        slug = slug_from_path(p)
        fm = [f'---']
        fm.append(f'title: "{data["title"].replace(chr(34), chr(39))}"')
        if data['date']:
            fm.append(f'date: {data["date"]}')
        if data['tags']:
            fm.append('tags: [' + ', '.join(data['tags']) + ']')
        fm.append('---')
        fm.append('')
        content = '\n'.join(fm) + '\n' + data['body']
        out = os.path.join(OUT, slug + '.md')
        with open(out, 'w', encoding='utf-8') as f:
            f.write(content)
        ok += 1
    print(f'OK: {ok}, FAIL: {fail}')
    # summary
    total_chars = 0
    for f in glob.glob(os.path.join(OUT, '*.md')):
        total_chars += os.path.getsize(f)
    print(f'total md size: {total_chars/1024:.0f} KB, files: {len(glob.glob(os.path.join(OUT, "*.md")))}')


if __name__ == '__main__':
    main()