#!/usr/bin/env python3
"""Reproducible quantitative analysis for the uploaded Zhang Yiming Weibo PDF.

Usage:
    python scripts/analyze.py /path/to/source.pdf

Outputs:
    data/analysis.json
    data/manifest.json
    data/record_metadata.csv
    data/tag_validation_sample.csv
    data/timeline/index.json
    data/timeline/YYYY.json

Core metrics use deterministic parsing/rules. Topic labels are transparent keyword rules
and are generated from the author's own visible text, not from quoted/forwarded bodies.
"""
from __future__ import annotations

import csv
import hashlib
import json
import math
import random
import re
import statistics
import sys
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path

from pypdf import PdfReader

ROOT = Path(__file__).resolve().parents[1]
OUT_DIR = ROOT / "data"
OUT_DIR.mkdir(exist_ok=True)

DATASET_VERSION = "1.1.0"
PARSER_VERSION = "1.2.0"
TAG_RULES_VERSION = "1.1.0"
COVER_CLAIM = 2286

SOURCES = sorted([
    '手机微博触屏版','微博浏览器插件','搜狗高速浏览器','Android 客户端','iPhone 客户端','iPad 客户端',
    'iPhone 6 Plus','iPhone 6s','iPhone 6','微博 weibo.com','微博手机版','专业版微博','微博搜索','今日头条',
    '房产资讯','豆瓣电影','豆瓣web','分享按钮','微公益','秒拍客户端','Weico.iPhone','WeicoPro','皮皮时光机',
    'iPhone客户端','Android客户端'
], key=len, reverse=True)

TIMESTAMP_RE = re.compile(
    r'(?m)^(?P<date>20\d{2}-\d{1,2}-\d{1,2})\s+'
    r'(?P<time>\d{1,2}:\d{2})\s+来自\s+(?P<tail>.*?)\s*$'
)

ADMIN_LINE_RE = re.compile(
    r'(?:抱歉[，,]?.*?(?:删除|不可见|查看权限)|'
    r'该微博因被多人投诉.*?(?:管理中心)?|'
    r'查看帮助[:：]?\s*O?\s*网页链接|O\s*网页链接)',
    re.S,
)

WEEKDAYS = ['周一','周二','周三','周四','周五','周六','周日']

TOPIC_RULES = {
    '产品 / 用户': ['产品','用户','体验','设计','功能','ui','交互','客户端','需求','反馈'],
    '技术 / 工程': ['技术','工程师','编程','代码','程序员','算法','系统','服务器','django','数据','数据库','开发','bug'],
    '组织 / 人才': ['团队','招聘','人才','员工','管理','管理者','面试','同事','候选人','组织','文化','领导','hr'],
    '创业 / 商业': ['创业','商业','融资','估值','投资','上市','收入','利润','市场','竞争','商业模式'],
    '学习 / 思考': ['学习','思考','理性','知识','读书','教科书','词典','定义','逻辑','注意力','延迟满足','认知','总结'],
    '生活 / 日常': ['电影','音乐','吃饭','睡觉','老婆','女朋友','父母','火车','地铁','天气','周末','旅行','回家','朋友'],
    '行业 / 公司': ['互联网','百度','腾讯','新浪','小米','facebook','google','苹果','微软','微博','微信','阿里','搜狐','网易'],
}


def sha256_file(path: Path) -> str:
    h = hashlib.sha256()
    with path.open('rb') as f:
        for chunk in iter(lambda: f.read(1024 * 1024), b''):
            h.update(chunk)
    return h.hexdigest()


def topic_tags(content: str) -> list[str]:
    low = (content or '').lower()
    return [label for label, words in TOPIC_RULES.items() if any(w.lower() in low for w in words)]


def structural_tags(record: dict) -> list[str]:
    tags = []
    tags.append('转发 / 分享' if record['share'] else '非显式分享')
    if record['share'] and record.get('author_text'):
        tags.append('有作者附言')
    if record.get('quoted_text'):
        tags.append('含转发正文')
    if record['unavailable_notice']:
        tags.append('删除 / 不可见')
    if record['question']:
        tags.append('含问号')
    if record['first_person']:
        tags.append('第一人称')
    if record['has_english']:
        tags.append('含英文')
    if record['has_number']:
        tags.append('含数字')
    if record.get('same_text_multiple_records'):
        tags.append('重复正文')
    return tags


def extract_pdf_text(pdf_path: Path) -> tuple[str, int]:
    reader = PdfReader(str(pdf_path))
    chunks = []
    for i, page in enumerate(reader.pages, 1):
        chunks.append(f"\n<<<PAGE {i}>>>\n")
        chunks.append(page.extract_text() or '')
    return ''.join(chunks), len(reader.pages)


def split_source(tail: str) -> tuple[str, str]:
    tail = tail.strip()
    for source in SOURCES:
        if tail.startswith(source):
            return source, tail[len(source):].strip()
    m = re.match(r'([^，。！？//@【\[]+?)(?=\s{2,}|\s(?=[\u4e00-\u9fff])|$)', tail)
    if m:
        return m.group(1).strip(), tail[m.end():].strip()
    return (tail[:32] if tail else '未知'), ''


def source_group(source: str) -> str:
    if source == '今日头条':
        return '今日头条分享'
    if any(k in source for k in ['iPhone','iPad','Android','微博手机版','手机微博触屏版','Weico']):
        return '移动端'
    if source in ['微博 weibo.com','微博搜索','微博浏览器插件','专业版微博','搜狗高速浏览器','分享按钮']:
        return '微博网页/工具'
    if '豆瓣' in source:
        return '豆瓣'
    return '其他'


def clean_content(text: str) -> str:
    text = text.replace('\r', '')
    text = re.sub(r'<<<PAGE \d+>>>', '\n', text)
    kept = []
    for line in text.splitlines():
        z = line.strip()
        if not z or z == '张一鸣':
            continue
        if re.fullmatch(r'\d{1,3}', z):
            continue
        if '添加微信' in z and '领取200个互联网创业项目' in z:
            continue
        kept.append(z)
    text = '\n'.join(kept)
    text = ADMIN_LINE_RE.sub(' ', text)
    return re.sub(r'\s+', ' ', text).strip()


def split_author_quote(content: str, share: bool, explicit_retweet: bool, toutiao_share: bool) -> tuple[str, str, str]:
    """Deterministically separate visible author note from quoted/shared text.

    This is conservative: when a shared record cannot be separated reliably, it is treated
    as quoted text with empty author_text, avoiding false attribution.
    """
    content = (content or '').strip()
    if not share:
        return content, '', 'not_share'

    # Common Weibo forwarding separator: "comment //@user: quoted..."
    pos = content.find('//@')
    if pos >= 0:
        return content[:pos].strip(' /'), content[pos + 2:].strip(), 'high'

    # Explicit "转发微博" marker normally carries no author note by itself.
    m = re.search(r'(^|\s)转发微博[。.\s]*', content)
    if m:
        before = content[:m.start()].strip()
        after = content[m.end():].strip()
        return before, after, 'high'

    if toutiao_share:
        # Most Toutiao shares use "comment //【headline】...".
        for marker in ('//【', '// 【'):
            pos = content.find(marker)
            if pos >= 0:
                return content[:pos].strip(' /'), content[pos + 2:].strip(), 'high'
        # A leading headline is treated as shared body, not author note.
        if content.startswith('【'):
            return '', content, 'high'
        # If a headline bracket exists later, treat preceding text as note.
        pos = content.find('【')
        if pos > 0:
            return content[:pos].strip(' /'), content[pos:].strip(), 'medium'

    # Conservative fallback for a known share whose split boundary is unclear.
    return '', content, 'low'


def parse_records(text: str) -> list[dict]:
    matches = list(TIMESTAMP_RE.finditer(text))
    records = []
    for i, m in enumerate(matches):
        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
        source, inline = split_source(m.group('tail'))
        raw_block = inline + '\n' + text[m.end():end]
        content = clean_content(raw_block)
        dt = datetime.strptime(m.group('date') + ' ' + m.group('time'), '%Y-%m-%d %H:%M')

        explicit_retweet = bool(re.search(r'(^|\s)(转发微博[。.]?|//@)|//@', content))
        toutiao_share = source == '今日头条' or bool(re.search(r'分享自\s*@?今日头条', content))
        share = explicit_retweet or toutiao_share
        author_text, quoted_text, split_confidence = split_author_quote(content, share, explicit_retweet, toutiao_share)
        unavailable_notice = bool(re.search(r'抱歉|不可见|查看权限|已被.*删除|社区公约', raw_block))
        normalized = re.sub(r'\s+', '', content).replace('O网页链接', '').replace('网页链接', '')
        author_basis = author_text if share else content

        records.append({
            'id': i + 1,
            'datetime': dt.isoformat(),
            'date': dt.date().isoformat(),
            'year': dt.year,
            'month': dt.month,
            'hour': dt.hour,
            'weekday': dt.weekday(),
            'source': source,
            'source_group': source_group(source),
            'char_count': len(re.sub(r'\s+', '', content)),
            'author_char_count': len(re.sub(r'\s+', '', author_text)),
            'quoted_char_count': len(re.sub(r'\s+', '', quoted_text)),
            'share': share,
            'explicit_retweet': explicit_retweet,
            'toutiao_share': toutiao_share,
            'split_confidence': split_confidence,
            'unavailable_notice': unavailable_notice,
            # Form features are author-attributed. Non-share records use the whole text.
            'question': bool(re.search(r'[?？]', author_basis)),
            'first_person': bool(re.search(r'我|自己|本人', author_basis)),
            'has_english': bool(re.search(r'[A-Za-z]{2,}', author_basis)),
            'has_number': bool(re.search(r'\d', author_basis)),
            'author_text': author_text,
            'quoted_text': quoted_text,
            '_content': content,
            '_norm': normalized,
        })

    content_counts = Counter(r['_norm'] for r in records if r['_norm'])
    for r in records:
        r['same_text_multiple_records'] = content_counts[r['_norm']] > 1 if r['_norm'] else False
        r['topic_tags'] = topic_tags(r['author_text'] if r['share'] else r['_content'])
        r['structural_tags'] = structural_tags(r)
    return records


def percentile(values: list[int | float], p: float) -> float:
    if not values:
        return float('nan')
    values = sorted(values)
    x = (len(values) - 1) * p
    lo, hi = math.floor(x), math.ceil(x)
    if lo == hi:
        return float(values[lo])
    return values[lo] * (hi - x) + values[hi] * (x - lo)


def ratio(n: int, d: int) -> float:
    return n / d if d else 0.0


def wilson_interval(k: int, n: int, z: float = 1.959963984540054) -> list[float] | None:
    if n <= 0:
        return None
    p = k / n
    z2 = z * z
    denom = 1 + z2 / n
    center = (p + z2 / (2 * n)) / denom
    margin = z * math.sqrt((p * (1 - p) / n) + z2 / (4 * n * n)) / denom
    return [round(max(0.0, center - margin), 4), round(min(1.0, center + margin), 4)]


def phrase_mining(records: list[dict], limit: int = 30) -> list[dict]:
    docs = [r['_content'] for r in records if not r['share'] and not r['same_text_multiple_records'] and r['char_count'] >= 10]
    sequences = []
    for d in docs:
        d = re.sub(r'@[^\s，。！？:：]+', ' ', d).replace('张一鸣', ' ')
        sequences.extend(re.findall(r'[\u4e00-\u9fff]{2,}', d))

    counts = {n: Counter() for n in range(1, 5)}
    totals = {n: 0 for n in range(1, 5)}
    for s in sequences:
        for n in range(1, 5):
            if len(s) < n:
                continue
            for i in range(len(s) - n + 1):
                g = s[i:i + n]
                counts[n][g] += 1
                totals[n] += 1

    stop_edge = set('的一了是在和与或也就都而及为有我你他她它这那其个些把被让对从到中上下来去里外时后前着过很更最还又但并如果因为所以而且以及等者们之于将可会能要想说做看用给得地呢吗啊呀吧哈哦嗯嘛么无未非')
    admin_terms = ('微博','网页','客户端','查看','抱歉','作者','删除','张一鸣')
    candidates = []
    for n in (2, 3, 4):
        for g, f in counts[n].items():
            if f < 6 or g[0] in stop_edge or g[-1] in stop_edge:
                continue
            if any(x in g for x in admin_terms) or sum(ch in stop_edge for ch in g) >= n - 1:
                continue
            p = f / totals[n]
            split_scores = []
            for k in range(1, n):
                left, right = g[:k], g[k:]
                pl = counts[k][left] / totals[k]
                pr = counts[n-k][right] / totals[n-k]
                split_scores.append(math.log((p + 1e-12) / (pl * pr + 1e-12)))
            cohesion = min(split_scores)
            score = math.log1p(f) * max(cohesion, 0)
            if cohesion > 1.2:
                candidates.append((score, f, cohesion, g))
    candidates.sort(reverse=True)

    selected = []
    for score, f, cohesion, g in candidates:
        if any(g in h and h != g and abs(f - f2) / max(f, f2) < 0.25 for _, f2, _, h in selected):
            continue
        selected.append((score, f, cohesion, g))
        if len(selected) >= limit:
            break
    return [
        {'phrase': g, 'occurrences': f, 'cohesion': round(cohesion, 3), 'score': round(score, 3)}
        for score, f, cohesion, g in selected
    ]



def _author_basis(record: dict) -> str:
    return record['author_text'] if record['share'] else record['_content']

def _chinese_ngrams(text: str, min_n: int = 2, max_n: int = 4) -> list[str]:
    text = re.sub(r'@[^\s，。！？:：]+', ' ', text or '').replace('张一鸣', ' ')
    stop_edge = set('的一了是在和与或也就都而及为有我你他她它这那其个些把被让对从到中上下来去里外时后前着过很更最还又但并如果因为所以而且以及等者们之于将可会能要想说做看用给得地呢吗啊呀吧哈哦嗯嘛么无未非')
    admin_terms = ('微博','网页','客户端','查看','抱歉','作者','删除','张一鸣')
    out = []
    for seq in re.findall(r'[\u4e00-\u9fff]{2,}', text):
        for n in range(min_n, max_n + 1):
            for i in range(max(0, len(seq) - n + 1)):
                g = seq[i:i+n]
                if len(g) != n or g[0] in stop_edge or g[-1] in stop_edge:
                    continue
                if any(x in g for x in admin_terms) or sum(ch in stop_edge for ch in g) >= n - 1:
                    continue
                out.append(g)
    return out

def year_distinctive_terms(records: list[dict], limit: int = 8) -> list[dict]:
    """Exploratory year-vs-rest z-scores for Chinese character n-grams."""
    year_counts = {}
    totals = {}
    for y in sorted({r['year'] for r in records}):
        docs = [_author_basis(r) for r in records if r['year'] == y and _author_basis(r)]
        c = Counter()
        for d in docs:
            c.update(_chinese_ngrams(d, 2, 4))
        year_counts[y] = c
        totals[y] = sum(c.values())
    global_counts = sum(year_counts.values(), Counter())
    total_all = sum(totals.values())
    out = []
    for y in sorted(year_counts):
        if sum(r['year'] == y for r in records) < 20 or totals[y] == 0:
            continue
        ty, tr = totals[y], total_all - totals[y]
        scored = []
        for term, cy in year_counts[y].items():
            if cy < 3:
                continue
            cr = global_counts[term] - cy
            py = cy / ty
            pr = cr / tr if tr else 0
            pooled = (cy + cr) / (ty + tr)
            se = math.sqrt(max(pooled * (1 - pooled) * (1 / ty + (1 / tr if tr else 0)), 1e-12))
            z = (py - pr) / se
            if z > 1.5:
                scored.append((z, cy, cr, term))
        scored.sort(key=lambda x: (x[0], len(x[3]), x[1]), reverse=True)
        selected = []
        for z, cy, cr, term in scored:
            if any(term in t2 and abs(cy - cy2) / max(cy, cy2) < 0.25 for _, cy2, _, t2 in selected):
                continue
            selected.append((z, cy, cr, term))
            if len(selected) >= limit:
                break
        out.append({'year': y, 'terms': [{'phrase': t, 'year_count': cy, 'rest_count': cr, 'z': round(z, 2)} for z, cy, cr, t in selected]})
    return out

def related_records(records: list[dict], limit: int = 4) -> dict[str, list[dict]]:
    """Exploratory related-record index using author-text 2–3 gram TF-IDF cosine similarity."""
    docs = []
    for r in records:
        text = _author_basis(r)
        toks = _chinese_ngrams(text, 2, 3) if text else []
        docs.append(Counter(toks))
    df = Counter()
    for c in docs:
        df.update(c.keys())
    n = len(records)
    vectors = []
    inverted = {}
    for i, c in enumerate(docs):
        weights = {}
        for tok, tf in c.items():
            idf = math.log((n + 1) / (df[tok] + 1)) + 1
            weights[tok] = (1 + math.log(tf)) * idf
        norm = math.sqrt(sum(v * v for v in weights.values())) or 1.0
        weights = {k: v / norm for k, v in weights.items()}
        vectors.append(weights)
        for tok, w in weights.items():
            inverted.setdefault(tok, []).append((i, w))
    result = {}
    for i, r in enumerate(records):
        if len(vectors[i]) < 2:
            continue
        scores = Counter()
        for tok, wi in vectors[i].items():
            for j, wj in inverted.get(tok, []):
                if j != i:
                    scores[j] += wi * wj
        picks = []
        for j, score in scores.most_common(24):
            if score < 0.16:
                break
            rr = records[j]
            if rr['_norm'] and rr['_norm'] == r['_norm']:
                continue
            basis = _author_basis(rr)
            picks.append({'id': rr['id'], 'year': rr['year'], 'date': rr['date'], 'score': round(float(score), 3), 'snippet': basis[:84]})
            if len(picks) >= limit:
                break
        if picks:
            result[str(r['id'])] = picks
    return result

def build_analysis(records: list[dict], page_count: int) -> dict:
    years = sorted(set(r['year'] for r in records))
    core = [r for r in records if not r['share'] and not r['same_text_multiple_records'] and r['char_count'] > 0]

    yearly = []
    for y in years:
        all_y = [r for r in records if r['year'] == y]
        core_y = [r for r in core if r['year'] == y]
        lengths = [r['char_count'] for r in core_y]
        share_n = sum(r['share'] for r in all_y)
        mobile_n = sum(r['source_group'] == '移动端' for r in all_y)
        toutiao_n = sum(r['source_group'] == '今日头条分享' for r in all_y)
        yearly.append({
            'year': y,
            'records': len(all_y),
            'share_records': share_n,
            'share_rate': round(ratio(share_n, len(all_y)), 4),
            'share_rate_ci95': wilson_interval(share_n, len(all_y)),
            'non_share_unique_records': len(core_y),
            'median_chars_non_share': round(statistics.median(lengths), 1) if lengths else None,
            'mobile_rate': round(ratio(mobile_n, len(all_y)), 4),
            'mobile_rate_ci95': wilson_interval(mobile_n, len(all_y)),
            'toutiao_source_rate': round(ratio(toutiao_n, len(all_y)), 4),
            'toutiao_source_rate_ci95': wilson_interval(toutiao_n, len(all_y)),
            'first_person_rate_non_share': round(ratio(sum(r['first_person'] for r in core_y), len(core_y)), 4),
            'question_rate_non_share': round(ratio(sum(r['question'] for r in core_y), len(core_y)), 4),
            'english_rate_non_share': round(ratio(sum(r['has_english'] for r in core_y), len(core_y)), 4),
        })

    monthly_counts = Counter(r['date'][:7] for r in records)
    months = []
    start = min(datetime.fromisoformat(r['datetime']) for r in records)
    end = max(datetime.fromisoformat(r['datetime']) for r in records)
    y, m = start.year, start.month
    while (y, m) <= (end.year, end.month):
        key = f'{y:04d}-{m:02d}'
        months.append({'month': key, 'records': monthly_counts[key]})
        m += 1
        if m == 13:
            y += 1
            m = 1

    hour_counts = Counter(r['hour'] for r in records)
    weekday_counts = Counter(r['weekday'] for r in records)
    heat = [[0 for _ in range(24)] for _ in range(7)]
    for r in records:
        heat[r['weekday']][r['hour']] += 1

    source_counts = Counter(r['source'] for r in records)
    source_groups = ['微博网页/工具','移动端','今日头条分享','豆瓣','其他']
    source_by_year = []
    for y in years:
        c = Counter(r['source_group'] for r in records if r['year'] == y)
        source_by_year.append({'year': y, **{g: c[g] for g in source_groups}})

    lengths = [r['char_count'] for r in core]
    length_bins = []
    for lo, hi in [(1,20),(21,50),(51,100),(101,200),(201,500),(501,10**9)]:
        c = sum(lo <= x <= hi for x in lengths)
        length_bins.append({'label': f'{lo}–{hi}' if hi < 10**9 else '501+', 'count': c, 'rate': round(ratio(c, len(lengths)), 4)})

    norm_counts = Counter(re.sub(r'\s+','',r['_content']) for r in records if r['_content'])
    duplicate_groups = [v for v in norm_counts.values() if v > 1]
    ordered = sorted(records, key=lambda r: r['datetime'])
    dts = [datetime.fromisoformat(r['datetime']) for r in ordered]
    intervals = [(dts[i] - dts[i - 1]).total_seconds() / 3600 for i in range(1, len(dts))]

    linguistic = [
        {'feature': '含第一人称（我/自己/本人）', 'count': sum(r['first_person'] for r in core), 'rate': round(ratio(sum(r['first_person'] for r in core), len(core)), 4)},
        {'feature': '含问号', 'count': sum(r['question'] for r in core), 'rate': round(ratio(sum(r['question'] for r in core), len(core)), 4)},
        {'feature': '含连续英文字符', 'count': sum(r['has_english'] for r in core), 'rate': round(ratio(sum(r['has_english'] for r in core), len(core)), 4)},
        {'feature': '含数字', 'count': sum(r['has_number'] for r in core), 'rate': round(ratio(sum(r['has_number'] for r in core), len(core)), 4)},
    ]

    active_days = len(set(r['date'] for r in records))
    early_count = sum(r['year'] <= 2012 for r in records)
    unavailable = sum(r['unavailable_notice'] for r in records)
    split_stats = Counter(r['split_confidence'] for r in records if r['share'])
    share_with_note = sum(bool(r['author_text']) for r in records if r['share'])

    return {
        'meta': {
            'dataset_version': DATASET_VERSION,
            'parser_version': PARSER_VERSION,
            'tag_rules_version': TAG_RULES_VERSION,
            'source_title': '张一鸣微博日记2286条',
            'pdf_pages': page_count,
            'cover_claimed_records': COVER_CLAIM,
            'parsed_timestamp_records': len(records),
            'difference_vs_cover': len(records) - COVER_CLAIM,
            'coverage_vs_cover': round(ratio(len(records), COVER_CLAIM), 4),
            'date_start': min(r['date'] for r in records),
            'date_end': max(r['date'] for r in records),
            'unavailable_or_deleted_notice_records': unavailable,
            'unavailable_or_deleted_notice_rate': round(ratio(unavailable, len(records)), 4),
            'same_text_multi_record_groups': len(duplicate_groups),
            'same_text_multi_record_records': sum(duplicate_groups),
            'active_days': active_days,
            'records_per_active_day': round(len(records) / active_days, 2),
            'records_2010_2012': early_count,
            'records_2010_2012_rate': round(ratio(early_count, len(records)), 4),
            'core_non_share_unique_records': len(core),
            'share_records_with_author_note': share_with_note,
            'share_split_confidence': dict(split_stats),
        },
        'yearly': yearly,
        'monthly': months,
        'hours': [{'hour': h, 'count': hour_counts[h]} for h in range(24)],
        'weekdays': [{'weekday': WEEKDAYS[d], 'index': d, 'count': weekday_counts[d]} for d in range(7)],
        'heatmap': {'weekdays': WEEKDAYS, 'matrix': heat},
        'sources': [{'source': s, 'count': c, 'rate': round(c / len(records), 4)} for s, c in source_counts.most_common(15)],
        'source_groups': source_groups,
        'source_by_year': source_by_year,
        'length': {
            'population': '未识别为转发/分享、且正文不完全重复的记录',
            'n': len(lengths),
            'mean': round(statistics.mean(lengths), 1),
            'median': round(statistics.median(lengths), 1),
            'p10': round(percentile(lengths, .10), 1),
            'p25': round(percentile(lengths, .25), 1),
            'p75': round(percentile(lengths, .75), 1),
            'p90': round(percentile(lengths, .90), 1),
            'p95': round(percentile(lengths, .95), 1),
            'bins': length_bins,
        },
        'linguistic_features': linguistic,
        'intervals': {
            'median_hours': round(statistics.median(intervals), 2),
            'p90_hours': round(percentile(intervals, .90), 2),
            'max_days': round(max(intervals) / 24, 2),
        },
        'phrases_exploratory': phrase_mining(records, 30),
        'year_terms_exploratory': year_distinctive_terms(records, 8),
        'method': {
            'record_unit': '每个匹配“YYYY-M-D HH:MM 来自 …”的时间戳视为一条记录。',
            'share_rule': '仅当正文出现显式“转发微博/ //@ ”标记，或来源为“今日头条/正文标记分享自今日头条”时，记为转发/分享。',
            'author_quote_rule': '转发记录用可观察分隔符拆成“作者附言”和“转发正文”；无法可靠拆分时保守地把整段放入转发正文，不归因给作者。主题标签只依据作者附言生成。',
            'length_rule': '移除页码、采集广告、作者名单独行与平台删除/不可见提示后，计算非空白字符数。长度分析进一步排除显式转发/分享与完全相同正文的多记录。',
            'source_rule': '来源按 PDF 时间戳行中的“来自”字段解析，并归为微博网页/工具、移动端、今日头条分享、豆瓣、其他。',
            'ci_rule': '年度比例同时给出 Wilson 95% 置信区间；它只表达有限样本下比例估计的不确定性，不修复语料缺失或选择偏差。',
            'phrase_rule': '实验性短语挖掘：对非分享文本抽取2–4字中文n-gram，以出现次数×最小切分凝聚度（PMI-like）排序；该结果不等同于主题分类。',
            'timezone': 'PDF 未声明时区；所有小时/星期统计按文件中显示的本地时间直接计算，不做时区转换。',
            'limitations': [
                '封面称2286条，但正文仅识别到2171个符合时间戳格式的记录；因此不能把该PDF视作完整微博全集。',
                '部分记录含删除/不可见提示；采集文本可能保留链接标题、被截断内容或不完整转发正文。',
                '作者附言/转发正文拆分是确定性启发式规则；低置信度记录保守地不把文本归因给作者。',
                '“转发/分享”是保守的规则识别，不等于对原创权或作者身份的判断。',
                '2014–2016样本量显著变小；即使提供Wilson区间，也不能消除语料覆盖变化造成的偏差。',
                '规则主题标签尚未完成人工金标准验证，因此只用于浏览筛选，不报告分类准确率。',
                '词组挖掘不使用中文分词词典，适合发现重复搭配，不应被解释为心理特征或价值取向。',
            ],
        },
    }


def write_validation_sample(records: list[dict], n: int = 100) -> None:
    rng = random.Random(20260921)
    pool = records[:]
    sample = rng.sample(pool, min(n, len(pool)))
    path = OUT_DIR / 'tag_validation_sample.csv'
    fields = ['id','datetime','author_text','quoted_text','rule_topic_tags','human_topic_tags','is_rule_correct','notes']
    with path.open('w', encoding='utf-8-sig', newline='') as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for r in sorted(sample, key=lambda x: x['datetime']):
            w.writerow({
                'id': r['id'],
                'datetime': r['datetime'],
                'author_text': r['author_text'],
                'quoted_text': r['quoted_text'],
                'rule_topic_tags': ' | '.join(r['topic_tags']),
                'human_topic_tags': '',
                'is_rule_correct': '',
                'notes': '',
            })


def write_outputs(records: list[dict], analysis: dict, source_pdf: Path, page_count: int) -> None:
    with (OUT_DIR / 'analysis.json').open('w', encoding='utf-8') as f:
        json.dump(analysis, f, ensure_ascii=False, indent=2)

    fields = [
        'id','datetime','date','year','month','hour','weekday','source','source_group','char_count','author_char_count','quoted_char_count',
        'share','explicit_retweet','toutiao_share','split_confidence','unavailable_notice','question','first_person','has_english','has_number',
        'same_text_multiple_records','topic_tags','structural_tags','author_text','quoted_text'
    ]
    with (OUT_DIR / 'record_metadata.csv').open('w', encoding='utf-8-sig', newline='') as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for r in records:
            row = {k: r[k] for k in fields}
            row['topic_tags'] = ' | '.join(r['topic_tags'])
            row['structural_tags'] = ' | '.join(r['structural_tags'])
            w.writerow(row)

    timeline_dir = OUT_DIR / 'timeline'
    timeline_dir.mkdir(exist_ok=True)
    years = sorted({r['year'] for r in records}, reverse=True)
    year_meta = []
    topic_counts, structural_counts, source_group_counts, source_counts = Counter(), Counter(), Counter(), Counter()
    for r in records:
        topic_counts.update(r['topic_tags'])
        structural_counts.update(r['structural_tags'])
        source_group_counts[r['source_group']] += 1
        source_counts[r['source']] += 1

    public_fields = [
        'id','datetime','date','year','month','hour','weekday','source','source_group','char_count','author_char_count','quoted_char_count',
        'share','split_confidence','unavailable_notice','question','first_person','has_english','has_number','same_text_multiple_records',
        'topic_tags','structural_tags','author_text','quoted_text'
    ]
    for year in years:
        group = [r for r in records if r['year'] == year]
        payload = []
        for r in sorted(group, key=lambda x: x['datetime'], reverse=True):
            item = {k: r[k] for k in public_fields}
            item['content'] = r['_content']
            payload.append(item)
        filename = f'{year}.json'
        with (timeline_dir / filename).open('w', encoding='utf-8') as f:
            json.dump(payload, f, ensure_ascii=False, separators=(',', ':'))
        year_meta.append({'year': year, 'count': len(payload), 'file': f'./data/timeline/{filename}'})

    index = {
        'dataset_version': DATASET_VERSION,
        'record_count': len(records),
        'date_start': min(r['date'] for r in records),
        'date_end': max(r['date'] for r in records),
        'years': year_meta,
        'source_groups': [{'name': k, 'count': v} for k, v in source_group_counts.most_common()],
        'sources': [{'name': k, 'count': v} for k, v in source_counts.most_common()],
        'topic_tags': [{'name': k, 'count': v} for k, v in topic_counts.most_common()],
        'structural_tags': [{'name': k, 'count': v} for k, v in structural_counts.most_common()],
        'topic_rules': TOPIC_RULES,
        'method_note': '正文来自PDF抽取并清除页码、采集广告与平台提示；不是微博官方API导出。转发记录区分作者附言与转发正文，规则主题标签仅根据作者附言生成；规则标签尚未人工金标准验证。',
    }
    with (timeline_dir / 'index.json').open('w', encoding='utf-8') as f:
        json.dump(index, f, ensure_ascii=False, indent=2)

    with (OUT_DIR / 'related.json').open('w', encoding='utf-8') as f:
        json.dump(related_records(records, 4), f, ensure_ascii=False, separators=(',', ':'))

    manifest = {
        'dataset_version': DATASET_VERSION,
        'parser_version': PARSER_VERSION,
        'tag_rules_version': TAG_RULES_VERSION,
        'generated_at_utc': datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
        'source': {
            'title': '张一鸣微博日记2286条',
            'filename': source_pdf.name,
            'sha256': sha256_file(source_pdf),
            'bytes': source_pdf.stat().st_size,
            'pages': page_count,
            'cover_claimed_records': COVER_CLAIM,
        },
        'parsed': {
            'timestamp_records': len(records),
            'coverage_vs_cover': round(len(records) / COVER_CLAIM, 6),
            'date_start': min(r['date'] for r in records),
            'date_end': max(r['date'] for r in records),
            'share_records': sum(r['share'] for r in records),
            'share_records_with_author_note': sum(r['share'] and bool(r['author_text']) for r in records),
            'low_confidence_share_splits': sum(r['share'] and r['split_confidence'] == 'low' for r in records),
        },
        'validation': {
            'topic_rule_gold_standard_completed': False,
            'precision_reported': False,
            'review_sample_file': './data/tag_validation_sample.csv',
            'note': '已生成固定随机种子的100条人工复核样本；在人工标注完成前不报告规则标签准确率。',
        },
        'artifacts': [
            './data/analysis.json', './data/record_metadata.csv', './data/timeline/index.json', './data/tag_validation_sample.csv', './data/related.json'
        ],
    }
    with (OUT_DIR / 'manifest.json').open('w', encoding='utf-8') as f:
        json.dump(manifest, f, ensure_ascii=False, indent=2)

    write_validation_sample(records)


if __name__ == '__main__':
    pdf = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / 'source.pdf'
    if not pdf.exists():
        raise SystemExit(f'PDF not found: {pdf}')
    raw, pages = extract_pdf_text(pdf)
    records = parse_records(raw)
    analysis = build_analysis(records, pages)
    write_outputs(records, analysis, pdf, pages)
    print(json.dumps(analysis['meta'], ensure_ascii=False, indent=2))