{
  "what": "用外部高质量站点的**互不重复**页面, 标定 scripts/dedup_gate.py 的判据阈值。",
  "why": "旧阈值 max cosine < 0.80 是拍出来的, 没有任何对照支撑。按它连改三轮之后出现了典型的追指标形状: 每从首页删一段, 剩下的内容就与另一个页面更像。于是拿公认独立、彼此毫无重复关系的页面来测这把尺子。",
  "finding": "同站内互不相关的页面 cosine 最高到 0.841 —— 它们全都会被 0.80 判 BLOCK。一元词 cosine 在同题材内容页上底噪很高: 共享的是题材词汇与页面骨架。 进一步比较发现: 真重复 (cosine 0.864-0.949) 与外部对照 (0.765-0.841) 在 cosine 上**区间重叠**, 任何单一 cosine 阈值都做不到既拦住前者又放过后者。改用 Jaccard: 真重复 0.532-0.772, 对照 0.247-0.409, 中间有干净空档。",
  "new_threshold": {
    "primary": {
      "metric": "jaccard",
      "block_at": 0.47,
      "rationale": "两类样本之间空档 (0.409, 0.532) 的中点。"
    },
    "fallback": {
      "metric": "cosine",
      "block_at": 0.92,
      "rationale": "只抓词频分布也几乎一致的整页复制; 远高于最高对照 0.841。"
    },
    "superseded": {
      "metric": "cosine",
      "block_at": 0.88,
      "why": "见 correction —— 它放过了两对真重复。"
    }
  },
  "controls": {
    "wikipedia:Stardew Valley": {
      "url": "https://en.wikipedia.org/wiki/Stardew_Valley",
      "bytes": 715795,
      "sha256": "d91dbd4a4125077ff2cdf498cbcd9a031fdefaaf48ea5257188a40a72830cc88"
    },
    "wikipedia:Terraria": {
      "url": "https://en.wikipedia.org/wiki/Terraria",
      "bytes": 720086,
      "sha256": "3ba5edb920c2db0a740bb7fe616a9252d8a6a486b6bc8481a103b97fbc42d794"
    },
    "wikipedia:Hades": {
      "url": "https://en.wikipedia.org/wiki/Hades_(video_game)",
      "bytes": 485515,
      "sha256": "e144b206eb6acd8c4c2ca0f5e2ecff45befd54ec26fd105a36c1742407ddf354"
    },
    "pcgamingwiki:Stardew Valley": {
      "url": "https://www.pcgamingwiki.com/wiki/Stardew_Valley",
      "bytes": 152581,
      "sha256": "6e4199368d6e41005f3986fde6e86253994669c06eb4912ba89cd0360347e68a"
    },
    "pcgamingwiki:Terraria": {
      "url": "https://www.pcgamingwiki.com/wiki/Terraria",
      "bytes": 176297,
      "sha256": "63cf0c216ab87f52701ac9b33dea4818c4c6029ec6369629254c939638378e35"
    }
  },
  "pairs": [
    {
      "a": "wikipedia:Stardew Valley",
      "b": "wikipedia:Terraria",
      "cosine": 0.805,
      "jaccard": 0.294
    },
    {
      "a": "wikipedia:Stardew Valley",
      "b": "wikipedia:Hades",
      "cosine": 0.818,
      "jaccard": 0.26
    },
    {
      "a": "wikipedia:Terraria",
      "b": "wikipedia:Hades",
      "cosine": 0.765,
      "jaccard": 0.247
    },
    {
      "a": "pcgamingwiki:Stardew Valley",
      "b": "pcgamingwiki:Terraria",
      "cosine": 0.841,
      "jaccard": 0.409
    }
  ],
  "correction": {
    "retracted": "本文件第一版写了「新阈值 0.88 仍然拦得住 0.949 / 0.877 / 0.864 三对」。**这是错的**: 0.877 与 0.864 都低于 0.88, 一次算术就能发现, 而我没有算。",
    "why_it_matters": "那句话的作用是让「放宽阈值」看起来没有代价 —— 一个顺着自己立场、又没有验算的断言。订正保留在这里, 不静默改掉。"
  },
  "blocked_by_jaccard_rule": {
    "games vs where-to-play (合并前)": 0.554,
    "zenonia-4 vs zenonia-5 (合并前)": 0.772,
    "zenonia-4 vs zenonia-5 (只删镜像段后)": 0.532,
    "note": "三对全部 >=0.47, 按新判据都会被拦 —— 这次是算过的。"
  }
}