-- cleanup_false_positives.sql
-- 按 Phase 0 新规则统计/删除误判文献
-- 用法:
--   统计:  docker exec -i scilit-postgres-1 psql -U scilit -d scilit < cleanup_false_positives.sql -v mode=stats
--   删除:  docker exec -i scilit-postgres-1 psql -U scilit -d scilit < cleanup_false_positives.sql -v mode=delete

-- 临时表：加载 mtrees
CREATE TEMP TABLE tmp_mtrees (
    desc_name TEXT NOT NULL,
    tree_number TEXT NOT NULL
);

-- 从文件导入 mtrees（需先 docker cp 到容器内）
-- 格式: "Lung Neoplasms;C04.588.894.797.520;D008175"
\set mtrees_path '/tmp/mtrees2025.bin'
COPY tmp_mtrees(desc_name, tree_number)
FROM :'mtrees_path'
WITH (FORMAT csv, DELIMITER ';', HEADER false);

-- 提取 C04 树号前缀的描述词
CREATE TEMP TABLE tmp_c04_descriptors AS
SELECT DISTINCT LOWER(desc_name) AS desc_name
FROM tmp_mtrees
WHERE tree_number LIKE 'C04%' OR tree_number LIKE 'C04.%';

-- 主查询：标记每条文献的审核结果
CREATE TEMP TABLE tmp_scan_result AS
SELECT
    gl.id,
    gl.pmid,
    gl.title,
    gl.source,
    CASE
        -- 有 MeSH → Tier 1: 检查 C04
        WHEN gl.mesh_headings IS NOT NULL
             AND jsonb_array_length(gl.mesh_headings::jsonb) > 0
        THEN
            CASE
                WHEN EXISTS (
                    SELECT 1
                    FROM jsonb_array_elements(gl.mesh_headings::jsonb) AS mh,
                         LATERAL (SELECT LOWER(mh->>'descriptor') AS d) AS d
                    INNER JOIN tmp_c04_descriptors c04 ON c04.desc_name = d.d
                ) THEN 'keep'
                ELSE 'delete_has_mesh_no_c04'
            END
        -- 无 MeSH → Tier 2: 文本评分
        ELSE
            CASE
                WHEN gl.title IS NULL AND gl.abstract IS NULL THEN 'delete_no_mesh_text_fail'
                ELSE
                    CASE
                        -- 标题命中 → keep
                        WHEN EXISTS (
                            SELECT 1 FROM (VALUES
                                ('cancer'),('carcinoma'),('tumor'),('tumour'),('neoplasm'),('malignancy'),
                                ('leukemia'),('leukaemia'),('lymphoma'),('sarcoma'),('melanoma'),
                                ('glioma'),('glioblastoma'),('myeloma'),('neuroblastoma'),
                                ('chemotherapy'),('radiotherapy'),('immunotherapy'),('oncolog'),
                                ('metasta'),('antineoplastic')
                            ) AS kw(k)
                            WHERE POSITION(k IN LOWER(COALESCE(gl.title, ''))) > 0
                        ) THEN 'keep'
                        -- 摘要 ≥2 关键词 → keep
                        WHEN (
                            SELECT COUNT(*)
                            FROM (VALUES
                                ('cancer'),('carcinoma'),('tumor'),('tumour'),('neoplasm'),('malignancy'),
                                ('leukemia'),('leukaemia'),('lymphoma'),('sarcoma'),('melanoma'),
                                ('glioma'),('glioblastoma'),('myeloma'),('neuroblastoma'),
                                ('chemotherapy'),('radiotherapy'),('immunotherapy'),('oncolog'),
                                ('metasta'),('antineoplastic')
                            ) AS kw(k)
                            WHERE POSITION(k IN LOWER(COALESCE(gl.title, '') || ' ' || COALESCE(gl.abstract, ''))) > 0
                        ) >= 2 THEN 'keep'
                        ELSE 'delete_no_mesh_text_fail'
                    END
            END
    END AS verdict
FROM global_literature gl;

-- 输出统计
SELECT verdict, COUNT(*) AS cnt FROM tmp_scan_result GROUP BY verdict ORDER BY verdict;

-- 根据模式执行
\if :{?mode}
    \if :mode = 'delete'
        -- 实际删除
        BEGIN;
        DELETE FROM global_literature WHERE id IN (
            SELECT id FROM tmp_scan_result WHERE verdict LIKE 'delete%'
        );
        COMMIT;
        -- 删除后统计
        SELECT 'After cleanup' AS stage, COUNT(*) AS total FROM global_literature;
    \else
        -- 预览样例
        SELECT pmid, LEFT(COALESCE(title, ''), 80) AS title_trunc, source, verdict
        FROM tmp_scan_result
        WHERE verdict LIKE 'delete%'
        ORDER BY pmid
        LIMIT 20;
    \endif
\endif
