-- cleanup_stats.sql
-- 加载 mtrees（行格式有 2 或 3 列，先按文本行读取再拆分）
-- 注意：mtrees 描述词前有前导空格（" Neoplasms;C04"），务必 TRIM
CREATE TEMP TABLE tmp_mtrees_raw (line TEXT);
COPY tmp_mtrees_raw FROM '/tmp/mtrees2025.bin' WITH (FORMAT text);

CREATE TEMP TABLE tmp_mtrees AS
SELECT
  TRIM(SPLIT_PART(line, ';', 1)) AS desc_name,
  TRIM(SPLIT_PART(line, ';', 2)) AS tree_number
FROM tmp_mtrees_raw;

CREATE TEMP TABLE tmp_c04 AS
SELECT DISTINCT LOWER(desc_name) AS name FROM tmp_mtrees
WHERE tree_number LIKE 'C04.%' OR tree_number = 'C04';

-- 验证：确认 C04 匹配生效
SELECT '--- C04 match check ---' AS info;
SELECT LOWER(mh->>'descriptor') AS desc_name, COUNT(*) AS cnt
FROM global_literature gl,
     jsonb_array_elements(gl.mesh_headings::jsonb) mh
WHERE gl.mesh_headings IS NOT NULL AND jsonb_array_length(gl.mesh_headings::jsonb) > 0
  AND LOWER(mh->>'descriptor') IN (SELECT name FROM tmp_c04)
GROUP BY LOWER(mh->>'descriptor')
ORDER BY cnt DESC
LIMIT 10;

-- 关键词
CREATE TEMP TABLE tmp_kw AS
SELECT unnest(ARRAY[
    'cancer','carcinoma','tumor','tumour','neoplasm','malignancy',
    'leukemia','leukaemia','lymphoma','sarcoma','melanoma',
    'glioma','glioblastoma','myeloma','neuroblastoma',
    'chemotherapy','radiotherapy','immunotherapy','oncolog',
    'metasta','antineoplastic'
]) AS kw;

-- 核心统计
SELECT '--- Cleanup stats ---' AS info;
WITH scan AS (
    SELECT gl.id,
        CASE
            WHEN gl.mesh_headings IS NOT NULL AND jsonb_array_length(gl.mesh_headings::jsonb) > 0 THEN
                CASE
                    WHEN EXISTS (
                        SELECT 1 FROM jsonb_array_elements(gl.mesh_headings::jsonb) mh
                        JOIN tmp_c04 c04 ON c04.name = LOWER(mh->>'descriptor')
                    ) THEN 'keep_has_mesh'
                    ELSE 'delete_has_mesh_no_c04'
                END
            ELSE
                CASE
                    WHEN gl.title IS NULL AND gl.abstract IS NULL THEN 'delete_no_mesh_no_text'
                    WHEN EXISTS (
                        SELECT 1 FROM tmp_kw
                        WHERE POSITION(kw IN LOWER(COALESCE(gl.title, ''))) > 0
                    ) THEN 'keep_no_mesh_title_hit'
                    WHEN (
                        SELECT COUNT(*) FROM tmp_kw
                        WHERE POSITION(kw IN LOWER(COALESCE(gl.title,'') || ' ' || COALESCE(gl.abstract,''))) > 0
                    ) >= 2 THEN 'keep_no_mesh_abstract_2plus'
                    ELSE 'delete_no_mesh_text_fail'
                END
        END AS verdict
    FROM global_literature gl
)
SELECT verdict, COUNT(*) AS cnt FROM scan GROUP BY verdict ORDER BY verdict;
