feat: initial commit - oncology literature search platform

OncoLit: a multi-tenant oncology literature search, feed, and collaboration platform. Built with FastAPI + Vue 3 + PostgreSQL. Includes PubMed pipeline, drug approvals, AI summaries, and systematic review tools.
2026-07-27 07:59:18 +08:00
commit a6cd99a4ca
473 changed files with 151472 additions and 0 deletions
@@ -0,0 +1,101 @@
+"""
+Test efetch date parsing for YYYY-01-01 records — CORRECTED parse
+"""
+import asyncio, httpx, lxml.etree as ET
+from datetime import date
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+
+DATABASE_URL = 'postgresql+asyncpg://scilit:scilit_prod_2026@postgres:5432/scilit'
+
+MONTH_MAP = {
+    'jan':1,'january':1,'1':1,'feb':2,'february':2,'2':2,'mar':3,'march':3,'3':3,
+    'apr':4,'april':4,'4':4,'may':5,'5':5,'jun':6,'june':6,'6':6,
+    'jul':7,'july':7,'7':7,'aug':8,'august':8,'8':8,'sep':9,'september':9,'9':9,
+    'oct':10,'october':10,'10':10,'nov':11,'november':11,'11':11,'dec':12,'december':12,'12':12,
+}
+DAYS = [0,31,29,31,30,31,30,31,31,30,31,30,31]
+
+def month_num(m): return MONTH_MAP.get(m.strip().lower(), 1)
+def clamp_day(m, d, y):
+    if d < 1: return 1
+    md = DAYS[m]
+    if m == 2 and d == 29 and not (y%4==0 and (y%100!=0 or y%400==0)): md = 28
+    return min(d, md)
+
+def parse_pub_date(elem):
+    """Same logic as fix_dates_via_efetch.py"""
+    pde = elem.find('.//PubDate')
+    if pde is None: return None
+    ye = pde.find('Year'); me = pde.find('Month'); de = pde.find('Day')
+    if ye is not None and ye.text:
+        y = int(ye.text.strip())
+        hm = me is not None and me.text; hd = de is not None and de.text
+        if not hm and not hd: return None
+        m = month_num(me.text.strip()) if hm else 1
+        d = int(de.text.strip()) if hd else 1
+        d = clamp_day(m, d, y)  # clamp, not swap
+        return date(y, m, d)
+    return None
+
+async def main():
+    engine = create_async_engine(DATABASE_URL)
+    async with engine.connect() as conn:
+        # Sample across all years with YYYY-01-01
+        r = await conn.execute(text("""
+            SELECT pub_year, COUNT(*) as cnt FROM global_literature
+            WHERE pub_date IS NOT NULL AND EXTRACT(MONTH FROM pub_date)=1 AND EXTRACT(DAY FROM pub_date)=1
+            AND pmid IS NOT NULL
+            GROUP BY pub_year ORDER BY pub_year DESC
+        """))
+        print('=== YYYY-01-01 records by year ===')
+        for row in r:
+            print(f'  {row.pub_year}: {row.cnt}')
+
+        # Get 2 PMIDs from each year that has >0 records
+        r = await conn.execute(text("""
+            SELECT id, pmid, pub_date, pub_year FROM global_literature
+            WHERE pub_date IS NOT NULL AND EXTRACT(MONTH FROM pub_date)=1 AND EXTRACT(DAY FROM pub_date)=1
+            AND pmid IS NOT NULL
+            ORDER BY pub_year DESC, pmid DESC LIMIT 40
+        """))
+        samples = r.all()
+    await engine.dispose()
+
+    # Efetch and report which PMIDs have better dates
+    test_pmids = [row.pmid for row in samples]
+    print(f'\n=== Testing {len(test_pmids)} PMIDs via efetch ===')
+    async with httpx.AsyncClient(timeout=30) as c:
+        resp = await c.get(
+            'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi',
+            params={'db': 'pubmed', 'id': ','.join(str(p) for p in test_pmids), 'retmode': 'xml'}
+        )
+        print(f'HTTP status={resp.status_code} content_len={len(resp.content)}')
+        root = ET.fromstring(resp.content)
+        arts = root.findall('PubmedArticle')
+        print(f'Articles returned: {len(arts)}')
+
+        fixed = 0; year_only = 0; still_jan01 = 0
+        efetch_map = {}
+        for art in arts:
+            pmid_e = art.find('MedlineCitation/PMID')
+            pmid = int(pmid_e.text) if pmid_e is not None else None
+            pd = parse_pub_date(art)
+            if pmid: efetch_map[pmid] = pd
+
+        for row in samples:
+            pd = efetch_map.get(row.pmid)
+            if pd is None:
+                year_only += 1
+            elif pd.month == 1 and pd.day == 1:
+                still_jan01 += 1
+            else:
+                fixed += 1
+                print(f'  CAN FIX: PMID={row.pmid} {row.pub_date} -> {pd}')
+
+        print(f'\n=== Summary ===')
+        print(f'  Can fix (better date found): {fixed}')
+        print(f'  Year-only (no month/day):   {year_only}')
+        print(f'  Still Jan-01:               {still_jan01}')
+
+asyncio.run(main())