#!/usr/bin/env python3
# collect_news.py (2026-07-15 전면개편) — 수집은 넓게, 필터는 alert.py가.
import feedparser, json, os, html, urllib.parse, socket
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor

socket.setdefaulttimeout(20)   # 피드 하나가 멈추면 전체가 무한 대기하던 문제

BASE  = os.path.expanduser("~/Claude/Projects/에이전트구축")
JSONL = os.path.join(BASE, "media_feed.jsonl")

TECH_FEEDS = [
    "https://www.eetimes.com/feed/",
    "https://semiengineering.com/feed/",
    "https://spectrum.ieee.org/rss/fulltext",
    "https://www.tomshardware.com/feeds/all",
    "https://tradingeconomics.com/rss/news.aspx",
]

# 종목 단위 커버리지 — 키워드 문구와 무관하게 유입 (7/16 Gemini·GLW 구멍이 계기)
YF_TICKERS = [
    "MU", "SNDK", "NBIS", "AAOI", "DRAM", "SOXL", "SOXX", "EWY", "SPCX",
    "NVDA", "META", "GOOGL", "MSFT", "AMZN", "ORCL", "IBM", "JPM",
    "LITE", "COHR", "GLW", "CIEN", "CRDO", "MRVL", "AVGO", "ALAB", "POET", "AXTI",
    "ASML", "AMAT", "LRCX", "KLAC", "TER", "AEHR", "GFS", "TSEM", "AMKR",
    "AMD", "INTC", "TSM", "CRWV", "IREN", "CBRS", "NOW", "CRM", "ACN",
]


STALE_CUTOFF_H = 96  # pub이 96시간(4일)보다 오래된 기사는 스테일로 버림

def _pubdt(v):
    if not v: return None
    for f in (lambda x: parsedate_to_datetime(x),
              lambda x: datetime.fromisoformat(x.replace("Z","+00:00"))):
        try:
            d = f(v)
            return d if d.tzinfo else d.replace(tzinfo=timezone.utc)
        except Exception:
            pass
    return None

def _is_stale(it):
    d = _pubdt(it.get("pub",""))
    if d is None:
        return False  # pub 없으면 통과(야후 종목뉴스 등)
    return d < datetime.now(timezone.utc) - timedelta(hours=STALE_CUTOFF_H)


def yf(t):
    return ("https://feeds.finance.yahoo.com/rss/2.0/headline?s="
            + t + "&region=US&lang=en-US")

def gnews(q, lang="en-US", geo="US"):
    ceid = f"{geo}:{lang.split('-')[0]}"
    return "https://news.google.com/rss/search?q=" + urllib.parse.quote(q) \
           + f"&hl={lang}&gl={geo}&ceid={ceid}"

# 축1~2 유니버스 (룰24)
FIN_EN = [
    "SK Hynix", "Samsung Electronics chip", "Micron memory", "SanDisk SNDK",
    "Nvidia stock", "Broadcom AI", "Marvell AI infrastructure",
    "Lumentum Coherent optical",
    "Corning optical fiber earnings",
    "Ciena networking earnings", "Applied Optoelectronics AAOI",
    "AXT Inc AXTI indium phosphide substrate",
    "AXTI earnings results",
    "Nebius contract deal",
    "CoreWeave contract backlog",
    "IREN AI cloud contract",
    "neocloud GPU pricing margin",
    "Meta Compute capacity resale", "Meta Platforms AI earnings",
    "semiconductor stocks", "Nasdaq chip stocks",

    # 장비 — 7/15 LRCX -5.83% 미포착이 계기. 쿼리 자체가 없었음
    "Lam Research earnings",
    "ASML Applied Materials KLA wafer fab equipment",
    "Teradyne Astera Labs earnings",
    "Aehr Test Systems burn-in",

    # 축3 수요측 — 반도체를 사는 쪽. IBM/JPM 놓친 게 계기
    "AI capex guidance hyperscaler",
    "hyperscaler capex gigawatt buildout announcement",
    "Meta Microsoft Google Amazon data center investment billion",
    "AI token cost inference spending",
    "IBM enterprise IT spending hardware",
    "ServiceNow Accenture Salesforce results",
    "JPMorgan bank AI spending",

    # 축6 기술 로드맵
    "Nvidia Rubin delay production",
    "HBM4 pricing supply",
    "HBF high bandwidth flash",
    "CXMT YMTC China DRAM NAND",
    "CoWoS interposer capacity",
    "co-packaged optics silicon photonics CPO",
    "KV cache inference memory efficiency",

    # 구글/Gemini — 7/16 Gemini 3.5 지연 미포착이 계기
    "Google Gemini model release delay",
    "Alphabet Google AI capex TPU",
    "Gemini model benchmark frontier",

    # 중국 프론티어 모델 — 룰6 반증축 (7/16 Kimi K3 미포착이 계기)
    "DeepSeek Qwen Chinese AI model release",
    "China frontier AI model open source pricing",
    "Kimi Moonshot MiniMax Zhipu AI model",
    "Chinese AI model training cost inference efficiency",
    # 축A 추론수요 실측 — 룰6 "최종수요 증거"
    "OpenAI tokens per minute API usage",
    "AI inference token demand growth enterprise",
    "OpenRouter token usage model ranking",
    "enterprise AI spending token cost control",
    # 축B 모델 경쟁·성능당 가격
    "frontier model pricing per million tokens",
    "OpenAI Anthropic model release coding benchmark",
    # 축C 최종재 수요 파괴 — 메모리값이 PC/폰을 죽이는가
    "notebook PC shipment forecast cut memory price",
    "smartphone price increase memory cost BOM",
    # 축D 기관 가격 전망 (트렌드포스 vs 제프리스 충돌)
    "TrendForce DRAM contract price forecast QoQ",
    "Jefferies Counterpoint memory price forecast 2026",
    "NAND flash contract price enterprise SSD",
    # 축E 엣지 — LPDDR 순풍 / HBM 역풍
    "LPDDR edge AI automotive memory demand",
    "on-device NPU inference memory smartphone PC",

    # CXMT 실물 검증축 (2026-07-17) — 점유율%가 아니라 이 셋이 트리거
    "HBM wafer capacity crowding out conventional DRAM",
    "Apple Dell Lenovo CXMT DRAM qualification adoption",
    "US ban Chinese memory chips CXMT YMTC lawmakers",

    # 전주 5사 지갑 감시 (2026-07-17) — CapEx 가이던스보다 빠른 신호
    "hyperscaler free cash flow negative capex crossover",
    "Meta Alphabet Amazon Microsoft bond issuance AI datacenter debt",
    "hyperscaler buyback pause suspend repurchase capex",
    "data center SPV private credit financing hyperscaler",
    "Alphabet Meta Amazon capex guidance raise 2027",

    # 룰29 논지 검증
    "memory long-term agreement pricing DRAM spot price",

    # 매크로 지표·연준·정책 감시축 (2026-07-20)
    "US consumer sentiment inflation expectations Michigan",
    "Fed speaker Hassett Jefferson inflation rate hike 2026",
    "Trump universal tariff 301 China July 2026 expiration",
    "housing starts industrial production import prices June 2026",
    "CPC Black Sea oil terminal drone attack crude supply",

    # AI모델 발표 감시축 (2026-07-19) — 인프라 투자방향 영향 판정용
    "DeepSeek V4 release inference cost active parameters",
    "Kimi K3 memory HBM demand impact semiconductor",
    "Qwen 3.8 Alibaba parameters MoE benchmark",
    "Chinese open weight model AI capex memory impact",
    "AI model inference efficiency breakthrough memory reduction",

    # 광학 정부육성 감시축 (2026-07-18) — 이노라이트 지배 심화 → 미국이 뭘 키우나
    # (1) 국내 모듈 캐파 (2) 초크포인트 부품 EML/DSP (3) CPO/SiPh 판갈이
    "US optical module domestic manufacturing capacity CHIPS",
    "photonics defense procurement DoD optical supply chain",
    "Innolight Zhongji Pentagon entity list optical module",
    "Chinese optical transceiver import restriction tariff",
    "Lumentum EML laser capacity expansion 1.6T",
    "hyperscaler US optical supplier diversification Amazon AAOI",
    "co-packaged optics CPO silicon photonics US foundry GlobalFoundries Tower",
    "Coherent vertical integration optics US onshore",

    # 텔레그램 우회 채널 2개 (alert.py가 1차소스만 통과시킴)
    "Iran Hormuz strait",
    "Trump tariff semiconductor export controls",
]

# 한국 매체는 메모리 논지·실적 검증용만. 정책/수급 쿼리는 2026-07-15 폐기(나스닥 집중).
FIN_KR = [
    "해외 거래소 규제 금감원",
    "가상자산 거래소 무허가 차단",
    "레버리지 ETF 규제 금융위",

    "SK하이닉스 실적 컨콜 HBM",
    "삼성전자 반도체 실적 전망",
    "HBM 가격 전망",
    "메모리 반도체 수출 단가",
]

FEEDS = list(TECH_FEEDS) + [gnews(q) for q in FIN_EN] \
        + [gnews(q, "ko-KR", "KR") for q in FIN_KR] \
        + [yf(t) for t in YF_TICKERS]

def domain(link):
    try: return urllib.parse.urlparse(link).netloc.replace("www.", "")
    except Exception: return ""

def _one(url):
    out = []
    try:
        d = feedparser.parse(url)
        for e in d.entries[:8]:
            link = e.get("link", "")
            if not link: continue
            src = e.source.title if (e.get("source") and e.source.get("title")) else domain(link)
            out.append({"url": link,
                "text": html.unescape(e.get("title", "")).strip(),
                "pub": e.get("published", e.get("updated", "")),
                "source": src,
                "ts": datetime.now(timezone.utc).isoformat()})
    except Exception as ex:
        print("feed error:", url[:70], ex)
    return out

def fetch():
    items = []
    with ThreadPoolExecutor(max_workers=8) as ex:
        for r in ex.map(_one, FEEDS):
            items.extend(r)
    return items

def load_seen():
    seen = set()
    if os.path.exists(JSONL):
        with open(JSONL, encoding="utf-8") as f:
            for line in f:
                try: seen.add(json.loads(line)["url"])
                except Exception: pass
    return seen

def save(items):
    seen = load_seen()
    new = [it for it in items if it["url"] and it["url"] not in seen and not _is_stale(it)]
    with open(JSONL, "a", encoding="utf-8") as f:
        for it in new:
            f.write(json.dumps(it, ensure_ascii=False) + "\n")
    return new

if __name__ == "__main__":
    os.makedirs(BASE, exist_ok=True)
    new = save(fetch())
    print(f"collected {len(new)} new items")