#!/usr/bin/env python3
# collect_news.py (수정판2 2026-07-13) — 기술RSS + 구글뉴스 금융RSS(영/한).
# rebuild_txt 제거: media_feed.txt는 .jsonl 심링크라 자동 동기화되며, txt 별도생성은 jsonl을 덮어써 손상시킴.
import feedparser, json, os, html, urllib.parse
from datetime import datetime, timezone

BASE  = os.path.expanduser("~/Claude/Projects/에이전트구축")
JSONL = os.path.join(BASE, "media_feed.jsonl")

TECH_FEEDS = [
    "https://www.eetimes.com/feed/",
    "https://semiengineering.com/feed/",
    "https://spectrum.ieee.org/rss/fulltext",
    "https://www.tomshardware.com/feeds/all",
    "https://rss.etnews.com/Section901.xml",
    "https://tradingeconomics.com/rss/news.aspx",
]

def gnews(q, lang="en-US", geo="US"):
    ceid = f"{geo}:{lang.split('-')[0]}"
    return "https://news.google.com/rss/search?q=" + urllib.parse.quote(q) \
           + f"&hl={lang}&gl={geo}&ceid={ceid}"

FIN_EN = ["SK Hynix", "Samsung Electronics chip", "Micron memory",
    "memory chip HBM DRAM oversupply", "Nvidia stock", "Broadcom AI",
    "Marvell AI infrastructure", "Lumentum Coherent optical", "CoreWeave Nebius",
    "semiconductor stocks", "Nasdaq chip stocks", "KOSPI"]
FIN_KR = ["SK하이닉스", "삼성전자 반도체", "메모리 반도체 HBM",
    "코스피 반도체", "엔비디아 반도체", "반도체주"]

FEEDS = list(TECH_FEEDS) + [gnews(q) for q in FIN_EN] \
        + [gnews(q, "ko-KR", "KR") for q in FIN_KR]

def domain(link):
    try: return urllib.parse.urlparse(link).netloc.replace("www.", "")
    except Exception: return ""

def fetch():
    items = []
    for url in FEEDS:
        try:
            d = feedparser.parse(url)
            for e in d.entries[:8]:
                link = e.get("link", "")
                if not link: continue
                src = e.source.title if (e.get("source") and e.source.get("title")) else domain(link)
                items.append({"url": link,
                    "text": html.unescape(e.get("title", "")).strip(),
                    "pub": e.get("published", e.get("updated", "")),
                    "source": src,
                    "ts": datetime.now(timezone.utc).isoformat()})
        except Exception as ex:
            print("feed error:", url, ex)
    return items

def load_seen():
    seen = set()
    if os.path.exists(JSONL):
        with open(JSONL, encoding="utf-8") as f:
            for line in f:
                try: seen.add(json.loads(line)["url"])
                except Exception: pass
    return seen

def save(items):
    seen = load_seen()
    new = [it for it in items if it["url"] and it["url"] not in seen]
    with open(JSONL, "a", encoding="utf-8") as f:
        for it in new:
            f.write(json.dumps(it, ensure_ascii=False) + "\n")
    return new

if __name__ == "__main__":
    os.makedirs(BASE, exist_ok=True)
    new = save(fetch())
    print(f"collected {len(new)} new items")