import subprocess, json, os, re, time, requests, websocket
from datetime import datetime, timezone

BASE = os.path.expanduser("~/Claude/Projects/에이전트구축")
XPROF = os.path.join(BASE, ".xchrome_profile")
FEED = os.path.join(BASE, "x_feed.jsonl")
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
PORT = 9333

EXTRACT_JS = r"""
(function(){
  var a=document.getElementsByTagName('article');var o=[];
  for(var i=0;i<a.length;i++){
    var tm=a[i].getElementsByTagName('time')[0];var link='';
    if(tm){var p=tm.parentNode;while(p&&p.tagName!='A'){p=p.parentNode}if(p)link=p.href}
    o.push((tm?tm.getAttribute('datetime'):'')+'@@@'+a[i].innerText.replace(/\n/g,' ')+'@@@'+link)
  }
  return o.join('###');
})()
"""

def refresh_session():
    src = os.path.expanduser("~/Library/Application Support/Google/Chrome/Default")
    dst = os.path.join(XPROF, "Default")
    os.makedirs(dst, exist_ok=True)
    for f in ["Cookies"]:
        subprocess.run(["cp", os.path.join(src, f), dst], capture_output=True)
    for d in ["Local Storage", "Network"]:
        subprocess.run(["cp", "-R", os.path.join(src, d), dst], capture_output=True)
    subprocess.run(["cp", os.path.expanduser("~/Library/Application Support/Google/Chrome/Local State"), XPROF], capture_output=True)

def launch_chrome():
    subprocess.run(["pkill", "-f", f"remote-debugging-port={PORT}"], capture_output=True)
    time.sleep(1)
    return subprocess.Popen([
        CHROME, "--headless=new", "--disable-gpu", f"--remote-debugging-port={PORT}",
        "--remote-allow-origins=*", f"--user-data-dir={XPROF}",
        "--no-first-run", "--no-default-browser-check", "https://x.com/home"
    ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

def cdp_extract():
    for _ in range(20):
        try:
            tabs = requests.get(f"http://localhost:{PORT}/json").json()
            page = next((t for t in tabs if t.get("type")=="page"), None)
            if page and page.get("webSocketDebuggerUrl"): break
        except Exception: pass
        time.sleep(0.5)
    ws = websocket.create_connection(page["webSocketDebuggerUrl"], timeout=20,
                                     header=[f"Origin: http://localhost:{PORT}"])
    mid = [0]
    def cmd(m, p=None):
        mid[0]+=1; myid=mid[0]
        ws.send(json.dumps({"id":myid,"method":m,"params":p or {}}))
        while True:
            r=json.loads(ws.recv())
            if r.get("id")==myid: return r
    cmd("Runtime.enable")
    time.sleep(7)  # 렌더 대기
    # 스크롤로 더 로드
    for _ in range(3):
        cmd("Runtime.evaluate", {"expression":"window.scrollBy(0,2000)"})
        time.sleep(2)
    r = cmd("Runtime.evaluate", {"expression":EXTRACT_JS, "returnByValue":True})
    ws.close()
    return r["result"]["result"].get("value","")

def main():
    refresh_session()
    proc = launch_chrome()
    try:
        raw = cdp_extract()
    finally:
        subprocess.run(["pkill", "-f", f"remote-debugging-port={PORT}"], capture_output=True)

    existing = set()
    if os.path.exists(FEED):
        for line in open(FEED):
            try: existing.add(json.loads(line).get("link") or json.loads(line).get("text","")[:80])
            except: pass

    new = 0
    with open(FEED, "a") as f:
        for item in raw.split("###"):
            parts = item.split("@@@")
            if len(parts) < 2: continue
            dt, text, link = (parts+["",""])[:3]
            key = link or text[:80]
            if not text.strip() or key in existing: continue
            existing.add(key)
            f.write(json.dumps({
                "datetime": dt, "text": text.strip(), "link": link,
                "collected_at": datetime.now(timezone.utc).isoformat()
            }, ensure_ascii=False)+"\n")
            new += 1
    print(f"{new}개 신규 수집")

if __name__ == "__main__":
    main()