import subprocess, json, os, re, time, requests, websocket
from datetime import datetime, timezone

BASE = os.path.expanduser("~/Claude/Projects/에이전트구축")
XPROF = os.path.join(BASE, ".xchrome_profile")
FEED = os.path.join(BASE, "x_feed.jsonl")
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
PORT = 9333

EXTRACT_JS = r"""
(function(){
  var a=document.getElementsByTagName('article');var o=[];
  for(var i=0;i<a.length;i++){
    var tm=a[i].getElementsByTagName('time')[0];var link='';
    if(tm){var p=tm.parentNode;while(p&&p.tagName!='A'){p=p.parentNode}if(p)link=p.href}
    o.push((tm?tm.getAttribute('datetime'):'')+'@@@'+a[i].innerText.replace(/\n/g,' ')+'@@@'+link)
  }
  return o.join('###');
})()
"""

def refresh_session():
    src = os.path.expanduser("~/Library/Application Support/Google/Chrome/Default")
    dst = os.path.join(XPROF, "Default")
    os.makedirs(dst, exist_ok=True)
    for f in ["Cookies"]:
        subprocess.run(["cp", os.path.join(src, f), dst], capture_output=True)
    for d in ["Local Storage", "Network"]:
        subprocess.run(["cp", "-R", os.path.join(src, d), dst], capture_output=True)
    subprocess.run(["cp", os.path.expanduser("~/Library/Application Support/Google/Chrome/Local State"), XPROF], capture_output=True)

def launch_chrome():
    subprocess.run(["pkill", "-f", f"remote-debugging-port={PORT}"], capture_output=True)
    time.sleep(1)
    return subprocess.Popen([
        CHROME, "--headless=new", "--disable-gpu", f"--remote-debugging-port={PORT}",
        "--remote-allow-origins=*", f"--user-data-dir={XPROF}",
        "--no-first-run", "--no-default-browser-check", "https://x.com/home"
    ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

def cdp_extract():
    for _ in range(20):
        try:
            tabs = requests.get(f"http://localhost:{PORT}/json").json()
            page = next((t for t in tabs if t.get("type")=="page"), None)
            if page and page.get("webSocketDebuggerUrl"): break
        except Exception: pass
        time.sleep(0.5)
    ws = websocket.create_connection(page["webSocketDebuggerUrl"], timeout=20,
                                     header=[f"Origin: http://localhost:{PORT}"])
    mid = [0]
    def cmd(m, p=None, retries=2):
        for attempt in range(retries+1):
            try:
                mid[0]+=1; myid=mid[0]
                ws.send(json.dumps({"id":myid,"method":m,"params":p or {}}))
                while True:
                    r=json.loads(ws.recv())
                    if r.get("id")==myid: return r
            except (ConnectionResetError, websocket.WebSocketException, OSError) as e:
                if attempt>=retries: raise
                time.sleep(1.5)
    cmd("Runtime.enable")
    time.sleep(9)  # 렌더 대기 (넉넉히)
    # 스크롤은 잘게, 실패해도 무시하고 진행 (소켓 리셋에 죽지 않게)
    for _ in range(6):
        try:
            cmd("Runtime.evaluate", {"expression":"window.scrollBy(0,1000)"})
        except Exception:
            break  # 스크롤 실패해도 그때까지 로드된 걸로 추출 시도
        time.sleep(1.5)
    try:
        r = cmd("Runtime.evaluate", {"expression":EXTRACT_JS, "returnByValue":True})
        val = r["result"]["result"].get("value","")
    except Exception:
        val = ""
    try: ws.close()
    except Exception: pass
    return val

def main():
    refresh_session()
    proc = launch_chrome()
    try:
        raw = cdp_extract()
    finally:
        subprocess.run(["pkill", "-f", f"remote-debugging-port={PORT}"], capture_output=True)

    existing = set()
    if os.path.exists(FEED):
        for line in open(FEED):
            try: existing.add(json.loads(line).get("link") or json.loads(line).get("text","")[:80])
            except: pass

    new = 0
    with open(FEED, "a") as f:
        for item in raw.split("###"):
            parts = item.split("@@@")
            if len(parts) < 2: continue
            dt, text, link = (parts+["",""])[:3]
            key = link or text[:80]
            if not text.strip() or key in existing: continue
            existing.add(key)
            f.write(json.dumps({
                "datetime": dt, "text": text.strip(), "link": link,
                "collected_at": datetime.now(timezone.utc).isoformat()
            }, ensure_ascii=False)+"\n")
            new += 1
    print(f"{new}개 신규 수집")

if __name__ == "__main__":
    main()