Initial commit: Screen Leads app
Screen-capture lead tool: capture agent, Claude vision extractor with pluggable site recipes (LinkedIn), canonical funnel model, SQLite storage, FastAPI backend, dashboard, and setup/run scripts. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
"""Capture controller: a background state machine driven by the dashboard.
|
||||
|
||||
States: IDLE -> RUNNING <-> PAUSED -> STOPPED. The loop detects the front tab,
|
||||
stops (or waits) on a non-target page per config, and for each new profile URL
|
||||
runs scroll -> screenshot -> extract -> normalise -> save.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
|
||||
import config
|
||||
from agent import detector, screenshot, scroller
|
||||
from ai import extractor, recipes
|
||||
from core.models import Lead
|
||||
from db import database
|
||||
|
||||
|
||||
class State(str, Enum):
|
||||
IDLE = "IDLE"
|
||||
RUNNING = "RUNNING"
|
||||
PAUSED = "PAUSED"
|
||||
STOPPED = "STOPPED"
|
||||
|
||||
|
||||
def _recipe_by_id(site_id: str):
|
||||
for r in recipes.REGISTRY:
|
||||
if r.site_id == site_id:
|
||||
return r
|
||||
return None
|
||||
|
||||
|
||||
class CaptureController:
|
||||
def __init__(self) -> None:
|
||||
self._state = State.IDLE
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._stop = threading.Event()
|
||||
self._pause = threading.Event()
|
||||
self._lock = threading.Lock()
|
||||
self.reason = ""
|
||||
self.last_event = ""
|
||||
self.captured_urls: set[str] = set()
|
||||
self.leads_captured = 0
|
||||
self._run_id: Optional[int] = None
|
||||
|
||||
# --- introspection -------------------------------------------------------
|
||||
def status(self) -> dict[str, Any]:
|
||||
return {
|
||||
"state": self._state.value,
|
||||
"reason": self.reason,
|
||||
"last_event": self.last_event,
|
||||
"leads_captured": self.leads_captured,
|
||||
"captured_urls": len(self.captured_urls),
|
||||
"on_invalid_page": config.ON_INVALID_PAGE,
|
||||
}
|
||||
|
||||
def _set(self, state: State, event: str = "") -> None:
|
||||
self._state = state
|
||||
if event:
|
||||
self.last_event = event
|
||||
|
||||
# --- controls ------------------------------------------------------------
|
||||
def start(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
if self._state in (State.RUNNING, State.PAUSED):
|
||||
return self.status()
|
||||
self._stop.clear()
|
||||
self._pause.clear()
|
||||
self.reason = ""
|
||||
self.leads_captured = 0
|
||||
self.captured_urls.clear()
|
||||
self._run_id = database.start_run()
|
||||
self._set(State.RUNNING, "started")
|
||||
self._thread = threading.Thread(target=self._loop, daemon=True)
|
||||
self._thread.start()
|
||||
return self.status()
|
||||
|
||||
def pause(self) -> dict[str, Any]:
|
||||
if self._state == State.RUNNING:
|
||||
self._pause.set()
|
||||
self._set(State.PAUSED, "paused")
|
||||
return self.status()
|
||||
|
||||
def resume(self) -> dict[str, Any]:
|
||||
if self._state == State.PAUSED:
|
||||
self._pause.clear()
|
||||
self._set(State.RUNNING, "resumed")
|
||||
return self.status()
|
||||
|
||||
def stop(self) -> dict[str, Any]:
|
||||
if self._state in (State.RUNNING, State.PAUSED):
|
||||
self._stop.set()
|
||||
self._pause.clear()
|
||||
self.reason = self.reason or "stopped by user"
|
||||
return self.status()
|
||||
|
||||
# --- main loop -----------------------------------------------------------
|
||||
def _finish(self, status: str, reason: str) -> None:
|
||||
self.reason = reason
|
||||
self._set(State.STOPPED, reason)
|
||||
if self._run_id is not None:
|
||||
database.finish_run(self._run_id, status, reason, self.leads_captured)
|
||||
self._run_id = None
|
||||
|
||||
def _countdown(self) -> None:
|
||||
"""Grace period so the user can focus the browser before capture."""
|
||||
remaining = int(round(config.START_DELAY))
|
||||
while remaining > 0 and not self._stop.is_set():
|
||||
self.last_event = f"focus your browser — starting in {remaining}s"
|
||||
time.sleep(1)
|
||||
remaining -= 1
|
||||
|
||||
def _loop(self) -> None:
|
||||
try:
|
||||
self._countdown()
|
||||
while not self._stop.is_set():
|
||||
if self._pause.is_set():
|
||||
time.sleep(0.3)
|
||||
continue
|
||||
|
||||
det = detector.detect()
|
||||
|
||||
if not det.ok:
|
||||
if config.ON_INVALID_PAGE == "wait":
|
||||
self.last_event = f"waiting: {det.reason}"
|
||||
time.sleep(config.WATCH_INTERVAL)
|
||||
continue
|
||||
self._finish("stopped", det.reason)
|
||||
return
|
||||
|
||||
url = det.url or ""
|
||||
if url and url in self.captured_urls:
|
||||
self.last_event = "waiting for a new profile"
|
||||
time.sleep(config.WATCH_INTERVAL)
|
||||
continue
|
||||
|
||||
recipe = _recipe_by_id(det.recipe_id or "")
|
||||
if recipe is None:
|
||||
self._finish("error", f"no recipe for {det.recipe_id}")
|
||||
return
|
||||
|
||||
self.last_event = f"capturing {url or det.recipe_id}"
|
||||
lead = self._capture(recipe, url)
|
||||
if lead is not None:
|
||||
self.leads_captured += 1
|
||||
if url:
|
||||
self.captured_urls.add(url)
|
||||
self.last_event = f"saved: {lead.full_name or url}"
|
||||
|
||||
time.sleep(config.WATCH_INTERVAL)
|
||||
|
||||
self._finish("stopped", self.reason or "stopped by user")
|
||||
except Exception as e: # keep the loop crash-visible on the dashboard
|
||||
self._finish("error", f"{e}\n{traceback.format_exc(limit=2)}")
|
||||
|
||||
def _capture(self, recipe, url: str) -> Optional[Lead]:
|
||||
# scroll + screenshot the full page
|
||||
try:
|
||||
scroller.scroll_to_top()
|
||||
except Exception as e:
|
||||
self.last_event = f"scroll unavailable: {e}"
|
||||
shots: list[str] = []
|
||||
for i in range(config.SCROLL_STEPS):
|
||||
if self._stop.is_set():
|
||||
break
|
||||
shots.append(screenshot.capture(prefix=f"{recipe.site_id}"))
|
||||
try:
|
||||
scroller.scroll_down()
|
||||
except Exception:
|
||||
break
|
||||
if not shots:
|
||||
return None
|
||||
|
||||
raw = extractor.extract_lead(recipe, shots)
|
||||
lead = recipe.to_canonical(raw)
|
||||
lead.source_url = url
|
||||
lead.screenshot_refs = shots
|
||||
database.upsert_lead(lead)
|
||||
return lead
|
||||
|
||||
|
||||
# module-level singleton shared by the API
|
||||
controller = CaptureController()
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
"""Canonical, site-agnostic data structures.
|
||||
|
||||
Every site recipe normalises its raw extraction into `Lead`, so leads from any
|
||||
source share one funnel table and one dashboard view.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
class FunnelStage(str, Enum):
|
||||
NEW = "NEW" # name/headline only
|
||||
ENRICHED = "ENRICHED" # company / title / location / about captured
|
||||
CONTACT_FOUND = "CONTACT_FOUND" # email / phone / website captured
|
||||
EXPORTED = "EXPORTED" # pushed downstream (set by the user)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Lead:
|
||||
# identity
|
||||
full_name: str = ""
|
||||
first_name: str = ""
|
||||
last_name: str = ""
|
||||
headline: str = ""
|
||||
title: str = ""
|
||||
# organisation
|
||||
company: str = ""
|
||||
company_url: str = ""
|
||||
industry: str = ""
|
||||
location: str = ""
|
||||
about: str = ""
|
||||
# contact (often empty on the first pass)
|
||||
email: str = ""
|
||||
phone: str = ""
|
||||
website: str = ""
|
||||
social_handles: dict[str, str] = field(default_factory=dict)
|
||||
# provenance
|
||||
source_site: str = ""
|
||||
source_url: str = ""
|
||||
captured_at: float = field(default_factory=time.time)
|
||||
updated_at: float = field(default_factory=time.time)
|
||||
screenshot_refs: list[str] = field(default_factory=list)
|
||||
# quality
|
||||
confidence: float = 0.0
|
||||
fields_found: list[str] = field(default_factory=list)
|
||||
needs_review: bool = False
|
||||
# funnel
|
||||
stage: str = FunnelStage.NEW.value
|
||||
|
||||
def compute_stage(self) -> str:
|
||||
"""Advance the funnel stage from data completeness (never regress EXPORTED)."""
|
||||
if self.stage == FunnelStage.EXPORTED.value:
|
||||
return self.stage
|
||||
if self.email or self.phone or self.website or self.social_handles:
|
||||
return FunnelStage.CONTACT_FOUND.value
|
||||
if self.company or self.title or self.location or self.about:
|
||||
return FunnelStage.ENRICHED.value
|
||||
return FunnelStage.NEW.value
|
||||
|
||||
def finalize(self) -> "Lead":
|
||||
"""Derive first/last name, fields_found, and funnel stage."""
|
||||
if self.full_name and not (self.first_name or self.last_name):
|
||||
parts = self.full_name.split()
|
||||
if parts:
|
||||
self.first_name = parts[0]
|
||||
self.last_name = " ".join(parts[1:])
|
||||
self.fields_found = [
|
||||
k for k, v in asdict(self).items()
|
||||
if k not in ("fields_found", "stage", "needs_review", "captured_at",
|
||||
"updated_at", "confidence") and v
|
||||
]
|
||||
self.stage = self.compute_stage()
|
||||
return self
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PageMatch:
|
||||
"""Result of a recipe testing a page context."""
|
||||
site_id: str
|
||||
page_type: str # e.g. "profile", "search", "feed", "other"
|
||||
is_target: bool # True only for a page we should capture
|
||||
|
||||
|
||||
@dataclass
|
||||
class PageContext:
|
||||
"""What the detector knows about the current front tab."""
|
||||
url: Optional[str] = None
|
||||
ai_site: Optional[str] = None # site guessed visually by the AI
|
||||
ai_page_type: Optional[str] = None # page type guessed visually
|
||||
ai_url_text: Optional[str] = None # address-bar text read from a screenshot
|
||||
|
||||
@property
|
||||
def best_url(self) -> str:
|
||||
return (self.url or self.ai_url_text or "").strip()
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectionResult:
|
||||
ok: bool # a valid, enabled target page is in front
|
||||
reason: str = "" # human-readable explanation (shown on stop)
|
||||
recipe_id: Optional[str] = None
|
||||
page_type: Optional[str] = None
|
||||
url: str = ""
|
||||
method: str = "" # "url" | "visual"
|
||||
|
||||
|
||||
def dumps(obj: Any) -> str:
|
||||
return json.dumps(obj, ensure_ascii=False)
|
||||
Reference in New Issue
Block a user