// Run + Settings modals — the "scraper console" extensions on top of the // design viewer. They talk to the local Python server (serve.py): // POST /api/run -> kicks a scrape, streams log lines // GET /api/run/log -> poll for log + status (for browsers without ReadableStream) // GET /api/config -> current config.yaml as JSON // POST /api/config -> patch + write back to disk // All endpoints are local-only (127.0.0.1) and require no auth. const { useState: _uS, useEffect: _uE, useRef: _uR, useCallback: _uC } = React; // Hard-coded only as a last-resort fallback if /api/scrape-options is // unreachable. The live list comes from Dubizzle's own Algolia facets. const FALLBACK_EMIRATES = [ { id: 2, name: "Dubai" }, { id: 3, name: "Abu Dhabi" }, { id: 12, name: "Sharjah" }, { id: 14, name: "Ajman" }, { id: 11, name: "Ras al Khaimah" }, { id: 15, name: "Umm al Quwain" }, { id: 13, name: "Fujairah" }, { id: 39, name: "Al Ain" }, ].map((e) => ({ ...e, count: null })); const FALLBACK_CATEGORIES = [ { name: "Property for Rent", count: null }, { name: "Property for Sale", count: null }, ]; const BACKENDS = [ { v: "csv", label: "CSV file (zero setup)" }, { v: "xlsx", label: "Excel file (xlsx)" }, { v: "gsheet", label: "Google Sheet (live, needs service account)" }, ]; // -------------------------------------------------------------- // RUN MODAL // -------------------------------------------------------------- function RunModal({ onClose, onDone, onStart }) { const [running, setRunning] = _uS(false); const [done, setDone] = _uS(false); const [lines, setLines] = _uS([]); const [stats, setStats] = _uS(null); const [err, setErr] = _uS(null); const logRef = _uR(null); const ranRef = _uR(false); const statsRef = _uR(null); const append = _uC((s) => { setLines((prev) => prev.concat(s.split("\n").filter(Boolean))); }, []); _uE(() => { logRef.current && (logRef.current.scrollTop = logRef.current.scrollHeight); }, [lines]); // start automatically on open _uE(() => { if (ranRef.current) return; ranRef.current = true; (async () => { setRunning(true); setErr(null); setLines([]); setStats(null); onStart && onStart(); try { const res = await fetch("/api/run", { method: "POST" }); if (!res.ok) throw new Error(`HTTP ${res.status}`); if (res.body && res.body.getReader) { // stream const reader = res.body.getReader(); const dec = new TextDecoder(); let buf = ""; while (true) { const { value, done: d } = await reader.read(); if (d) break; buf += dec.decode(value, { stream: true }); const parts = buf.split("\n"); buf = parts.pop(); for (const p of parts) handleLine(p); } if (buf) handleLine(buf); } else { const txt = await res.text(); for (const p of txt.split("\n")) handleLine(p); } } catch (e) { setErr(String(e.message || e)); } finally { setRunning(false); setDone(true); } })(); function handleLine(p) { if (!p) return; // server emits JSON sentinels on their own line; everything else = log text if (p.startsWith("__STATS__ ")) { try { const s = JSON.parse(p.slice(10)); statsRef.current = s; setStats(s); } catch {} return; } if (p.startsWith("__ERROR__ ")) { setErr(p.slice(10)); return; } append(p); } }, [append]); const closeAndRefresh = () => { onDone && onDone(statsRef.current); onClose(); }; return (
e.stopPropagation()}>
SCRAPE {running ? "in progress" : (err ? "failed" : done ? "complete" : "starting…")}
{running ? ( <> Pulling listings, reconciling sheet… ) : err ? ( Run failed. ) : stats ? ( Done. fetched {stats.fetched} · new {stats.new} {stats.price_changes > 0 && <> · price changes {stats.price_changes}} {stats.delisted > 0 && <> · delisted {stats.delisted}} . ) : ( Done. )}
{lines.join("\n")}
{err &&
{err}
}
); } // -------------------------------------------------------------- // SETTINGS MODAL // -------------------------------------------------------------- function InlineSettings({ onSaved, flushRef }) { const [cfg, setCfg] = _uS(null); const [opts, setOpts] = _uS({ emirates: FALLBACK_EMIRATES, categories: FALLBACK_CATEGORIES, loading: true, fellBack: false, }); const [err, setErr] = _uS(null); const [saving, setSaving] = _uS(false); const [savedAt, setSavedAt] = _uS(null); const [dirty, setDirty] = _uS(false); // refs for auto-save const cfgRef = _uR(null); const lastSavedJsonRef = _uR(null); const debounceRef = _uR(null); const inflightRef = _uR(null); const mountedRef = _uR(true); _uE(() => { (async () => { try { const r = await fetch("/api/config"); if (!r.ok) throw new Error(`HTTP ${r.status}`); const j = await r.json(); setCfg(j); cfgRef.current = j; lastSavedJsonRef.current = JSON.stringify(stripDerived(j)); } catch (e) { setErr(String(e.message || e)); } })(); (async () => { try { const r = await fetch("/api/scrape-options"); if (!r.ok) throw new Error(`HTTP ${r.status}`); const j = await r.json(); const emirates = (j.emirates && j.emirates.length) ? j.emirates : FALLBACK_EMIRATES; const categories = (j.categories && j.categories.length) ? j.categories : FALLBACK_CATEGORIES; setOpts({ emirates, categories, loading: false, fellBack: false }); } catch (e) { setOpts((p) => ({ ...p, loading: false, fellBack: true })); } })(); return () => { mountedRef.current = false; // flush on unmount so navigating away preserves changes if (debounceRef.current) { clearTimeout(debounceRef.current); if (cfgRef.current && JSON.stringify(stripDerived(cfgRef.current)) !== lastSavedJsonRef.current) { doSave(cfgRef.current, /*silent*/ true); } } }; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); async function doSave(payload, silent = false) { const body = JSON.stringify(stripDerived(payload)); inflightRef.current = body; if (!silent) setSaving(true); setErr(null); try { const r = await fetch("/api/config", { method: "POST", headers: { "Content-Type": "application/json" }, body, }); if (!r.ok) throw new Error(`HTTP ${r.status}: ${await r.text()}`); lastSavedJsonRef.current = body; if (mountedRef.current) { setSavedAt(new Date()); setDirty(false); onSaved && onSaved(); } } catch (e) { if (mountedRef.current) setErr(String(e.message || e)); } finally { if (mountedRef.current) setSaving(false); inflightRef.current = null; } } const scheduleSave = (nextCfg) => { cfgRef.current = nextCfg; setDirty(JSON.stringify(stripDerived(nextCfg)) !== lastSavedJsonRef.current); if (debounceRef.current) clearTimeout(debounceRef.current); debounceRef.current = setTimeout(() => doSave(nextCfg), 350); }; // public: parent calls flushRef.current() before Run to force pending save if (flushRef) { flushRef.current = async () => { if (debounceRef.current) { clearTimeout(debounceRef.current); debounceRef.current = null; } if (cfgRef.current && JSON.stringify(stripDerived(cfgRef.current)) !== lastSavedJsonRef.current) { await doSave(cfgRef.current); } }; } const patch = (path, value) => { setCfg((c) => { const next = JSON.parse(JSON.stringify(c)); const segs = path.split("."); let o = next; for (let i = 0; i < segs.length - 1; i++) { o[segs[i]] = o[segs[i]] || {}; o = o[segs[i]]; } o[segs[segs.length - 1]] = value; scheduleSave(next); return next; }); }; const save = () => { if (debounceRef.current) { clearTimeout(debounceRef.current); debounceRef.current = null; } if (cfgRef.current) doSave(cfgRef.current); }; if (err && !cfg) { return
{err}
; } if (!cfg) { return
Loading…
; } // current numeric filters -> friendly inputs const minSize = parseNF(cfg.search.numeric_filters, "size>=") ?? 3000; const maxPrice = parseNF(cfg.search.numeric_filters, "price<=") ?? 250000; const setNF = (newMinSize, newMaxPrice) => { patch("search.numeric_filters", [ `size>=${Math.max(0, +newMinSize || 0)}`, `price<=${Math.max(1, +newMaxPrice || 1)}`, ]); }; return (

Scrape criteria {opts.loading && · loading live options…} {opts.fellBack && · live options unreachable, using fallback}

{/* If current site_id isn't in the live list, surface it so it stays selectable. */}
setNF(e.target.value, maxPrice)}/>
setNF(minSize, e.target.value)}/>
{cfg._derived && (
Files will land in {cfg._derived.csv_path.replace(/\.csv$/, "")}.{"{csv,xlsx,json}"} (slug derived from criteria above).
)}

Output

{cfg.sheets.backend === "gsheet" && ( <>
patch("sheets.spreadsheet_id", e.target.value)}/>
patch("sheets.worksheet", e.target.value)}/>
patch("sheets.credentials_file", e.target.value)}/>
)} {cfg.sheets.backend === "csv" && cfg._derived && (
patch("sheets.csv_path", e.target.value || null)}/>
)} {cfg.sheets.backend === "xlsx" && cfg._derived && (
patch("sheets.xlsx_path", e.target.value || null)}/>
)}
{err &&
{err}
} {savedAt && !err && (
Saved to config.yaml at {savedAt.toLocaleTimeString()}. Click Run scrape to apply.
)}
); } function SaveStatus({ dirty, saving, err, savedAt }) { let cls = "settings__statuspill"; let label; if (err) { cls += " is-err"; label = "Save failed"; } else if (saving) { cls += " is-saving"; label = "Saving…"; } else if (dirty) { cls += " is-dirty"; label = "Unsaved — auto-saving"; } else if (savedAt) { cls += " is-ok"; label = `Saved · ${savedAt.toLocaleTimeString([], {hour:"2-digit", minute:"2-digit"})}`; } else { cls += " is-idle"; label = "Up to date"; } return {label}; } function stripDerived(cfg) { if (!cfg || typeof cfg !== "object") return cfg; const { _derived, ...rest } = cfg; return rest; } function parseNF(list, prefix) { for (const s of list || []) { const m = (s || "").replace(/\s+/g, "").match(new RegExp("^" + prefix + "(\\d+)$")); if (m) return +m[1]; } return null; } Object.assign(window, { RunModal, InlineSettings });