// Run + Settings modals — the "scraper console" extensions on top of the
// design viewer. They talk to the local Python server (serve.py):
// POST /api/run -> kicks a scrape, streams log lines
// GET /api/run/log -> poll for log + status (for browsers without ReadableStream)
// GET /api/config -> current config.yaml as JSON
// POST /api/config -> patch + write back to disk
// All endpoints are local-only (127.0.0.1) and require no auth.
const { useState: _uS, useEffect: _uE, useRef: _uR, useCallback: _uC } = React;
// Hard-coded only as a last-resort fallback if /api/scrape-options is
// unreachable. The live list comes from Dubizzle's own Algolia facets.
const FALLBACK_EMIRATES = [
{ id: 2, name: "Dubai" },
{ id: 3, name: "Abu Dhabi" },
{ id: 12, name: "Sharjah" },
{ id: 14, name: "Ajman" },
{ id: 11, name: "Ras al Khaimah" },
{ id: 15, name: "Umm al Quwain" },
{ id: 13, name: "Fujairah" },
{ id: 39, name: "Al Ain" },
].map((e) => ({ ...e, count: null }));
const FALLBACK_CATEGORIES = [
{ name: "Property for Rent", count: null },
{ name: "Property for Sale", count: null },
];
const BACKENDS = [
{ v: "csv", label: "CSV file (zero setup)" },
{ v: "xlsx", label: "Excel file (xlsx)" },
{ v: "gsheet", label: "Google Sheet (live, needs service account)" },
];
// --------------------------------------------------------------
// RUN MODAL
// --------------------------------------------------------------
function RunModal({ onClose, onDone, onStart }) {
const [running, setRunning] = _uS(false);
const [done, setDone] = _uS(false);
const [lines, setLines] = _uS([]);
const [stats, setStats] = _uS(null);
const [err, setErr] = _uS(null);
const logRef = _uR(null);
const ranRef = _uR(false);
const statsRef = _uR(null);
const append = _uC((s) => {
setLines((prev) => prev.concat(s.split("\n").filter(Boolean)));
}, []);
_uE(() => {
logRef.current && (logRef.current.scrollTop = logRef.current.scrollHeight);
}, [lines]);
// start automatically on open
_uE(() => {
if (ranRef.current) return;
ranRef.current = true;
(async () => {
setRunning(true); setErr(null); setLines([]); setStats(null);
onStart && onStart();
try {
const res = await fetch("/api/run", { method: "POST" });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
if (res.body && res.body.getReader) {
// stream
const reader = res.body.getReader();
const dec = new TextDecoder();
let buf = "";
while (true) {
const { value, done: d } = await reader.read();
if (d) break;
buf += dec.decode(value, { stream: true });
const parts = buf.split("\n");
buf = parts.pop();
for (const p of parts) handleLine(p);
}
if (buf) handleLine(buf);
} else {
const txt = await res.text();
for (const p of txt.split("\n")) handleLine(p);
}
} catch (e) {
setErr(String(e.message || e));
} finally {
setRunning(false);
setDone(true);
}
})();
function handleLine(p) {
if (!p) return;
// server emits JSON sentinels on their own line; everything else = log text
if (p.startsWith("__STATS__ ")) {
try {
const s = JSON.parse(p.slice(10));
statsRef.current = s;
setStats(s);
} catch {}
return;
}
if (p.startsWith("__ERROR__ ")) {
setErr(p.slice(10));
return;
}
append(p);
}
}, [append]);
const closeAndRefresh = () => { onDone && onDone(statsRef.current); onClose(); };
return (
e.stopPropagation()}>
SCRAPE
{running ? "in progress" : (err ? "failed" : done ? "complete" : "starting…")}
{running ? (
<>
Pulling listings, reconciling sheet…
>
) : err ? (
Run failed.
) : stats ? (
Done. fetched {stats.fetched} · new {stats.new}
{stats.price_changes > 0 && <> · price changes {stats.price_changes}>}
{stats.delisted > 0 && <> · delisted {stats.delisted}>}
.
) : (
Done.
)}
{lines.join("\n")}
{err &&
{err}
}
);
}
// --------------------------------------------------------------
// SETTINGS MODAL
// --------------------------------------------------------------
function InlineSettings({ onSaved, flushRef }) {
const [cfg, setCfg] = _uS(null);
const [opts, setOpts] = _uS({
emirates: FALLBACK_EMIRATES,
categories: FALLBACK_CATEGORIES,
loading: true,
fellBack: false,
});
const [err, setErr] = _uS(null);
const [saving, setSaving] = _uS(false);
const [savedAt, setSavedAt] = _uS(null);
const [dirty, setDirty] = _uS(false);
// refs for auto-save
const cfgRef = _uR(null);
const lastSavedJsonRef = _uR(null);
const debounceRef = _uR(null);
const inflightRef = _uR(null);
const mountedRef = _uR(true);
_uE(() => {
(async () => {
try {
const r = await fetch("/api/config");
if (!r.ok) throw new Error(`HTTP ${r.status}`);
const j = await r.json();
setCfg(j);
cfgRef.current = j;
lastSavedJsonRef.current = JSON.stringify(stripDerived(j));
} catch (e) { setErr(String(e.message || e)); }
})();
(async () => {
try {
const r = await fetch("/api/scrape-options");
if (!r.ok) throw new Error(`HTTP ${r.status}`);
const j = await r.json();
const emirates = (j.emirates && j.emirates.length) ? j.emirates : FALLBACK_EMIRATES;
const categories = (j.categories && j.categories.length) ? j.categories : FALLBACK_CATEGORIES;
setOpts({ emirates, categories, loading: false, fellBack: false });
} catch (e) {
setOpts((p) => ({ ...p, loading: false, fellBack: true }));
}
})();
return () => {
mountedRef.current = false;
// flush on unmount so navigating away preserves changes
if (debounceRef.current) {
clearTimeout(debounceRef.current);
if (cfgRef.current && JSON.stringify(stripDerived(cfgRef.current)) !== lastSavedJsonRef.current) {
doSave(cfgRef.current, /*silent*/ true);
}
}
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
async function doSave(payload, silent = false) {
const body = JSON.stringify(stripDerived(payload));
inflightRef.current = body;
if (!silent) setSaving(true);
setErr(null);
try {
const r = await fetch("/api/config", {
method: "POST",
headers: { "Content-Type": "application/json" },
body,
});
if (!r.ok) throw new Error(`HTTP ${r.status}: ${await r.text()}`);
lastSavedJsonRef.current = body;
if (mountedRef.current) {
setSavedAt(new Date());
setDirty(false);
onSaved && onSaved();
}
} catch (e) {
if (mountedRef.current) setErr(String(e.message || e));
} finally {
if (mountedRef.current) setSaving(false);
inflightRef.current = null;
}
}
const scheduleSave = (nextCfg) => {
cfgRef.current = nextCfg;
setDirty(JSON.stringify(stripDerived(nextCfg)) !== lastSavedJsonRef.current);
if (debounceRef.current) clearTimeout(debounceRef.current);
debounceRef.current = setTimeout(() => doSave(nextCfg), 350);
};
// public: parent calls flushRef.current() before Run to force pending save
if (flushRef) {
flushRef.current = async () => {
if (debounceRef.current) { clearTimeout(debounceRef.current); debounceRef.current = null; }
if (cfgRef.current && JSON.stringify(stripDerived(cfgRef.current)) !== lastSavedJsonRef.current) {
await doSave(cfgRef.current);
}
};
}
const patch = (path, value) => {
setCfg((c) => {
const next = JSON.parse(JSON.stringify(c));
const segs = path.split(".");
let o = next;
for (let i = 0; i < segs.length - 1; i++) {
o[segs[i]] = o[segs[i]] || {};
o = o[segs[i]];
}
o[segs[segs.length - 1]] = value;
scheduleSave(next);
return next;
});
};
const save = () => {
if (debounceRef.current) { clearTimeout(debounceRef.current); debounceRef.current = null; }
if (cfgRef.current) doSave(cfgRef.current);
};
if (err && !cfg) {
return ;
}
if (!cfg) {
return Loading…
;
}
// current numeric filters -> friendly inputs
const minSize = parseNF(cfg.search.numeric_filters, "size>=") ?? 3000;
const maxPrice = parseNF(cfg.search.numeric_filters, "price<=") ?? 250000;
const setNF = (newMinSize, newMaxPrice) => {
patch("search.numeric_filters", [
`size>=${Math.max(0, +newMinSize || 0)}`,
`price<=${Math.max(1, +newMaxPrice || 1)}`,
]);
};
return (
Scrape criteria
{opts.loading && · loading live options…}
{opts.fellBack && · live options unreachable, using fallback}
{/* If current site_id isn't in the live list, surface it so it stays selectable. */}
setNF(e.target.value, maxPrice)}/>
setNF(minSize, e.target.value)}/>
{cfg._derived && (
Files will land in
{cfg._derived.csv_path.replace(/\.csv$/, "")}.{"{csv,xlsx,json}"}
(slug derived from criteria above).
)}
{err &&
{err}
}
{savedAt && !err && (
Saved to config.yaml at {savedAt.toLocaleTimeString()}.
Click Run scrape to apply.
)}
);
}
function SaveStatus({ dirty, saving, err, savedAt }) {
let cls = "settings__statuspill";
let label;
if (err) { cls += " is-err"; label = "Save failed"; }
else if (saving) { cls += " is-saving"; label = "Saving…"; }
else if (dirty) { cls += " is-dirty"; label = "Unsaved — auto-saving"; }
else if (savedAt) { cls += " is-ok"; label = `Saved · ${savedAt.toLocaleTimeString([], {hour:"2-digit", minute:"2-digit"})}`; }
else { cls += " is-idle"; label = "Up to date"; }
return {label};
}
function stripDerived(cfg) {
if (!cfg || typeof cfg !== "object") return cfg;
const { _derived, ...rest } = cfg;
return rest;
}
function parseNF(list, prefix) {
for (const s of list || []) {
const m = (s || "").replace(/\s+/g, "").match(new RegExp("^" + prefix + "(\\d+)$"));
if (m) return +m[1];
}
return null;
}
Object.assign(window, { RunModal, InlineSettings });