Build a care-home directory

Save as query.mjs, then run node query.mjs. Download the JavaScript example.

// Node.js 20+. No packages required. Optional key: DATASCOOP_API_KEY.
const endpoint = "https://datascoop.io/v1/datasets/cqc/query";
const dataset = "cqc";
const filters = {
  "careHome": "true",
  "localAuthorityExact": "Surrey",
  "minBeds": "40"
};
const PAGE_SIZE = 25, MAX_ROWS = 2500, MAX_PAGES = 100;
const headers = { Accept: "application/json" };
const key = (process.env.DATASCOOP_API_KEY || "").trim();
if (key) headers.Authorization = "Bearer " + key;

async function retrieve() {
  const rows = [];
  let total = null, importedAt = null, first = null;
  for (let page = 0; page < MAX_PAGES; page++) {
    const url = new URL(endpoint);
    url.search = new URLSearchParams({ ...filters, strict_filters: "1",
      limit: String(PAGE_SIZE), offset: String(rows.length) });
    const response = await fetch(url, { headers, signal: AbortSignal.timeout(30000) });
    if (!response.ok) throw new Error("Query failed: HTTP " + response.status +
      ". Check filters and access; retry later for rate limits or server errors.");
    const result = await response.json();
    if (result.dataset !== dataset || !Array.isArray(result.rows) ||
        result.rows.some(row => !row || typeof row !== "object" || Array.isArray(row)) ||
        !Array.isArray(result.ignored) || result.ignored.length ||
        !Number.isSafeInteger(result.total) || result.total < 0 ||
        result.offset !== rows.length || result.returned !== result.rows.length ||
        result.returned > PAGE_SIZE || typeof result.truncated !== "boolean") {
      throw new Error("Unexpected query response; no partial result was printed.");
    }
    if (result.total > MAX_ROWS) throw new Error("More than " + MAX_ROWS +
      " rows match. Narrow the filters or review the bound before increasing it.");
    for (const [name, value] of Object.entries(filters)) {
      if (String(result.applied_filters?.[name]) !== String(value))
        throw new Error("A requested filter was not applied: " + name);
    }
    const imported = result.provenance?.imported_at;
    if (typeof imported !== "string" || !Number.isFinite(Date.parse(imported)))
      throw new Error("Import date is missing; no partial result was printed.");
    if (first === null) { first = result; total = result.total; importedAt = imported; }
    if (result.total !== total || imported !== importedAt)
      throw new Error("The import or match count changed. Run the request again.");
    const next = rows.length + result.returned;
    if (next > total || result.truncated !== (next < total) ||
        (result.returned === 0 && next < total))
      throw new Error("Pagination stopped making sense; no partial result was printed.");
    rows.push(...result.rows);
    if (rows.length === total) return { dataset, filters, total, rows,
      provenance: first.provenance, licence: first.licence, attribution: first.attribution };
  }
  throw new Error("Page limit reached. Narrow the filters; no partial result was printed.");
}

try {
  const result = await retrieve();
  if (result.total === 0) console.error("No matching records. Check the selected area and filters.");
  // Keep provenance and attribution with cached or republished data.
  console.log(JSON.stringify(result, null, 2));
} catch (error) {
  console.error(error instanceof Error ? error.message : "Request failed.");
  process.exitCode = 1;
}

Save as query.py, then run python3 query.py. Download the Python example.

# Python 3. No packages required. Optional key: DATASCOOP_API_KEY.
import json
import os
import sys
from datetime import datetime
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen

ENDPOINT = "https://datascoop.io/v1/datasets/cqc/query"
DATASET = "cqc"
FILTERS = json.loads("{\"careHome\":\"true\",\"localAuthorityExact\":\"Surrey\",\"minBeds\":\"40\"}")
PAGE_SIZE, MAX_ROWS, MAX_PAGES = 25, 2500, 100
HEADERS = {"Accept": "application/json"}
key = os.environ.get("DATASCOOP_API_KEY", "").strip()
if key:
    HEADERS["Authorization"] = "Bearer " + key

def text_value(value):
    return "true" if value is True else "false" if value is False else str(value)

def retrieve():
    rows, total, imported_at, first = [], None, None, None
    for _ in range(MAX_PAGES):
        params = dict(FILTERS, strict_filters="1", limit=PAGE_SIZE, offset=len(rows))
        request = Request(ENDPOINT + "?" + urlencode(params), headers=HEADERS)
        with urlopen(request, timeout=30) as response:
            result = json.load(response)
        if (not isinstance(result, dict) or result.get("dataset") != DATASET or
            not isinstance(result.get("rows"), list) or
            any(not isinstance(row, dict) for row in result["rows"]) or
            not isinstance(result.get("ignored"), list) or result["ignored"] or
            type(result.get("total")) is not int or result["total"] < 0 or
            type(result.get("offset")) is not int or result["offset"] != len(rows) or
            type(result.get("returned")) is not int or
            result.get("returned") != len(result["rows"]) or
            result["returned"] > PAGE_SIZE or type(result.get("truncated")) is not bool):
            raise ValueError("Unexpected query response; no partial result was printed.")
        if result["total"] > MAX_ROWS:
            raise ValueError("More than 2500 rows match. Narrow filters or review the bound.")
        applied = result.get("applied_filters")
        if not isinstance(applied, dict):
            raise ValueError("Applied filters are missing.")
        for name, value in FILTERS.items():
            if name not in applied or text_value(applied[name]) != text_value(value):
                raise ValueError("A requested filter was not applied: " + name)
        provenance = result.get("provenance")
        imported = provenance.get("imported_at") if isinstance(provenance, dict) else None
        if not isinstance(imported, str):
            raise ValueError("Import date is missing; no partial result was printed.")
        datetime.fromisoformat(imported.replace("Z", "+00:00"))
        if first is None:
            first, total, imported_at = result, result["total"], imported
        if result["total"] != total or imported != imported_at:
            raise ValueError("The import or match count changed. Run the request again.")
        next_offset = len(rows) + result["returned"]
        if (next_offset > total or result["truncated"] != (next_offset < total) or
            (result["returned"] == 0 and next_offset < total)):
            raise ValueError("Pagination stopped making sense; no partial result was printed.")
        rows.extend(result["rows"])
        if len(rows) == total:
            return dict(dataset=DATASET, filters=FILTERS, total=total, rows=rows,
                        provenance=first["provenance"], licence=first.get("licence"),
                        attribution=first.get("attribution"))
    raise ValueError("Page limit reached. Narrow filters; no partial result was printed.")

try:
    result = retrieve()
    if result["total"] == 0:
        print("No matching records. Check the selected area and filters.", file=sys.stderr)
    # Keep provenance and attribution with cached or republished data.
    print(json.dumps(result, indent=2))
except HTTPError as error:
    print("Query failed: HTTP " + str(error.code) + ". Check filters and access; retry later for rate limits or server errors.", file=sys.stderr)
    sys.exit(1)
except (URLError, TimeoutError, ValueError, TypeError, KeyError) as error:
    print(str(error), file=sys.stderr)
    sys.exit(1)

// Node.js 20+. No packages. Set DATASCOOP_API_KEY privately in your environment.
// Save and check need active annual/Pro access. Read and review remain available after expiry.
// This program never marks a comparison reviewed while saving, reading or checking it.
const BASE = "https://datascoop.io";
const FILTERS = { careHome: true, localAuthorityExact: "Surrey", minBeds: 40 };
const PAGE_SIZE = 100, MAX_CHANGES = 10000, MAX_BYTES = 32 * 1024 * 1024;
const MAX_POLLS = 30, MAX_WAIT_MS = 90000, REQUEST_TIMEOUT_MS = 10000;
const HELP = `Usage:
  node territory.mjs save REQUEST_ID [LABEL]
  node territory.mjs read TERRITORY_ID
  node territory.mjs check TERRITORY_ID
  node territory.mjs review TERRITORY_ID DISPLAYED_REVISION --confirm-reviewed

Use a different REQUEST_ID for each new selection; retry the same save with the same ID and label.
Save this script's returned territory ID for later checks. Edit FILTERS before saving a different selection.
Read waits briefly for existing work; check explicitly requests the latest available import.
Inspect every change before explicitly running review with the revision from that output.
No scheduled checks, emails, payment requests or automatic review are performed.`;
class RecipeError extends Error {}
const fail = message => { throw new RecipeError(message); };
const object = value => Boolean(value) && typeof value === "object" && !Array.isArray(value);
const integer = (value, max = Number.MAX_SAFE_INTEGER) => Number.isSafeInteger(value) && value >= 0 && value <= max;
const date = value => typeof value === "string" && Number.isFinite(Date.parse(value));
const idValid = value => typeof value === "string" && /^ter_[a-f0-9]{32}$/.test(value);
const record = value => object(value) && Object.values(value).every(v => typeof v === "string");
const args = process.argv.slice(2), command = args[0];
let headers, deadline, responseBytes = 0;

function validateMeta(meta) {
  return object(meta) && typeof meta.import_version === "string" && meta.import_version.length > 0 &&
    date(meta.imported_at) && date(meta.captured_at) && integer(meta.total, 5000) && meta.schema_version === 1 &&
    typeof meta.attribution === "string" && typeof meta.licence === "string" && object(meta.provenance) &&
    meta.provenance.imported_at === meta.imported_at && Array.isArray(meta.provenance.source_urls) &&
    meta.provenance.source_urls.every(url => typeof url === "string");
}
function validateDetail(data, id, offset, limit) {
  if (!object(data) || data.id !== id || !integer(data.revision) || typeof data.label !== "string" ||
      !object(data.filters) || !["pending", "ready", "failed"].includes(data.status)) fail("Invalid territory response; no comparison was printed.");
  if (data.status === "failed") fail("The capture failed. Saved work remains on your account; retry check when the source or service is available.");
  if (data.status === "pending") return false;
  const change = data.changes;
  if (data.error !== null || !validateMeta(data.baseline) || !validateMeta(data.latest) || !object(change) ||
      !object(change.counts) || !["entered", "left", "changed"].every(kind => integer(change.counts[kind], MAX_CHANGES)) ||
      !integer(change.total, MAX_CHANGES) || change.total !== change.counts.entered + change.counts.left + change.counts.changed ||
      change.offset !== offset || change.limit !== limit || !Array.isArray(change.items) ||
      change.items.length !== Math.min(limit, Math.max(0, change.total - offset)) ||
      data.latest.total - data.baseline.total !== change.counts.entered - change.counts.left) {
    fail("Incomplete or invalid comparison; no partial result was printed.");
  }
  for (const item of change.items) {
    if (!object(item) || typeof item.locationId !== "string" || !item.locationId || typeof item.name !== "string" ||
        !["entered", "left", "changed"].includes(item.kind) || !Array.isArray(item.fields) ||
        !(item.before === null || record(item.before)) || !(item.after === null || record(item.after)) ||
        (item.before && item.before.locationId !== item.locationId) || (item.after && item.after.locationId !== item.locationId) ||
        item.fields.some(field => !object(field) || !["field", "label", "before", "after"].every(key => typeof field[key] === "string")) ||
        item.kind === "entered" && (item.before !== null || item.after === null || item.fields.length !== 0) ||
        item.kind === "left" && (item.before === null || item.after !== null || item.fields.length !== 0) ||
        item.kind === "changed" && (item.before === null || item.after === null || item.fields.length === 0)) {
      fail("Invalid changed record; no partial result was printed.");
    }
  }
  return true;
}
function identity(data) {
  // Last-checked/current-source timestamps can advance without changing this retained comparison.
  return JSON.stringify([data.id, data.label, data.filters, data.revision, data.baseline, data.latest, data.changes.counts, data.changes.total]);
}
async function request(path, method = "GET", body) {
  const remaining = deadline - Date.now();
  if (remaining <= 0) fail("Time budget reached. Run read with the saved territory ID to resume; saved work is retained.");
  const response = await fetch(new URL("/v1/care-territories" + path, BASE), {
    method, headers, ...(body === undefined ? {} : { body: JSON.stringify(body) }), redirect: "error",
    signal: AbortSignal.timeout(Math.min(REQUEST_TIMEOUT_MS, remaining)),
  });
  if (![200, 202].includes(response.status)) {
    const explanation = {
      400: "Check the command and supported filters.", 401: "Check the key in your private environment.",
      402: "Active annual or Pro access is required for save/check.", 404: "The territory is unavailable to this key, or the feature is disabled.",
      409: "The revision, save request ID or account limit conflicts. Read the current territory; never automatically accept a newer revision.",
      422: "Narrow the selection; the source or retained-state limits rejected it.",
      429: "Wait before trying again.", 503: "The source or capture is unavailable; saved comparisons are retained.",
    }[response.status] || "Try again later; keep the same request ID when retrying a save.";
    await response.body?.cancel();
    fail("Request failed: HTTP " + response.status + ". " + explanation);
  }
  if (!response.body) fail("Missing response body; no comparison was printed.");
  const reader = response.body.getReader(), decoder = new TextDecoder("utf-8", { fatal: true });
  let text = "";
  while (true) {
    const { done, value } = await reader.read();
    if (done) break;
    responseBytes += value.byteLength;
    if (responseBytes > MAX_BYTES) { await reader.cancel(); fail("Response budget exceeded; no partial result was printed."); }
    text += decoder.decode(value, { stream: true });
  }
  text += decoder.decode();
  let data;
  try { data = JSON.parse(text); } catch { fail("Invalid JSON response; no comparison was printed."); }
  return data;
}
async function ready(id) {
  for (let attempt = 0; attempt < MAX_POLLS; attempt++) {
    const data = await request("/" + id + "?kind=all&offset=0&limit=" + PAGE_SIZE);
    if (validateDetail(data, id, 0, PAGE_SIZE)) return data;
    if (attempt + 1 < MAX_POLLS && Date.now() + 2000 < deadline) await new Promise(resolve => setTimeout(resolve, 2000));
    else break;
  }
  fail("Capture is still pending. Run read with this territory ID later; repeating read does not start a capture.");
}
async function completeComparison(id, expectedFilters) {
  const first = await ready(id), signature = identity(first), items = [], seen = new Set();
  if (expectedFilters && (Object.keys(first.filters).length !== Object.keys(expectedFilters).length ||
      Object.entries(expectedFilters).some(([name, value]) => String(first.filters[name]) !== String(value)))) {
    fail("The saved selection differs from the requested filters. Inspect it in your account; no comparison was printed.");
  }
  const counts = { entered: 0, left: 0, changed: 0 };
  let data = first;
  while (true) {
    for (const item of data.changes.items) {
      if (seen.has(item.locationId)) fail("Duplicate location across change pages; no partial result was printed.");
      seen.add(item.locationId); counts[item.kind]++; items.push(item);
    }
    if (items.length === first.changes.total) break;
    data = await request("/" + id + "?kind=all&offset=" + items.length + "&limit=" + PAGE_SIZE);
    if (!validateDetail(data, id, items.length, PAGE_SIZE) || identity(data) !== signature) {
      fail("The comparison changed while reading pages. Run read again; no partial result was printed.");
    }
  }
  if (["entered", "left", "changed"].some(kind => counts[kind] !== first.changes.counts[kind])) fail("Change totals do not match the records; no partial result was printed.");
  const final = await request("/" + id + "?kind=all&offset=0&limit=" + PAGE_SIZE);
  if (!validateDetail(final, id, 0, PAGE_SIZE) || identity(final) !== signature) fail("The comparison changed before completion. Run read again; no partial result was printed.");
  return {
    territory: { id, label: first.label, filters: first.filters, revision: first.revision },
    baseline: first.baseline, latest: first.latest, changes: { counts, total: items.length, items },
    review_command: "node territory.mjs review " + id + " " + first.revision + " --confirm-reviewed",
    note: "History starts at first capture. Import dates are not record update dates. Entered/left means selection membership, not proof of opening/closure. Review only after inspecting the complete comparison.",
  };
}
async function main() {
  if (command === "--help" && args.length === 1) { console.log(HELP); return; }
  if (command === "save") {
    if (args.length < 2 || args.length > 3 || !/^[A-Za-z0-9_-]{1,80}$/.test(args[1]) ||
        args[2] !== undefined && (!args[2].trim() || args[2].trim().length > 120)) fail(HELP);
  } else if (command === "read" || command === "check") {
    if (args.length !== 2 || !idValid(args[1])) fail(HELP);
  } else if (command === "review") {
    if (args.length !== 4 || !idValid(args[1]) || !/^\d+$/.test(args[2]) || !integer(Number(args[2])) || args[3] !== "--confirm-reviewed") fail(HELP);
  } else fail(HELP);
  const endpoint = new URL(BASE);
  if (endpoint.username || endpoint.password || endpoint.search || endpoint.hash || endpoint.pathname !== "/" ||
      !(endpoint.protocol === "https:" || endpoint.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(endpoint.hostname))) fail("Use an HTTPS service origin, or loopback HTTP for local testing.");
  const key = (process.env.DATASCOOP_API_KEY || "").trim();
  if (!key) fail("Set DATASCOOP_API_KEY privately in your environment. Do not put a key in commands, prompts or URLs.");
  headers = { Accept: "application/json", "Content-Type": "application/json", Authorization: "Bearer " + key };
  deadline = Date.now() + MAX_WAIT_MS;
  let id = args[1];
  if (command === "save") {
    const created = await request("", "POST", { label: (args[2] || "Surrey care homes · 40+ beds").trim(), filters: FILTERS, requestId: args[1] });
    if (!idValid(created?.id)) fail("Save response was incomplete. Retry the same request ID and label; do not create a new request ID for a retry.");
    id = created.id;
    console.error("Saved territory ID: " + id + ". Keep it for read/check. Retry a lost save response with the same request ID and label.");
  } else if (command === "check") {
    const checked = await request("/" + id + "/check", "POST", {});
    if (checked?.id !== id) fail("Check response was incomplete. Read the saved territory before retrying.");
  } else if (command === "review") {
    const revision = Number(args[2]), reviewed = await request("/" + id + "/review", "POST", { revision });
    if (!validateDetail(reviewed, id, 0, 50) || ![revision, revision + 1].includes(reviewed.revision) ||
        reviewed.baseline.import_version !== reviewed.latest.import_version || reviewed.changes.total !== 0) {
      fail("Review response was incomplete or changed. Read the saved territory before retrying; review is never retried automatically.");
    }
    console.log(JSON.stringify({ reviewed_territory: id, accepted_revision: revision, revision: reviewed.revision, baseline: reviewed.baseline }, null, 2));
    return;
  }
  console.log(JSON.stringify(await completeComparison(id, command === "save" ? FILTERS : undefined), null, 2));
}
try { await main(); }
catch (error) {
  console.error(error instanceof RecipeError ? error.message : "The request could not finish safely. No partial comparison was printed. Read saved work before retrying; reuse the same request ID and label for a lost save response.");
  process.exitCode = 1;
}