micromelon-website/scripts/fetch-schools.ts

/**
 * Fetches active AU school organisations from Airtable,
 * geocodes their addresses via Nominatim, and writes
 * an anonymous lat/lng JSON file for the homepage map.
 *
 * Usage:
 *   AIRTABLE_API_KEY=pat... AIRTABLE_BASE_ID=app... npx tsx scripts/fetch-schools.ts
 */

import { writeFileSync, mkdirSync } from "fs";
import { join } from "path";

const AIRTABLE_API_KEY = process.env.AIRTABLE_API_KEY;
const AIRTABLE_BASE_ID = process.env.AIRTABLE_BASE_ID;
const TABLE_NAME = "Organisation";

if (!AIRTABLE_API_KEY || !AIRTABLE_BASE_ID) {
  console.error("Missing AIRTABLE_API_KEY or AIRTABLE_BASE_ID env vars");
  process.exit(1);
}

interface AirtableRecord {
  id: string;
  fields: {
    "Google Formatted Address"?: string;
    State?: string;
  };
}

interface AirtableResponse {
  records: AirtableRecord[];
  offset?: string;
}

interface SchoolPin {
  lat: number;
  lng: number;
}

async function fetchAllRecords(): Promise<AirtableRecord[]> {
  const allRecords: AirtableRecord[] = [];
  let offset: string | undefined;

  const filterFormula = "{Organisation Type}='Customer'";
  const fields = ["Google Formatted Address", "State"];

  do {
    const params = new URLSearchParams({
      filterByFormula: filterFormula,
    });
    fields.forEach((f) => params.append("fields[]", f));
    if (offset) params.set("offset", offset);

    const url = `https://api.airtable.com/v0/${AIRTABLE_BASE_ID}/${encodeURIComponent(TABLE_NAME)}?${params}`;

    const res = await fetch(url, {
      headers: { Authorization: `Bearer ${AIRTABLE_API_KEY}` },
    });

    if (!res.ok) {
      const body = await res.text();
      throw new Error(`Airtable API error ${res.status}: ${body}`);
    }

    const data: AirtableResponse = await res.json();
    allRecords.push(...data.records);
    offset = data.offset;
  } while (offset);

  return allRecords;
}

function simplifyAddress(address: string): string[] {
  const variants: string[] = [];

  // Strip unit/lot prefixes like "4506/793" → "793"
  let simplified = address.replace(/\d+\/(\d+)/, "$1");
  // Replace "&" intersections with just the first street
  simplified = simplified.replace(/\s*&\s*[^,]+/, "");
  if (simplified !== address) variants.push(simplified);

  // Try just suburb + state + postcode (everything after the first comma)
  const afterFirstComma = address.replace(/^[^,]+,\s*/, "");
  if (afterFirstComma !== address) variants.push(afterFirstComma);

  return variants;
}

async function geocodeSingle(query: string): Promise<{ lat: number; lng: number } | null> {
  const params = new URLSearchParams({
    q: query,
    format: "json",
    limit: "1",
    countrycodes: "au",
  });

  const res = await fetch(`https://nominatim.openstreetmap.org/search?${params}`, {
    headers: { "User-Agent": "micromelon-website-build/1.0" },
  });

  if (!res.ok) return null;

  const data = await res.json();
  if (!data.length) return null;

  return { lat: parseFloat(data[0].lat), lng: parseFloat(data[0].lon) };
}

async function geocode(address: string): Promise<{ lat: number; lng: number } | null> {
  // Try the full address first
  const result = await geocodeSingle(address);
  if (result) return result;

  // Try simplified variants
  for (const variant of simplifyAddress(address)) {
    await new Promise((r) => setTimeout(r, 1100));
    const fallback = await geocodeSingle(variant);
    if (fallback) return fallback;
  }

  return null;
}

async function main() {
  console.log("Fetching records from Airtable...");
  const records = await fetchAllRecords();
  console.log(`Found ${records.length} active AU school records`);

  const pins: SchoolPin[] = [];
  let failed = 0;

  for (const record of records) {
    const address = record.fields["Google Formatted Address"];
    if (!address) {
      failed++;
      continue;
    }

    // Rate-limit: Nominatim asks for max 1 req/sec
    await new Promise((r) => setTimeout(r, 1100));

    const coords = await geocode(address);
    if (coords) {
      pins.push(coords);
      process.stdout.write(`  Geocoded ${pins.length}/${records.length}\r`);
    } else {
      console.warn(`  Failed to geocode: ${address}`);
      failed++;
    }
  }

  console.log(`\nGeocoded ${pins.length} schools (${failed} failed)`);

  const outDir = join(__dirname, "..", "src", "data");
  mkdirSync(outDir, { recursive: true });
  const outPath = join(outDir, "schools.json");
  writeFileSync(outPath, JSON.stringify(pins, null, 2) + "\n");
  console.log(`Wrote ${outPath}`);
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});