Switch CI BOM upload from InvenTree (hfsntree) to Astable

Replaces the `inventree_dev` + `inventree_main` jobs with
`astable_dev` + `astable_main` calling `POST /api/v1/bom/upload`
on the Astable instance. The old hfsntree path is removed —
Astable is now the source of truth for BOMs, and owns the
supplier-API credentials (Digi-Key / Mouser / etc.) so the CI
runner doesn't need them any more.

Revision targeting matches the previous convention:
- dev branch: uploads to the DRAFT "dev" rev
- main branch: uploads to the rev named from commit message,
  promoted to ACTIVE
- MR-to-main: uploads to dev (matches old hfsntree behaviour)

Strict mode: any MPN that can't be resolved against an enabled
supplier integration aborts the whole upload — no orphan parts
get created when a CSV ships with typos.

Files uploaded alongside the BOM CSV (each optional, skipped
when absent so dev and main share the same template):

  gerbers, panel gerbers, STEP, schematic PDF, PCB layout PDF,
  iBOM HTML, BOM xlsx, PCBA + bare-PCB renders, DRC + ERC
  reports, CPL CSV.

Astable attaches each file to the PCBA part, the bare PCB part,
or both (per fab-file dispatch table). Files attached to both
parts dedupe storage via content-addressed paths.

Required CI variables (set group-level on
Micromelon/education/hardware/, masked + protected):

  ASTABLE_URL         e.g. https://astable.timhadwen.com
  ASTABLE_API_TOKEN   minted at /manage/api-tokens (ast_…)

scripts/upload-bom.sh vendored from the Astable repo
(scripts/ci/upload-bom.sh) — the .astable_base before_script
curls a copy from this repo's raw URL at job time so the include:
project: pattern stays self-contained.

Downstream `upload_packages` needs: updated from inventree_main
→ astable_main.

kibot-ci.yml

@@ -310,19 +310,50 @@ generate_panel:
       fi
 
 # =============================================================================
-# Stage: InvenTree
+# Stage: Astable (replaces the old InvenTree/hfsntree upload path)
 # =============================================================================
-.inventree_base:
+#
-  stage: inventree
+# Uploads the BOM + every fabrication output to Astable via
-  image: python:3.11-slim
+# POST /api/v1/bom/upload. Astable owns the supplier-API
-  before_script:
+# credentials, so the CI runner doesn't need Digi-Key / Mouser
-    - |
+# tokens any more. The upload is strict: any MPN that can't be
-      cd hfsntree
+# resolved against an enabled supplier integration aborts the
-      pip install -e . --quiet
+# upload and writes nothing.
-      cd $CI_PROJECT_DIR
+#
+# Revision-targeting follows the existing dev/main convention:
+#   - dev branch:  uploads to the part's DRAFT "dev" revision.
+#   - main branch: uploads to the rev named from the commit
+#                  message's "VX.Y" token, promoted to ACTIVE.
+#   - MR-to-main:  uploads to "dev" (matches old behaviour).
+#
+# Required CI variables (set group-level on
+# Micromelon/education/hardware/, masked + protected):
+#
+#   ASTABLE_URL         e.g. https://astable.timhadwen.com
+#   ASTABLE_API_TOKEN   minted at /manage/api-tokens (ast_…)
+#
+# The script vendored at scripts/upload-bom.sh in this repo handles
+# the POST + polling loop. PROJECT_NAME flows through as the
+# part-ipn — Astable splits on the first hyphen to derive the
+# PCBA IPN ("<base>A") + bare-PCB IPN ("<base>") + display name.
 
-inventree_dev:
+.astable_base:
-  extends: .inventree_base
+  stage: inventree            # keep stage name so existing `needs:`
+                              # downstream (upload_packages) still match
+  image: curlimages/curl:8.10.1
+  before_script:
+    - apk add --no-cache bash jq
+    # Fetch the upload script from the kicad-ci repo so the yml stays
+    # self-contained (GitLab's `include: project:` only ships the yml
+    # itself, not sibling files in the repo). The ref pins to kicad9
+    # — bump if you move the script.
+    - |
+      curl -fsSL -o /usr/local/bin/upload-bom.sh \
+        "https://gitlab.com/Micromelon/education/hardware/kicad-ci/-/raw/kicad9/scripts/upload-bom.sh"
+      chmod +x /usr/local/bin/upload-bom.sh
+
+astable_dev:
+  extends: .astable_base
   rules:
     - !reference [.dev_rules, rules]
   needs:
@@ -334,21 +365,36 @@ inventree_dev:
       artifacts: true
     - job: generate_3d
       artifacts: true
-    - job: generate_gerbers
-      artifacts: true
-    - job: generate_position
-      artifacts: true
-    - job: generate_panel
-      artifacts: true
-      optional: true
   script:
     - |
-      echo "Running InvenTree upload for dev branch (revision: dev)"
+      FAB="$CI_PROJECT_DIR/Fabrication"
-      cd hfsntree
+      SUB="$FAB/${PROJECT_NAME}"
-      python main.py batch $CI_PROJECT_DIR/Fabrication --version-override dev -y
+      VAL="$CI_PROJECT_DIR/Validation"
 
-inventree_main:
+      # Build args list — each --flag is optional; the script skips
-  extends: .inventree_base
+      # files that aren't present. Dev runs have fewer outputs than
+      # main (no gerbers/panel), so we just pass what exists.
+      ARGS=(
+        --file "$FAB/${PROJECT_NAME}_dev_bom.csv"
+        --part-ipn "$PROJECT_NAME"
+        --branch dev
+        --commit-sha "$CI_COMMIT_SHA"
+        --commit-message "$CI_COMMIT_MESSAGE"
+      )
+      [ -f "$FAB/${PROJECT_NAME}_dev_cpl.csv" ]            && ARGS+=(--cpl            "$FAB/${PROJECT_NAME}_dev_cpl.csv")
+      [ -f "$FAB/${PROJECT_NAME}_dev.step" ]               && ARGS+=(--step           "$FAB/${PROJECT_NAME}_dev.step")
+      [ -f "$FAB/${PROJECT_NAME}_dev_schematic.pdf" ]      && ARGS+=(--schematic-pdf  "$FAB/${PROJECT_NAME}_dev_schematic.pdf")
+      [ -f "$SUB/${PROJECT_NAME}_dev_ibom.html" ]          && ARGS+=(--ibom-html      "$SUB/${PROJECT_NAME}_dev_ibom.html")
+      [ -f "$SUB/${PROJECT_NAME}_dev_bom.xlsx" ]           && ARGS+=(--bom-xlsx       "$SUB/${PROJECT_NAME}_dev_bom.xlsx")
+      [ -f "$SUB/PCBA_${PROJECT_NAME}_dev.png" ]           && ARGS+=(--pcba-image     "$SUB/PCBA_${PROJECT_NAME}_dev.png")
+      [ -f "$SUB/PCB_${PROJECT_NAME}_dev.png" ]            && ARGS+=(--pcb-image      "$SUB/PCB_${PROJECT_NAME}_dev.png")
+      [ -f "$VAL/${PROJECT_NAME}-erc.html" ]               && ARGS+=(--erc-report     "$VAL/${PROJECT_NAME}-erc.html")
+      [ -f "$VAL/${PROJECT_NAME}-drc.html" ]               && ARGS+=(--drc-report     "$VAL/${PROJECT_NAME}-drc.html")
+
+      upload-bom.sh "${ARGS[@]}"
+
+astable_main:
+  extends: .astable_base
   rules:
     - !reference [.main_rules, rules]
   needs:
@@ -374,26 +420,42 @@ inventree_main:
     expire_in: 1 week
   script:
     - |
-      cd hfsntree
+      FAB="$CI_PROJECT_DIR/Fabrication"
+      SUB="$FAB/${PROJECT_NAME}"
+      VAL="$CI_PROJECT_DIR/Validation"
 
-      # MRs targeting main upload to "dev" revision for pre-ordering parts
+      # MR-to-main → upload to the dev rev (pre-ordering window).
-      # Actual commits to main upload to the versioned revision
+      # Actual main pushes → released rev named from the commit
+      # message. The BOM CSV's filename always uses ${VERSION} —
+      # extract_version sets it to "dev" for MRs and the V-token
+      # for direct pushes, so we can read the same path either way.
       if [[ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]]; then
-        echo "Running InvenTree upload for MR (revision: dev)"
+        BRANCH=dev
-        python main.py batch $CI_PROJECT_DIR/Fabrication --version-override dev -y
       else
-        echo "Running InvenTree upload for main branch (revision: $VERSION)"
+        BRANCH=main
-        python main.py batch $CI_PROJECT_DIR/Fabrication -y
-
-        # Generate Samsung P&P files only on actual release
-        echo "Generating Samsung pick-and-place files..."
-        python main.py samsung $CI_PROJECT_DIR/Fabrication
-
-        # Deactivate old revisions (keep only current release + dev)
-        echo "Deactivating old InvenTree revisions..."
-        python main.py deactivate-old $CI_PROJECT_DIR/Fabrication
       fi
-      cd $CI_PROJECT_DIR
+
+      ARGS=(
+        --file "$FAB/${PROJECT_NAME}_${VERSION}_bom.csv"
+        --part-ipn "$PROJECT_NAME"
+        --branch "$BRANCH"
+        --commit-sha "$CI_COMMIT_SHA"
+        --commit-message "$CI_COMMIT_MESSAGE"
+      )
+      [ -f "$FAB/${PROJECT_NAME}_${VERSION}_cpl.csv" ]            && ARGS+=(--cpl            "$FAB/${PROJECT_NAME}_${VERSION}_cpl.csv")
+      [ -f "$FAB/${PROJECT_NAME}_${VERSION}_JLC.zip" ]            && ARGS+=(--gerbers        "$FAB/${PROJECT_NAME}_${VERSION}_JLC.zip")
+      [ -f "$FAB/${PROJECT_NAME}_panel_${VERSION}_JLC.zip" ]      && ARGS+=(--panel-gerbers  "$FAB/${PROJECT_NAME}_panel_${VERSION}_JLC.zip")
+      [ -f "$FAB/${PROJECT_NAME}_${VERSION}.step" ]               && ARGS+=(--step           "$FAB/${PROJECT_NAME}_${VERSION}.step")
+      [ -f "$FAB/${PROJECT_NAME}_${VERSION}_schematic.pdf" ]      && ARGS+=(--schematic-pdf  "$FAB/${PROJECT_NAME}_${VERSION}_schematic.pdf")
+      [ -f "$SUB/${PROJECT_NAME}_${VERSION}_PCB.pdf" ]            && ARGS+=(--pcb-pdf        "$SUB/${PROJECT_NAME}_${VERSION}_PCB.pdf")
+      [ -f "$SUB/${PROJECT_NAME}_${VERSION}_ibom.html" ]          && ARGS+=(--ibom-html      "$SUB/${PROJECT_NAME}_${VERSION}_ibom.html")
+      [ -f "$SUB/${PROJECT_NAME}_${VERSION}_bom.xlsx" ]           && ARGS+=(--bom-xlsx       "$SUB/${PROJECT_NAME}_${VERSION}_bom.xlsx")
+      [ -f "$SUB/PCBA_${PROJECT_NAME}_${VERSION}.png" ]           && ARGS+=(--pcba-image     "$SUB/PCBA_${PROJECT_NAME}_${VERSION}.png")
+      [ -f "$SUB/PCB_${PROJECT_NAME}_${VERSION}.png" ]            && ARGS+=(--pcb-image      "$SUB/PCB_${PROJECT_NAME}_${VERSION}.png")
+      [ -f "$VAL/${PROJECT_NAME}-erc.html" ]                      && ARGS+=(--erc-report     "$VAL/${PROJECT_NAME}-erc.html")
+      [ -f "$VAL/${PROJECT_NAME}-drc.html" ]                      && ARGS+=(--drc-report     "$VAL/${PROJECT_NAME}-drc.html")
+
+      upload-bom.sh "${ARGS[@]}"
 
 # =============================================================================
 # Stage: Release
@@ -407,7 +469,7 @@ upload_packages:
   needs:
     - job: extract_version
       artifacts: true
-    - job: inventree_main
+    - job: astable_main
       artifacts: true
   script:
     - apt-get update && apt-get -y install zip

scripts/upload-bom.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+# Upload a BOM to Astable from CI. POSTs the multipart form to
+# /api/v1/bom/upload, then polls the returned jobId until the worker
+# finishes (done/failed). Prints the result JSON on done; exits non-zero
+# on any failure (timeout, parse error, worker failure, …) so the CI
+# job fails visibly.
+#
+# Env vars (required):
+#   ASTABLE_URL         — base URL (e.g. https://astable.timhadwen.com)
+#   ASTABLE_API_TOKEN   — bearer token minted at /manage/api-tokens
+#
+# Args:
+#   --file <path>            — BOM CSV to upload (required)
+#   --part-ipn <ipn>         — InvPart.ipn of the assembly (required)
+#   --branch <dev|main>      — drives revision targeting (required)
+#   --commit-sha <sha>       — for audit (optional)
+#   --commit-message <msg>   — drives "main" revision label (optional)
+#   --poll-interval <sec>    — default 3
+#   --timeout <sec>          — default 300 (5 min)
+#
+# Designed to be runnable under gitlab-ci-local — just set ASTABLE_URL
+# to your dev URL (e.g. http://host.docker.internal:3000 if Astable is
+# on the host) and gitlab-ci-local will pick it up from .env or the
+# `--variable` flag.
+
+set -euo pipefail
+
+POLL_INTERVAL=3
+TIMEOUT=300
+FILE=""
+CPL=""
+GERBERS=""
+PANEL_GERBERS=""
+STEP=""
+PCB_PDF=""
+SCHEMATIC_PDF=""
+IBOM_HTML=""
+BOM_XLSX=""
+PCBA_IMAGE=""
+PCB_IMAGE=""
+DRC_REPORT=""
+ERC_REPORT=""
+PART_IPN=""
+BRANCH=""
+COMMIT_SHA=""
+COMMIT_MESSAGE=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --file)            FILE="$2"; shift 2 ;;
+    --cpl)             CPL="$2"; shift 2 ;;
+    --gerbers)         GERBERS="$2"; shift 2 ;;
+    --panel-gerbers)   PANEL_GERBERS="$2"; shift 2 ;;
+    --step)            STEP="$2"; shift 2 ;;
+    --pcb-pdf)         PCB_PDF="$2"; shift 2 ;;
+    --schematic-pdf)   SCHEMATIC_PDF="$2"; shift 2 ;;
+    --ibom-html)       IBOM_HTML="$2"; shift 2 ;;
+    --bom-xlsx)        BOM_XLSX="$2"; shift 2 ;;
+    --pcba-image)      PCBA_IMAGE="$2"; shift 2 ;;
+    --pcb-image)       PCB_IMAGE="$2"; shift 2 ;;
+    --drc-report)      DRC_REPORT="$2"; shift 2 ;;
+    --erc-report)      ERC_REPORT="$2"; shift 2 ;;
+    --part-ipn)        PART_IPN="$2"; shift 2 ;;
+    --branch)          BRANCH="$2"; shift 2 ;;
+    --commit-sha)      COMMIT_SHA="$2"; shift 2 ;;
+    --commit-message)  COMMIT_MESSAGE="$2"; shift 2 ;;
+    --poll-interval)   POLL_INTERVAL="$2"; shift 2 ;;
+    --timeout)         TIMEOUT="$2"; shift 2 ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+: "${ASTABLE_URL:?ASTABLE_URL must be set}"
+: "${ASTABLE_API_TOKEN:?ASTABLE_API_TOKEN must be set}"
+: "${FILE:?--file is required}"
+: "${PART_IPN:?--part-ipn is required}"
+: "${BRANCH:?--branch is required}"
+[[ -f "$FILE" ]] || { echo "BOM file not found: $FILE" >&2; exit 2; }
+[[ -z "$CPL" || -f "$CPL" ]] || { echo "CPL file not found: $CPL" >&2; exit 2; }
+[[ "$BRANCH" == "dev" || "$BRANCH" == "main" ]] || { echo "--branch must be dev or main" >&2; exit 2; }
+
+echo "[bom-upload] POST $ASTABLE_URL/api/v1/bom/upload"
+echo "[bom-upload]   partIpn=$PART_IPN branch=$BRANCH file=$FILE${CPL:+ cpl=$CPL}"
+
+CURL_ARGS=(
+  -fsS -X POST
+  -H "authorization: Bearer $ASTABLE_API_TOKEN"
+  -F "file=@$FILE"
+  -F "partIpn=$PART_IPN"
+  -F "branch=$BRANCH"
+  -F "commitSha=$COMMIT_SHA"
+  -F "commitMessage=$COMMIT_MESSAGE"
+)
+[[ -n "$CPL"            ]] && CURL_ARGS+=(-F "cpl=@$CPL")
+[[ -n "$GERBERS"        ]] && CURL_ARGS+=(-F "gerbers=@$GERBERS")
+[[ -n "$PANEL_GERBERS"  ]] && CURL_ARGS+=(-F "panelGerbers=@$PANEL_GERBERS")
+[[ -n "$STEP"           ]] && CURL_ARGS+=(-F "step=@$STEP")
+[[ -n "$PCB_PDF"        ]] && CURL_ARGS+=(-F "pcbPdf=@$PCB_PDF")
+[[ -n "$SCHEMATIC_PDF"  ]] && CURL_ARGS+=(-F "schematicPdf=@$SCHEMATIC_PDF")
+[[ -n "$IBOM_HTML"      ]] && CURL_ARGS+=(-F "ibomHtml=@$IBOM_HTML")
+[[ -n "$BOM_XLSX"       ]] && CURL_ARGS+=(-F "bomXlsx=@$BOM_XLSX")
+[[ -n "$PCBA_IMAGE"     ]] && CURL_ARGS+=(-F "pcbaImage=@$PCBA_IMAGE")
+[[ -n "$PCB_IMAGE"      ]] && CURL_ARGS+=(-F "pcbImage=@$PCB_IMAGE")
+[[ -n "$DRC_REPORT"     ]] && CURL_ARGS+=(-F "drcReport=@$DRC_REPORT")
+[[ -n "$ERC_REPORT"     ]] && CURL_ARGS+=(-F "ercReport=@$ERC_REPORT")
+RESPONSE=$(curl "${CURL_ARGS[@]}" "$ASTABLE_URL/api/v1/bom/upload")
+
+JOB_ID=$(echo "$RESPONSE" | jq -r '.jobId // empty')
+POLL_URL=$(echo "$RESPONSE" | jq -r '.pollUrl // empty')
+LINE_COUNT=$(echo "$RESPONSE" | jq -r '.lineCount // 0')
+
+if [[ -z "$JOB_ID" || -z "$POLL_URL" ]]; then
+  echo "[bom-upload] enqueue failed: $RESPONSE" >&2
+  exit 1
+fi
+
+echo "[bom-upload] queued ($LINE_COUNT lines parsed) → job $JOB_ID"
+
+START=$(date +%s)
+while true; do
+  NOW=$(date +%s)
+  ELAPSED=$((NOW - START))
+  if (( ELAPSED > TIMEOUT )); then
+    echo "[bom-upload] timed out after ${TIMEOUT}s" >&2
+    exit 1
+  fi
+
+  STATE=$(curl -fsS \
+    -H "authorization: Bearer $ASTABLE_API_TOKEN" \
+    "$ASTABLE_URL$POLL_URL")
+  STATUS=$(echo "$STATE" | jq -r '.status')
+  STEP=$(echo "$STATE" | jq -r '.progress.message // empty')
+
+  case "$STATUS" in
+    done)
+      echo "[bom-upload] done"
+      echo "$STATE" | jq -r '.result | "  added=\(.added) updated=\(.updated) removed=\(.removed) missingPartsCreated=\(.missingPartsCreated) unresolvedSuppliers=\(.unresolvedSuppliers)"'
+      REV=$(echo "$STATE" | jq -r '.result.revision.label // "(none)"')
+      echo "  revision: $REV"
+      exit 0
+      ;;
+    failed)
+      ERROR=$(echo "$STATE" | jq -r '.error // "(no error message)"')
+      echo "[bom-upload] failed: $ERROR" >&2
+      exit 1
+      ;;
+    pending|running)
+      [[ -n "$STEP" ]] && echo "  · $STATUS — $STEP"
+      sleep "$POLL_INTERVAL"
+      ;;
+    *)
+      echo "[bom-upload] unexpected status: $STATUS" >&2
+      echo "$STATE" >&2
+      exit 1
+      ;;
+  esac
+done