Inventree

2025-01-16 07:12:50 +00:00
parent ea94aadb13
commit 51cacff367
19 changed files with 1434 additions and 550 deletions
--- a/kibot-ci.yml
+++ b/kibot-ci.yml
@@ -8,16 +8,19 @@ workflow:
 variables:
  GIT_STRATEGY: clone
  GIT_SUBMODULE_STRATEGY: recursive
-  GIT_SUBMODULE_DEPTH: 1
  GIT_SUBMODULE_FORCE_HTTPS: "true"
+  # GIT_SUBMODULE_UPDATE_FLAGS: --remote --merge
+  PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/kicad"

 stages:
  - gen_mech
  - gen_fab
+  - inventree
+  - upload
  - release

 image:
-  name: ghcr.io/inti-cmnb/kicad7_auto:1.6.3
+  name: ghcr.io/inti-cmnb/kicad8_auto:1.8.2

 .main_rules:
  rules:
@@ -34,61 +37,84 @@ image:
  get_files:
    - FILES=$(find . -name *$SEARCH -not -path "./.gitlab/*")
  get_dirs:
-    - FILES=$(find . -name *$SEARCH -not -path "./.gitlab/*")
+    - !reference [.commands, get_files]  
    - | 
-          if [[ $FILES == "" ]]; then
-            DIRS=""
-          else
-            DIRS=$(dirname $FILES)
-          fi
+      if [[ $FILES == "" ]]; then
+        DIRS=""
+      else
+        DIRS=$(dirname $FILES)
+      fi
  sch_from_pro: 
    - 'SCHEMS=$(for f in $FILES ; do echo "${f%.*}.kicad_sch"; done)'
    - 'SCHEMS=$(for f in $SCHEMS ; do echo "${f##**/}"; done)'
+  get_dirs_filt:
+    - !reference [.commands, get_files]  
+    - | 
+      FILT_FILES=""
+      FILT_DIRS=""
+      for f in $FILES
+      do 
+        dir=$(dirname $f)
+        echo $dir
+        if [[ "$dir" =~ ^.?\/?[a-z]?[0-9]{6}-.*$ || "$dir" =~ .+_panel ]] 
+        then 
+          FILT_FILES=$(echo "$FILT_FILES $f")
+          FILT_DIRS=$(echo "$FILT_DIRS $dir")
+        fi
+      done
+      FILES=$FILT_FILES
+      DIRS=$FILT_DIRS
  dir_arr:
-    - !reference [.commands, get_dirs]  
-    - END=$(wc -w <<< $DIRS)
-    - dir_arr=($DIRS)
+    - !reference [.commands, get_dirs_filt]  
+    - |
+      END=$(echo $DIRS | wc -w)
+      dir_arr=($DIRS)
  kibot: 
    - 'SEARCH=".kicad_pro"'
    - !reference [.commands, dir_arr]  
    - !reference [.commands, sch_from_pro]  
    - sch_arr=($SCHEMS)
    - |
-          cd $CI_PROJECT_DIR
-          for i in $(seq 1 $END)
-            do
-              if [[ ${dir_arr[i-1]} == "./Frame" ]]; then
-                continue
-              fi
-              echo ${dir_arr[i-1]} 
-              echo ${sch_arr[i-1]} 
-              python3 $CI_PROJECT_DIR/.gitlab/.scripts/orig.py ${dir_arr[i-1]}
-              kibot -e ${dir_arr[i-1]}/${sch_arr[i-1]} -c $CI_PROJECT_DIR/.gitlab/default.kibot.yaml -d $CI_PROJECT_DIR/Fabrication/${dir_arr[i-1]} -s $SUFFIX
-            done
+      cd $CI_PROJECT_DIR
+      for i in $(seq 1 $END)
+        do
+          if [[ ${dir_arr[i-1]} == "./Frame" ]]; then
+            continue
+          fi
+          echo ${dir_arr[i-1]} 
+          echo ${sch_arr[i-1]} 
+          python3 $CI_PROJECT_DIR/.gitlab/.scripts/orig.py ${dir_arr[i-1]}
+          kibot -e ${dir_arr[i-1]}/${sch_arr[i-1]} -c $CI_PROJECT_DIR/.gitlab/default.kibot.yaml -d $CI_PROJECT_DIR/Fabrication/${dir_arr[i-1]} -s $SUFFIX
+          mv $CI_PROJECT_DIR/Fabrication/${dir_arr[i-1]}/*.zip Fabrication/ 2> /dev/null || true
+        done
    - cd $CI_PROJECT_DIR

  panel:
    - 'SEARCH="_panel.json"'
    - !reference [.commands, get_dirs]  
    - |
+          cd $CI_PROJECT_DIR
          for d in $DIRS
            do
+              echo "found panel"
              echo $d
-              cd $d
-              JSON=$(find *_panel.json)
-              NAME=$(echo "${JSON%.json}")
-              PCB=$(find *.kicad_pcb)
-              mkdir $CI_PROJECT_DIR/$NAME
-              kikit panelize -p $JSON $PCB $CI_PROJECT_DIR/$NAME/$NAME.kicad_pcb
-              touch $CI_PROJECT_DIR/$NAME/$NAME.kicad_sch
-              cp fp-lib-table $CI_PROJECT_DIR/$NAME/
-              python3 $CI_PROJECT_DIR/.gitlab/.scripts/post_panel.py $CI_PROJECT_DIR/$NAME/$NAME.kicad_pro $PCB
+              JSON=$(find $d/*_panel.json)
+              FILE=$(basename "${JSON}")
+              NAME=$(echo "${FILE%.json}")
+              PCB=$(find $d/*.kicad_pcb)
+              echo "mkdring"
+              mkdir $NAME
+              echo "panelising"
+              kikit panelize -p $JSON $PCB $NAME/$NAME.kicad_pcb
+              cp .gitlab/micromelon_default/micromelon_default.kicad_sch $NAME/$NAME.kicad_sch
+              cp $d/fp-lib-table $NAME/
+              python3 .gitlab/.scripts/post_panel.py $NAME/$NAME.kicad_pro $PCB
            done
    - cd $CI_PROJECT_DIR

  neo:
    - 'SEARCH=".kicad_pro"'
-    - !reference [.commands, get_dirs]  
+    - !reference [.commands, get_dirs_filt]  
    - |
          for d in $DIRS
            do
@@ -99,12 +125,109 @@ image:
              python3 $CI_PROJECT_DIR/.gitlab/.scripts/neo.py $d
            done

+  boms:
+    - client=$(echo $CI_PROJECT_PATH | cut -d'/' -f3 | sed -r 's/\<./\U&/g')
+    - proj=$(echo $CI_PROJECT_NAME | tr -d '0123456789' | tr '-' ' ' | sed -r 's/\<./\U&/g')
+    - desc_suffix=$(echo $client $proj)
+    - cd $CI_PROJECT_DIR/.gitlab/.scripts/ki-ntree/
+    - apt remove libcurl4 -y
+    - apt update && apt install curl pip -y && pip install --break-system-packages poetry
+    - poetry install
+    - url="https://gitlab.com/api/v4/projects/${DIGI_API_PRJ_ID}/packages/generic/digikey_api/0/token_storage.json"
+    - | 
+      if [[ -z "$KINTREE_DIGI_TOKEN" ]]; then
+        curl --header "JOB-TOKEN: $CI_JOB_TOKEN" $url >> token_storage.json
+      else
+          cp $KINTREE_DIGI_TOKEN token_storage.json
+      fi
+    - cat token_storage.json
+    - cd $CI_PROJECT_DIR/Fabrication
+    - 'SEARCH="bom.csv"'
+    - !reference [.commands, strip_tag]
+    - !reference [.commands, get_files]
+    - cd $CI_PROJECT_DIR/.gitlab/.scripts/ki-ntree/
+    - FAIL=0
+    - |
+      for f in $FILES
+        do
+          echo $f
+          name=$(echo $f | cut -c3- | cut -d'-' -f1)A
+          fab_path=$CI_PROJECT_DIR/Fabrication/$(dirname $f | cut -c3-)
+          name_n_rev=$(echo $f | cut -c3- | rev | cut -d'/' -f1 | cut -c9- | rev)
+          pcb_im=$fab_path/PCB_$name_n_rev.png
+          pcba_im=$fab_path/PCBA_$name_n_rev.png
+          pcb_attach=$(echo "['$fab_path/${name_n_rev}_PCB.pdf', '$CI_PROJECT_DIR/Fabrication/${name_n_rev}_JLC.zip']")
+          pcba_attach=$(echo "['$fab_path/${name_n_rev}_schematic.pdf', '$fab_path/${name_n_rev}_bom.csv', '$fab_path/${name_n_rev}-neo-pos_top.csv', '$fab_path/${name_n_rev}-neo-pos_bot.csv', '$fab_path/${name_n_rev}_ibom.html']")
+
+          DRY="--dry"
+          if [[ $CI_COMMIT_BRANCH == "main" ]]; then
+            DRY=""
+          fi
+          poetry run python -m kintree_cli $DRY -p $CI_PROJECT_DIR/Fabrication/$f --assembly "{'ipn': '$name', 'rev': '$TAG', 'image': ['$pcb_im', '$pcba_im'], 'desc': '$desc_suffix', 'attachments': [$pcb_attach, $pcba_attach]}" --settings $KINTREE_SETT --digi_token token_storage.json || FAIL=1
+        done
+    # - cp token_storage.json /tmp
+    - |
+      cat /tmp/token_storage.json
+      curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file /tmp/token_storage.json $url
+    - exit $FAIL 
+  # USE PRIVATE-TOKEN to upload from gitlab-ci
+
  git_tag: 
+    - !reference [.commands, strip_tag]  
+    - |
+      if [[ $GITLAB_CI == 'true' && $CI_COMMIT_BRANCH == "main" ]]; then 
+        echo "running git tag"
+        git tag $TAG
+      elif [[ $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" && "$CI_MERGE_REQUEST_TITLE" =~ [^a-zA-Z0-9.-_] ]]; then
+        exit 1
+      fi
+
+  merge_libs: 
    - |
      if [[ $GITLAB_CI == 'true' && $CI_COMMIT_BRANCH == "main" ]]; then
-        git tag $CI_COMMIT_MESSAGE
+        git config --global user.name "KicadCi"
+        git config --global user.email "andrew@micromelon.com.au"
+        cd $CI_PROJECT_DIR
+        for d in "libs/melonlib" "libs/melon3d" 
+          do
+            cd $d
+            if [ $d == "libs/melonlib" ] 
+            then
+              TOKEN=$KI_LIB_TOKEN 
+            else
+              TOKEN=$KI_LIB_3D_TOKEN 
+            fi
+            REPO=$(basename $(git remote get-url origin))
+            cd $CI_PROJECT_DIR
+            rm -rf $d
+            rm -rf .git/modules/$d
+            git submodule set-url $d https://ci_push:$TOKEN@gitlab.com/Micromelon/education/hardware/$REPO
+            git submodule update --init --remote --merge $d
+            cd $d
+            git remote set-url origin https://ci_push:$TOKEN@gitlab.com/Micromelon/education/hardware/$REPO
+            cd $CI_PROJECT_DIR
+          done
+        git submodule status
+        for d in "libs/melonlib" "libs/melon3d" 
+          do
+            cd $d
+            if [ $(git rev-parse HEAD) != $(git rev-parse main) ]
+            then
+              CURR_B=$(git config -f $CI_PROJECT_DIR/.gitmodules submodule.$d.branch)
+              echo CURR_B
+              echo $CURR_B
+              git checkout $CURR_B 
+              git commit --allow-empty -m "Merge"
+              git remote get-url origin
+              git push -u origin HEAD:$CURR_B -o ci.skip -o merge_request.create -o merge_request.remove_source_branch=false -o merge_request.merge_when_pipeline_succeeds -o merge_request.target=main
+            fi
+            cd $CI_PROJECT_DIR
+          done
      fi

+  strip_tag:
+    - TAG=$(echo ${CI_COMMIT_MESSAGE} | tr -d \\n)
+
 output_mech: 
  stage: gen_mech
  artifacts:
@@ -112,7 +235,7 @@ output_mech:
    paths:
      - Fabrication/**/*
  variables:
-    SUFF_MECH: run_erc,run_drc step
+    SUFF_MECH: run_erc,run_drc step pcb_render pcba_render
  script:
    - !reference [.commands, git_tag]
    - SUFFIX=$SUFF_MECH
@@ -129,9 +252,10 @@ outputs_dev:
      - Fabrication/**/*
    expire_in: 1 week
  variables:
-    SUFF_SCH: run_drc print_sch
+    SUFF_SCH: run_drc sch
    SUFF_PCB: run_erc,update_xml,set_text_variables print_pcb
  script:
+    - !reference [.commands, git_tag]
    - SUFFIX=$SUFF_SCH
    - !reference [.commands, kibot]
    - SUFFIX=$SUFF_PCB
@@ -152,12 +276,36 @@ outputs_all:
    - !reference [.commands, panel]
    - !reference [.commands, kibot]
    - !reference [.commands, neo]
+    - ls Fabrication/

-
-release_job:
-  stage: release
-  image: registry.gitlab.com/gitlab-org/release-cli:latest
+inventree_job_dev:
+  stage: inventree
  needs:
+    - job: output_mech
+      artifacts: true
+    - job: outputs_dev
+      artifacts: true
+  rules:
+    - !reference [.dev_rules, rules]
+  script:
+    - !reference [.commands, boms]
+
+inventree_job:
+  stage: inventree
+  needs:
+    - job: output_mech
+      artifacts: true
+    - job: outputs_all
+      artifacts: true
+  rules:
+    - !reference [.main_rules, rules]
+  script:
+    - !reference [.commands, boms]
+
+upload_job:
+  stage: upload
+  needs:
+    - job: inventree_job
    - job: outputs_all
      artifacts: true
  rules:
@@ -165,18 +313,58 @@ release_job:
      when: never                                  # Do not run this job when a tag is created manually
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH  # Run this job when commits are pushed or merged to the default branch
  script:
-    - echo "running release_job for $TAG"
+    - !reference [.commands, strip_tag]
+    - apt remove libcurl4 -y
+    - apt-get update && apt-get -y install zip curl
+    - |
+      zip -r Fabrication/All.zip Fabrication/
+      for d in $(find Fabrication/* -maxdepth 0 -type d)
+      do
+        zip=$d.zip
+        zip -r $zip $d   
+      done
+      for d in $(find Fabrication/ -maxdepth 1 -name '*.zip')
+      do 
+        b=$(basename $d)
+        f=$(echo "${b%.*}")
+        url=${PACKAGE_REGISTRY_URL}/$TAG/$f-$TAG.zip
+        echo "uploading: $d to $url"
+        curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file $d $url
+      done
  artifacts:
    when: always
-    name: "$CI_PROJECT_TITLE-$CI_COMMIT_MESSAGE"
    paths:
      - Fabrication/**/*
-    expire_in: never
-  release:                                         # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties
-    tag_name: '$CI_COMMIT_MESSAGE'            # The version is incremented per pipeline.
-    description: '$CI_COMMIT_MESSAGE'
-    ref: '$CI_COMMIT_SHA'                          # The tag is created from the pipeline SHA.
-    assets:
-      links:
-        - name: All Fab
-          url: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/download"
+    expire_in: 1 week
+
+release_job:
+  stage: release
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  needs:
+    - job: upload_job
+      artifacts: true
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: never                                  # Do not run this job when a tag is created manually
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH  # Run this job when commits are pushed or merged to the default branch
+  script:
+    - !reference [.commands, strip_tag]
+    - apk add jq curl
+    - |
+      echo "running release_job for $TAG"
+      echo "#!/bin/sh" >> fab.sh
+      packid=$(curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages | jq .[-1].id)
+      echo "release-cli create --name \"Release $TAG\" --tag-name \"$TAG\" \\" >> fab.sh
+      cnt=$(curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/$packid/package_files | jq length)
+      ids=$(curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/$packid/package_files | jq .[].id)
+      names=$(curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/$packid/package_files | jq .[].file_name)
+      i=1
+      while [ "$i" -le $cnt ]
+      do 
+        id=$(echo $ids | cut -d' ' -f $i)
+        name=$(echo $names | cut -d' ' -f $i)
+        echo "  --assets-link \"{\\\"name\\\":\\\"$name\\\",\\\"url\\\":\\\"${CI_PROJECT_URL}/-/package_files/$id/download\\\"}\" \\" >> fab.sh
+        i=$(( i + 1 ))
+      done
+      chmod +x fab.sh
+      ./fab.sh