build-deb-v4.yml

name: Package Builder Debian v4
run-name: Build ${{ inputs.stage }} Packages ${{ inputs.distro }} ${{ inputs.codename }} ${{ inputs.arch }}

on:
  workflow_dispatch:
    inputs:
      stage:
        description: "Stage to build"
        type: choice
        options:
          - all
          - experimental
          - unstable
          - testing
          - release-3_0
          - release-3_1
          - release-3_2
        required: true
        default: "unstable"
      distro:
        description: "Distro to build (debian, ubuntu)"
        type: choice
        options:
          - ""
          - debian
          - ubuntu
        required: false
        default: ""
      codename:
        description: "Codename to build (e.g. noble, bookworm)"
        type: string
        required: false
        default: ""
      arch:
        description: "Architecture to build (amd64, arm64)"
        type: choice
        options:
          - ""
          - amd64
          - arm64
        required: false
        default: ""
      only-package:
        description: "Only build package"
        type: string
        required: false
        default: ""
  workflow_call:
    inputs:
      stage:
        description: "Stage to build"
        type: string
        required: true
        default: "unstable"
      distro:
        description: "Distro to build (debian, ubuntu)"
        type: string
        required: false
      codename:
        description: "Codename to build (e.g. noble, bookworm)"
        type: string
        required: false
      arch:
        description: "Architecture to build (amd64, arm64)"
        type: string
        required: false

concurrency:
  group: debian_builder_v4
  cancel-in-progress: true

jobs:
  matrix-builder:
    runs-on: ubuntu-24.04
    outputs:
      stages: ${{ steps.calc-matrix.outputs.stages }}
      distros: ${{ steps.calc-matrix.outputs.distros }}
      codenames: ${{ steps.calc-matrix.outputs.codenames }}
      arches: ${{ steps.calc-matrix.outputs.arches }}
      suites: ${{ steps.calc-matrix.outputs.suites }}
      runners: ${{ steps.calc-matrix.outputs.runners }}
      includes: ${{ steps.calc-matrix.outputs.includes }}
      excludes: ${{ steps.calc-matrix.outputs.excludes }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Calculate Matrix
        id: calc-matrix
        run: |
          if [ -n "${{ inputs.stage }}" ] && [ "${{ inputs.stage }}" != "all" ]; then
            STAGES=(${{ inputs.stage }})
          else
            STAGES=(experimental testing unstable)
          fi
          DISTROS=()
          CODENAMES=()
          if [ -n "${{ inputs.arch }}" ]; then
            ARCHES=(${{ inputs.arch }})
          else
            ARCHES=(amd64 arm64)
          fi
          INCLUDES=()
          EXCLUDES=()

          valid_distro_codenames=()

          for stage in "${STAGES[@]}"; do
            if [ ! -d "stage/${stage}" ]; then
              echo "Package model for stage ${stage} not found!"
              continue
            fi

            for dir in $(find stage/${stage}/ -mindepth 2 -maxdepth 2 -type d | sort); do
              for arch in "${ARCHES[@]}"; do
                distro=$(echo $dir | cut -d/ -f3)
                if [ -n "${{ inputs.distro }}" ] && [ "${{ inputs.distro }}" != "$distro" ]; then
                  continue
                fi

                codename=$(echo $dir | cut -d/ -f4)
                if [ -n "${{ inputs.codename }}" ] && [ "${{ inputs.codename }}" != "$codename" ]; then
                  continue
                fi

                if [[ ! " ${DISTROS[*]} " =~ [[:space:]]${distro}[[:space:]] ]]; then
                  DISTROS+=("${distro}")
                fi
                if [[ ! " ${CODENAMES[*]} " =~ [[:space:]]${codename}[[:space:]] ]]; then
                  CODENAMES+=("${codename}")
                fi
                if [[ ! " ${valid_distro_codenames[*]} " =~ [[:space:]]${distro}-${codename}[[:space:]] ]]; then
                  valid_distro_codenames+=("${distro}-${codename}")
                fi
              done
            done
          done

          for distro in "${DISTROS[@]}"; do
            for codename in "${CODENAMES[@]}"; do
              if [[ ! " ${valid_distro_codenames[*]} " =~ [[:space:]]${distro}-${codename}[[:space:]] ]]; then
                EXCLUDES+=($(jq -n -c --arg distro "$distro" --arg codename "$codename" '$ARGS.named'))
              fi
            done
          done

          if [[ "${{ inputs.stage }}" == "release-"* ]]; then
            EXCLUDES+=($(jq -n -c --arg distro "debian" --arg codename "testing" '$ARGS.named'))
          fi

          SUITES=$(jq -n "$(jq -n -c \
            --argjson experimental "$(jq -n -c --arg suite "experimental" --arg component "main" '$ARGS.named')" \
            --argjson unstable "$(jq -n -c --arg suite "unstable" --arg component "main" '$ARGS.named')" \
            --argjson testing "$(jq -n -c --arg suite "testing" --arg component "main" '$ARGS.named')" \
            --argjson stable "$(jq -n -c --arg suite "stable" --arg component "main" '$ARGS.named')" \
            --argjson release-3_0 "$(jq -n -c --arg suite "stable" --arg component "v3.0" '$ARGS.named')" \
            --argjson release-3_1 "$(jq -n -c --arg suite "stable" --arg component "v3.1" '$ARGS.named')" \
            --argjson release-3_2 "$(jq -n -c --arg suite "stable" --arg component "v3.2" '$ARGS.named')" \
            '$ARGS.named'\
          )" '$ARGS.named')

          echo "stages=$(jq -n -c '$ARGS.positional' --args -- "${STAGES[@]}")" >> $GITHUB_OUTPUT
          echo "distros=$(jq -n -c '$ARGS.positional' --args -- "${DISTROS[@]}")" >> $GITHUB_OUTPUT
          echo "codenames=$(jq -n -c '$ARGS.positional' --args -- "${CODENAMES[@]}")" >> $GITHUB_OUTPUT
          echo "arches=$(jq -n -c '$ARGS.positional' --args -- "${ARCHES[@]}")" >> $GITHUB_OUTPUT
          echo "suites=$(jq -n -c "${SUITES}" '$ARGS.named')" >> $GITHUB_OUTPUT
          echo "runners=$(jq -n -c "$(jq -n -c --arg amd64 "ubuntu-24.04" --arg arm64 "ubuntu-24.04-arm" '$ARGS.named')" '$ARGS.named')" >> $GITHUB_OUTPUT
          echo "includes=$(jq -n -c "[$(printf '%s\n' "${INCLUDES[@]}" | paste -sd,)]" '$ARGS.named')" >> $GITHUB_OUTPUT
          echo "excludes=$(jq -n -c "[$(printf '%s\n' "${EXCLUDES[@]}" | paste -sd,)]" '$ARGS.named')" >> $GITHUB_OUTPUT

  # build packages and sources
  build:
    runs-on: ${{ fromJSON(needs.matrix-builder.outputs.runners)[matrix.arch] }}
    container: "ghcr.io/regolith-linux/ci-${{ matrix.distro }}:${{ matrix.codename }}-${{ matrix.arch }}"
    needs: matrix-builder
    strategy:
      fail-fast: false
      matrix:
        stage: ${{ fromJSON(needs.matrix-builder.outputs.stages) }}
        distro: ${{ fromJSON(needs.matrix-builder.outputs.distros) }}
        codename: ${{ fromJSON(needs.matrix-builder.outputs.codenames) }}
        arch: ${{ fromJSON(needs.matrix-builder.outputs.arches) }}

        include: ${{ fromJSON(needs.matrix-builder.outputs.includes) }}
        exclude: ${{ fromJSON(needs.matrix-builder.outputs.excludes) }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set Job Parameters
        id: init
        run: |
          echo "gh-repo-path=/__w/voulage/voulage" >> $GITHUB_OUTPUT
          echo "changelogs-path=/__w/voulage/voulage/changelogs" >> $GITHUB_OUTPUT
          echo "manifest-path=/__w/voulage/voulage/manifests" >> $GITHUB_OUTPUT
          echo "package-build-path=/__w/voulage/voulage/packages" >> $GITHUB_OUTPUT
          echo "package-publish-path=/__w/voulage/voulage/publish" >> $GITHUB_OUTPUT
          echo "stage=${{ matrix.stage }}" >> $GITHUB_OUTPUT
          echo "distro=${{ matrix.distro }}" >> $GITHUB_OUTPUT
          echo "codename=${{ matrix.codename }}" >> $GITHUB_OUTPUT
          echo "arch=${{ matrix.arch }}" >> $GITHUB_OUTPUT
          echo "suite=${{ fromJSON(needs.matrix-builder.outputs.suites)[matrix.stage]['suite'] }}" >> $GITHUB_OUTPUT
          echo "component=${{ fromJSON(needs.matrix-builder.outputs.suites)[matrix.stage]['component'] }}" >> $GITHUB_OUTPUT
          echo "target=${{ matrix.stage }}-${{ matrix.distro }}-${{ matrix.codename }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT

      - name: Environment Setup
        run: |
          set -e
          mkdir -p ${{ steps.init.outputs.changelogs-path }} || true
          mkdir -p ${{ steps.init.outputs.manifest-path }} || true

          sudo rm -rf /etc/apt/sources.list.d/regolith.list
          sudo apt update
          DEBIAN_FRONTEND=noninteractive sudo apt install -y --no-install-recommends jq git devscripts wget dput diffutils

      - name: Pull Manifest
        run: |
          set -e
          cp "${{ steps.init.outputs.manifest-path }}/${{ steps.init.outputs.distro }}/${{ steps.init.outputs.codename }}/${{ steps.init.outputs.suite }}-${{ steps.init.outputs.component }}/manifest.txt" ${{ steps.init.outputs.manifest-path }} || true
          echo "Current manifest:"
          cat ${{ steps.init.outputs.manifest-path }}/manifest.txt || true

      - name: Check for changes
        id: changes
        run: |
          set -e
          set -x

          CHANGE_OUTPUT=$(${{ steps.init.outputs.gh-repo-path }}/.github/scripts/main.sh \
            check \
            --extension ${{ steps.init.outputs.gh-repo-path }}/.github/scripts/ext-debian.sh \
            --git-repo-path ${{ steps.init.outputs.gh-repo-path }} \
            --manifest-path ${{ steps.init.outputs.manifest-path }} \
            --pkg-build-path ${{ steps.init.outputs.package-build-path }} \
            --pkg-publish-path ${{ steps.init.outputs.package-publish-path }} \
            --distro "${{ steps.init.outputs.distro }}" \
            --codename "${{ steps.init.outputs.codename }}" \
            --arch "${{ steps.init.outputs.arch }}" \
            --stage "${{ steps.init.outputs.stage }}" \
            --suite "${{ steps.init.outputs.suite }}" \
            --component "${{ steps.init.outputs.component }}" \
            --only-package "${{ inputs.only-package }}" | tail -n1)

          if [ "$CHANGE_OUTPUT" == "No package changes found, exiting." ]; then
            echo "changed=0" >> $GITHUB_OUTPUT
            echo "No package changes to build"
          else
            echo "changed=1" >> $GITHUB_OUTPUT
            echo "New Manifest: "
            cat ${{ steps.init.outputs.manifest-path }}/next-manifest.txt
          fi

      # - name: Setup SSH agent
      #   uses: webfactory/ssh-agent@v0.9.0
      #   if: steps.changes.outputs.changed == 1
      #   with:
      #     ssh-private-key: ${{ secrets.KAMATERA_SSH_KEY }}

      - name: Import GPG Key
        uses: regolith-linux/actions/import-gpg@main
        if: steps.changes.outputs.changed == 1
        with:
          gpg-key: "${{ secrets.PACKAGE_PRIVATE_KEY2 }}"

      - name: Build Packages
        if: steps.changes.outputs.changed == 1
        run: |
          set -e

          export DEBEMAIL="regolith.linux@gmail.com"
          export DEBFULLNAME="Regolith Linux"
          export DEBIAN_FRONTEND=noninteractive

          ${{ steps.init.outputs.gh-repo-path }}/.github/scripts/main.sh \
            build \
            --extension ${{ steps.init.outputs.gh-repo-path }}/.github/scripts/ext-debian.sh \
            --git-repo-path ${{ steps.init.outputs.gh-repo-path }} \
            --manifest-path ${{ steps.init.outputs.manifest-path }} \
            --pkg-build-path ${{ steps.init.outputs.package-build-path }} \
            --pkg-publish-path ${{ steps.init.outputs.package-publish-path }} \
            --distro "${{ steps.init.outputs.distro }}" \
            --codename "${{ steps.init.outputs.codename }}" \
            --arch "${{ steps.init.outputs.arch }}" \
            --stage "${{ steps.init.outputs.stage }}" \
            --suite "${{ steps.init.outputs.suite }}" \
            --component "${{ steps.init.outputs.component }}" \
            --only-package "${{ inputs.only-package }}" | tee -a ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt

          if [ -f ${{ steps.init.outputs.manifest-path }}/next-manifest.txt ]; then
            echo "Temp manifest not deleted by main.sh, build aborted/failed."
            exit 1
          fi

          mv ${{ steps.init.outputs.manifest-path }}/manifest.txt ${{ steps.init.outputs.manifest-path }}/${{ steps.init.outputs.distro }}_${{ steps.init.outputs.codename }}_${{ steps.init.outputs.suite }}-${{ steps.init.outputs.component }}_manifest.txt

          cat ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt | grep ^CHLOG: | cut -c 7- > ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.txt
          cat ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt | grep ^SRCLOG: | cut -c 8- > ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt

          if [ ! -s ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.txt ] ; then
            rm ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.txt
          fi

          if [ ! -s ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt ] ; then
            rm ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt
          fi

      - name: Setup SSH
        uses: regolith-linux/actions/setup-ssh@main
        if: steps.changes.outputs.changed == 1
        with:
          ssh-host: "${{ secrets.KAMATERA_HOSTNAME2 }}"
          ssh-key: "${{ secrets.KAMATERA_SSH_KEY }}"

      - name: Upload Package
        uses: regolith-linux/actions/upload-files@main
        if: steps.changes.outputs.changed == 1
        env:
          server-address: "${{ secrets.KAMATERA_HOSTNAME2 }}"
          server-username: "${{ secrets.KAMATERA_USERNAME }}"
        with:
          upload-from: "${{ steps.init.outputs.package-publish-path }}"
          upload-pattern: "*"
          upload-to-base: "/opt/archives/"
          upload-to-folder: "packages/"

      - name: Upload SourceLog
        uses: regolith-linux/actions/upload-files@main
        if: steps.changes.outputs.changed == 1
        env:
          server-address: "${{ secrets.KAMATERA_HOSTNAME2 }}"
          server-username: "${{ secrets.KAMATERA_USERNAME }}"
        with:
          upload-from: "${{ steps.init.outputs.changelogs-path }}"
          upload-pattern: "SOURCELOG_${{ steps.init.outputs.target }}.txt"
          upload-to-base: "/opt/archives/"
          upload-to-folder: "workspace/"

      # - name: Deploy via rsync
      #   if: steps.changes.outputs.changed == 1
      #   run: |
      #     set -e
      #     set -x

      #     ssh-keyscan -H ${{ secrets.KAMATERA_HOSTNAME2 }} >> ~/.ssh/known_hosts

      #     for i in 1 2 3 4 5; do
      #       echo "Attempt $i"
      #       rsync \
      #         -avzhH \
      #         ${{ steps.init.outputs.package-publish-path }}/* \
      #         root@${{ secrets.KAMATERA_HOSTNAME2 }}:/opt/archives/packages/ && break || sleep 5
      #     done

      #     rsync --ignore-missing-args ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt root@${{ secrets.KAMATERA_HOSTNAME2 }}:/opt/archives/workspace/

      - name: Log Build Output
        if: steps.changes.outputs.changed == 1
        run: |
          cat ${{ steps.init.outputs.manifest-path }}/manifest.txt || true
          echo "package-publish-path:"
          find ${{ steps.init.outputs.package-publish-path }}

      - name: Upload Manifests
        uses: actions/upload-artifact@v4
        if: steps.changes.outputs.changed == 1
        with:
          name: MANIFESTS_${{ steps.init.outputs.target }}
          path: "${{ steps.init.outputs.manifest-path }}/*_manifest.txt"

      - name: Upload Changelogs
        uses: actions/upload-artifact@v4
        if: steps.changes.outputs.changed == 1
        with:
          name: CHANGELOG_${{ steps.init.outputs.target }}
          path: ${{ steps.init.outputs.changelogs-path }}/*${{ steps.init.outputs.target }}.txt

  # # update manifests
  # manifests:
  #   runs-on: ubuntu-24.04
  #   needs: build
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v4

  #     - name: Download Artifacts
  #       uses: actions/download-artifact@v4
  #       with:
  #         path: manifests
  #         pattern: MANIFESTS_*
  #         merge-multiple: true

  #     - name: Check Manifests
  #       run: |
  #         for file in manifests/*_manifest.txt; do
  #           if [ ! -f "$file" ]; then
  #             continue
  #           fi

  #           filename="$(basename $file)"
  #           distro="$(echo $filename | cut -d"_" -f1)"
  #           codename="$(echo $filename | cut -d"_" -f2)"
  #           suite="$(echo $filename | cut -d"_" -f3)"
  #           name="$(echo $filename | cut -d"_" -f4)"

  #           mkdir -p manifests/$distro/$codename/$suite/
  #           mv $file manifests/$distro/$codename/$suite/$name
  #         done

  #     - name: Push Manifest Changes
  #       uses: stefanzweifel/git-auto-commit-action@v5
  #       env:
  #         GITHUB_TOKEN: ${{ secrets.ORG_BROADCAST_TOKEN }}
  #       with:
  #         file_pattern: "*"
  #         commit_message: "chore: update ${{ inputs.stage }} manifest for all packages"
  #         commit_user_name: regolith-bot
  #         commit_user_email: bot@regolith-desktop.com
  #         commit_author: "regolith-bot <bot@regolith-desktop.com>"

  # calculate changelogs
  changelogs:
    runs-on: ubuntu-24.04
    needs: build
    outputs:
      package-changed: ${{ steps.check.outputs.package-changed }}
      source-changed: ${{ steps.check.outputs.source-changed }}
    steps:
      - name: Download Artifacts
        uses: actions/download-artifact@v4
        with:
          path: changelogs
          pattern: CHANGELOG_*
          merge-multiple: true

      - name: Check Changelogs
        id: check
        run: |
          set -e

          if [ ! -d changelogs ]; then
            echo "No package file found to publish!"
            echo "No source file found to rebuild!"

            echo "package-changed=0" >> $GITHUB_OUTPUT
            echo "source-changed=0" >> $GITHUB_OUTPUT
          else
            ls -R changelogs/

            echo "package-changed=$(find changelogs -name CHANGELOG_\*.txt | wc -l)" >> $GITHUB_OUTPUT
            echo "source-changed=$(find changelogs -name SOURCELOG_\*.txt | wc -l)" >> $GITHUB_OUTPUT
          fi

  # rebuild sources
  source:
    needs: [changelogs]
    if: ${{ !failure() && !cancelled() && needs.changelogs.outputs.source-changed != 0 }}
    uses: ./.github/workflows/rebuild-sources.yml
    with:
      pull-from: /opt/archives/workspace/
      push-to: /opt/archives/packages/
    secrets: inherit

  # publish archives
  publish:
    needs: [changelogs, source]
    if: ${{ !failure() && !cancelled() && needs.changelogs.outputs.package-changed != 0 }}
    uses: ./.github/workflows/publish-packages.yml
    with:
      packages-path: /opt/archives/packages/
    secrets: inherit

  # # create a release with changlogs
  # release:
  #   runs-on: ubuntu-24.04
  #   needs: [changelogs, publish]
  #   if: ${{ !failure() && !cancelled() && needs.changelogs.outputs.package-changed != 0 }}
  #   steps:
  #     - name: Download Artifacts
  #       uses: actions/download-artifact@v4
  #       with:
  #         path: changelogs
  #         pattern: CHANGELOG_*
  #         merge-multiple: true

  #     - name: Prepare Release
  #       id: prepare
  #       run: |
  #         echo "TIMESTAMP=$(date +%Y%m%d_%H%M%S)" >> $GITHUB_OUTPUT
  #         find changelogs/ -name CHANGELOG_\*.txt -exec sh -c 'cat "$1" >> CHANGELOG.txt' -- {} \;
  #         cat CHANGELOG.txt

  #     - uses: softprops/action-gh-release@v2
  #       with:
  #         body: See CHANGELOG.txt for updates and manifests for current state of repos.
  #         name: Package Build ${{ steps.prepare.outputs.TIMESTAMP }}
  #         tag_name: pkgbuild-${{ steps.prepare.outputs.TIMESTAMP }}
  #         files: |
  #           *.txt

  # run the tests
  test:
    needs: [changelogs]
    if: ${{ !failure() && !cancelled() }}
    uses: ./.github/workflows/test-desktop-installable2.yml
    with:
      stage: ${{ inputs.stage }}
      distro: ${{ inputs.distro }}
      codename: ${{ inputs.codename }}
      arch: ${{ inputs.arch }}