diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92a3b6fa..da0fe7e2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,25 +2,57 @@ name: Build and Test env: GITHUB_TAG: ghcr.io/${{ github.repository }} - DOCKERHUB_TAG: godebos/debos on: push: branches-ignore: - '*.tmp' + tags: + - '*' + # Build at 04:00am every Monday + schedule: + - cron: "0 4 * * 1" pull_request: workflow_dispatch: jobs: + test: + strategy: + fail-fast: false + matrix: + variant: + - debos-arch + - debos-bookworm + - debos-bullseye + runs-on: ubuntu-latest + defaults: + run: + shell: bash + container: + image: ghcr.io/go-debos/test-containers/${{matrix.variant}}:main + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Test build + run: go build -o debos cmd/debos/debos.go + + - name: Run unit tests + run: go test -v ./... | tee test.out + + - name: Ensure no tests were skipped + run: "! grep -q SKIP test.out" + build: name: Build Docker container runs-on: ubuntu-latest + needs: test steps: - name: Repository checkout uses: actions/checkout@v3 - name: Setup Docker buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Use cache uses: actions/cache@v3 @@ -29,7 +61,7 @@ jobs: key: ${{ runner.os }}-docker - name: Build Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . push: false @@ -56,7 +88,7 @@ jobs: uses: actions/checkout@v3 - name: Setup Docker buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Use cache uses: actions/cache@v3 @@ -65,7 +97,7 @@ jobs: key: ${{ runner.os }}-docker - name: Build Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . push: false @@ -92,12 +124,16 @@ jobs: - { name: "uml", backend: "--fakemachine-backend=uml" } test: - { name: "recipes", case: "recipes" } - - { name: "debian (amd64}", case: "debian", variables: "-t architecture:amd64" } + - { name: "debian (amd64)", case: "debian", variables: "-t architecture:amd64" } - { name: "debian (arm64)", case: "debian", variables: "-t architecture:arm64" } - { name: "debian (armhf)", case: "debian", variables: "-t architecture:armhf" } include: + - backend: { name: "arch", backend: "--fakemachine-backend=qemu" } + test: { name: "arch", case: "arch" } - backend: { name: "qemu", backend: "--fakemachine-backend=qemu" } test: { name: "partitioning", case: "partitioning" } + - backend: { name: "uml", backend: "--fakemachine-backend=uml" } + test: { name: "apertis", case: "apertis" } - backend: { name: "uml", backend: "--fakemachine-backend=uml" } test: { name: "partitioning", case: "partitioning" } - backend: { name: "qemu", backend: "--fakemachine-backend=qemu" } @@ -133,22 +169,25 @@ jobs: --tmpfs /run --privileged -e TMP=/scratch + -e SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 debos -v ${{matrix.backend.backend}} ${{matrix.test.variables}} ${{matrix.test.case}}/test.yaml - # Job to key the bors success status against - bors: - name: bors - if: success() + # Job to key success status against + allgreen: + name: allgreen + if: always() needs: - unit-tests - recipe-tests runs-on: ubuntu-latest steps: - - name: Mark the job as a success - run: exit 0 + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} publish-github: name: Publish to GHCR @@ -167,7 +206,7 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ${{ env.GITHUB_TAG }} tags: | @@ -178,14 +217,14 @@ jobs: "type=ref,event=pr" - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Setup Docker buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Use cache uses: actions/cache@v3 @@ -194,7 +233,7 @@ jobs: key: ${{ runner.os }}-docker - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . push: true @@ -203,12 +242,37 @@ jobs: file: docker/Dockerfile cache-from: type=local,src=/tmp/.build-cache + check-dockerhub-secrets: + name: Check DockerHub secrets exist + runs-on: ubuntu-latest + outputs: + has-secrets: ${{ steps.check-secrets.outputs.has-secrets }} + steps: + - id: check-secrets + name: Check secrets exist + run: | + if [[ "${{ secrets.DOCKERHUB_IMAGE }}" != "" && \ + "${{ secrets.DOCKERHUB_USERNAME }}" != "" && \ + "${{ secrets.DOCKERHUB_PASSWORD }}" != "" ]]; \ + then + echo "Dockerhub secrets exist" + echo "has-secrets=true" >> $GITHUB_OUTPUT + else + echo "Dockerhub secrets do not exist; not pushing to Dockerhub" + echo "Please set the following secrets on GitHub (settings > secrets > actions > new):" + echo "DOCKERHUB_IMAGE, DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD" + echo "has-secrets=false" >> $GITHUB_OUTPUT + fi + publish-dockerhub: name: Publish to DockerHub needs: + - check-dockerhub-secrets - unit-tests - recipe-tests - if: github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + if: | + needs.check-dockerhub-secrets.outputs.has-secrets == 'true' && + github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: read @@ -220,9 +284,9 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: - images: ${{ env.DOCKERHUB_TAG }} + images: ${{ secrets.DOCKERHUB_IMAGE }} tags: | "type=ref,event=branch" "type=ref,suffix=-{{sha}},event=branch" @@ -231,14 +295,14 @@ jobs: "type=ref,event=pr" - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 continue-on-error: true with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Setup Docker buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Use cache uses: actions/cache@v3 @@ -247,7 +311,7 @@ jobs: key: ${{ runner.os }}-docker - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 continue-on-error: true with: context: . diff --git a/README.md b/README.md index ff2fc49e..6e1c08af 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ Some of the actions provided by debos to customize and produce images are: * ostree-deploy: deploy an OSTree branch to the image * overlay: do a recursive copy of directories or files to the target filesystem * pack: create a tarball with the target filesystem +* pacman: install packages and their dependencies with pacman +* pacstrap: construct the target rootfs with pacstrap * raw: directly write a file to the output image at a given offset * recipe: includes the recipe actions at the given path * run: allows to run a command or script in the filesystem or in the host @@ -66,8 +68,8 @@ See [docker/README.md](https://github.com/go-debos/debos/blob/master/docker/READ sudo apt install golang git libglib2.0-dev libostree-dev qemu-system-x86 \ qemu-user-static debootstrap systemd-container - export GOPATH=/opt/src/gocode # or whatever suites your needs - go get -u github.com/go-debos/debos/cmd/debos + export GOPATH=/opt/src/gocode # or whatever suits your needs + go install -v github.com/go-debos/debos/cmd/debos@latest /opt/src/gocode/bin/debos --help ## Simple example @@ -82,10 +84,10 @@ make a tarball. actions: - action: debootstrap - suite: "buster" + suite: bookworm components: - main - - non-free + - non-free-firmware mirror: https://deb.debian.org/debian variant: minbase @@ -112,7 +114,8 @@ this: ## Other examples -This example builds a customized image for a Raspberry Pi 3. +Example recipes are collected in a separate repository: + https://github.com/go-debos/debos-recipes ## Environment variables @@ -136,7 +139,7 @@ no_proxy defined, both will be propagated to fakemachine respecting the case. The command line options --environ-var and -e can be used to specify, overwrite, and unset environment variables for fakemachine with the syntax: -$ debos -e ENVIRONVAR:VALUE ... + $ debos -e ENVIRONVAR:VALUE ... To unset an enviroment variable, or in other words, to prevent an environment variable to be propagated to fakemachine, use the same syntax without a value. @@ -162,6 +165,18 @@ Fakemachine can use different virtualisation backends to spawn the virtualmachin for more information see the documentation under the [fakemachine repository](https://github.com/go-debos/fakemachine). By default the backend will automatically be selected based on what is supported -on the host machine, but this can be overridden using the `--fakemachine-backend` +on the host machine, but this can be overridden using the `--fakemachine-backend` / `-b` option. If no backends are supported, debos reverts to running the recipe on the host without creating a fakemachine. + +Performance of the backends is roughly as follows: `kvm` is faster than `uml` is faster than `qemu`. +Using `--disable-fakemachine` is slightly faster than `kvm`, but requires root permissions. + +Numbers for running [pine-a64-plus/debian.yaml](https://github.com/go-debos/debos-recipes/blob/9a25b4be6c9136f4a27e542f39ab7e419fc852c9/pine-a64-plus/debian.yaml) on an Intel Pentium G4560T with SSD: + +| Backend | Wall Time | Prerequisites | +| --- | --- | --- | +| `--disable-fakemachine` | 8 min | root permissions | +| `-b kvm` | 9 min | access to `/dev/kvm` | +| `-b uml` | 18 min | package `user-mode-linux` installed | +| `-b qemu` | 166 min | none | diff --git a/actions/actions_doc.go b/actions/actions_doc.go index 7ebd532d..7c4bd53a 100644 --- a/actions/actions_doc.go +++ b/actions/actions_doc.go @@ -2,5 +2,16 @@ /* Package 'actions' implements 'debos' modules used for OS creation. + +The origin property + +Several actions have the 'origin' property. Possible values for the +'origin' property are: + + 1) 'recipe' ....... directory the recipe is in + 2) 'filesystem' ... target filesystem root directory from previous filesystem-deploy action or + a previous ostree action. + 3) name property of a previous download action + */ package actions diff --git a/actions/apt_action.go b/actions/apt_action.go index ccc2aa43..82267a56 100644 --- a/actions/apt_action.go +++ b/actions/apt_action.go @@ -3,7 +3,7 @@ Apt Action Install packages and their dependencies to the target rootfs with 'apt'. -Yaml syntax: + # Yaml syntax: - action: apt recommends: bool unauthenticated: bool diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index c219ec2d..e7bd4006 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -7,7 +7,7 @@ Please keep in mind -- file `/etc/resolv.conf` will be removed after execution. Most of the OS scripts used by `debootstrap` copy `resolv.conf` from the host, and this may lead to incorrect configuration when becoming part of the created rootfs. -Yaml syntax: + # Yaml syntax: - action: debootstrap mirror: URL suite: "name" @@ -52,6 +52,7 @@ package actions import ( "fmt" "io" + "log" "os" "path" "strings" @@ -157,6 +158,24 @@ func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { return err } +// Guess if suite is something before usr-is-merged was introduced +func (d *DebootstrapAction) isLikelyOldSuite() bool { + switch strings.ToLower(d.Suite) { + case "sid", "unstable": + return false + case "testing": + return false + case "bookworm": + return false + case "trixie": + return false + case "forky": + return false + default: + return true + } +} + func (d *DebootstrapAction) Run(context *debos.DebosContext) error { d.LogStart() cmdline := []string{"debootstrap"} @@ -203,6 +222,12 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error { cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant)) } + // workaround for https://github.com/go-debos/debos/issues/361 + if d.isLikelyOldSuite() { + log.Println("excluding usr-is-merged as package is not in suite") + cmdline = append(cmdline, "--exclude=usr-is-merged") + } + cmdline = append(cmdline, d.Suite) cmdline = append(cmdline, context.Rootdir) cmdline = append(cmdline, d.Mirror) diff --git a/actions/download_action.go b/actions/download_action.go index a4099e7c..e664e5fd 100644 --- a/actions/download_action.go +++ b/actions/download_action.go @@ -3,7 +3,7 @@ Download Action Download a single file from Internet and unpack it in place if needed. -Yaml syntax: + # Yaml syntax: - action: download url: http://example.domain/path/filename.ext name: firmware diff --git a/actions/filesystem_deploy_action.go b/actions/filesystem_deploy_action.go index 15199046..094fb161 100644 --- a/actions/filesystem_deploy_action.go +++ b/actions/filesystem_deploy_action.go @@ -9,7 +9,7 @@ action requires 'image-partition' action to be executed before it. After this action has ran, subsequent actions are executed on the mounted output image. -Yaml syntax: + # Yaml syntax: - action: filesystem-deploy setup-fstab: bool setup-kernel-cmdline: bool diff --git a/actions/image_partition_action.go b/actions/image_partition_action.go index fb53091b..aa56fdc9 100644 --- a/actions/image_partition_action.go +++ b/actions/image_partition_action.go @@ -7,7 +7,7 @@ build, and optionally (but by-default) mounted at boot in the final system. The mountpoints are sorted on their position in the filesystem hierarchy so the order in the recipe does not matter. -Yaml syntax: + # Yaml syntax: - action: image-partition imagename: image_name imagesize: size @@ -45,8 +45,7 @@ should be in GUID format (e.g.: '00002222-4444-6666-AAAA-BBBBCCCCFFFF' where eac character is an hexadecimal digit). For 'msdos' partition table, 'diskid' should be a 32 bits hexadecimal number (e.g. '1234ABCD' without any dash separator). -Yaml syntax for partitions: - + # Yaml syntax for partitions: partitions: - name: partition name partlabel: partition label @@ -54,6 +53,7 @@ Yaml syntax for partitions: start: offset end: offset features: list of filesystem features + extendedoptions: list of filesystem extended options flags: list of flags fsck: bool fsuuid: string @@ -103,8 +103,10 @@ ext2, ext3, ext4 and xfs. - partuuid -- GPT partition UUID string. -Yaml syntax for mount points: +- extendedoptions -- list of additional filesystem extended options which need +to be enabled for the partition. + # Yaml syntax for mount points: mountpoints: - mountpoint: path partition: partition label @@ -130,8 +132,7 @@ to define a `mountpoint` path which is temporary and unique for the image, for example: `/mnt/temporary_mount`. Defaults to false. -Layout example for Raspberry PI 3: - + # Layout example for Raspberry PI 3: - action: image-partition imagename: "debian-rpi3.img" imagesize: 1GB @@ -178,18 +179,19 @@ import ( ) type Partition struct { - number int - Name string - PartLabel string - PartType string - PartUUID string - Start string - End string - FS string - Flags []string - Features []string - Fsck bool "fsck" - FSUUID string + number int + Name string + PartLabel string + PartType string + PartUUID string + Start string + End string + FS string + Flags []string + Features []string + ExtendedOptions []string + Fsck bool "fsck" + FSUUID string } type Mountpoint struct { @@ -200,6 +202,23 @@ type Mountpoint struct { part *Partition } +type imageLocker struct { + fd *os.File +} + +func lockImage(context *debos.DebosContext) (*imageLocker, error) { + fd, err := os.Open(context.Image) + if err != nil { + return nil, err + } + syscall.Flock(int(fd.Fd()), syscall.LOCK_EX) + return &imageLocker{fd: fd}, nil +} + +func (i imageLocker) unlock() { + i.fd.Close() +} + type ImagePartitionAction struct { debos.BaseAction `yaml:",inline"` ImageName string @@ -357,6 +376,9 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC if len(p.Features) > 0 { cmdline = append(cmdline, "-O", strings.Join(p.Features, ",")) } + if len(p.ExtendedOptions) > 0 { + cmdline = append(cmdline, "-E", strings.Join(p.ExtendedOptions, ",")) + } if len(p.FSUUID) > 0 { if p.FS == "ext2" || p.FS == "ext3" || p.FS == "ext4" { cmdline = append(cmdline, "-U", p.FSUUID) @@ -370,7 +392,7 @@ func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosC cmd := debos.Command{} /* Some underlying device driver, e.g. the UML UBD driver, may manage holes - * incorrectly which will prevent to retrieve all usefull zero ranges in + * incorrectly which will prevent to retrieve all useful zero ranges in * filesystem, e.g. when using 'bmaptool create', see patch * http://lists.infradead.org/pipermail/linux-um/2022-January/002074.html * @@ -424,30 +446,18 @@ func (i *ImagePartitionAction) PreNoMachine(context *debos.DebosContext) error { func (i ImagePartitionAction) Run(context *debos.DebosContext) error { i.LogStart() - /* Exclusively Lock image device file to prevent udev from triggering - * partition rescans, which cause confusion as some time asynchronously the - * partition device might disappear and reappear due to that! */ - imageFD, err := os.Open(context.Image) - if err != nil { - return err - } - /* Defer will keep the fd open until the function returns, at which points - * the filesystems will have been mounted protecting from more udev funnyness - * After the fd is closed the kernel needs to be informed of partition table - * changes (defer calls are executed in LIFO order) */ - defer i.triggerDeviceNodes(context) - defer imageFD.Close() - - err = syscall.Flock(int(imageFD.Fd()), syscall.LOCK_EX) - if err != nil { - return err - } - + /* On certain disk device events udev will call the BLKRRPART ioctl to + * re-read the partition table. This will cause the partition devices + * (e.g. vda3) to temporarily disappear while the rescanning happens. + * udev does this while holding an exclusive flock. This means to avoid partition + * devices disappearing while doing operations on them (e.g. formatting + * and mounting) we need to do it while holding an exclusive lock + */ command := []string{"parted", "-s", context.Image, "mklabel", i.PartitionType} if len(i.GptGap) > 0 { command = append(command, i.GptGap) } - err = debos.Command{}.Run("parted", command...) + err := debos.Command{}.Run("parted", command...) if err != nil { return err } @@ -517,13 +527,19 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { } } - devicePath := i.getPartitionDevice(p.number, *context) + lock, err := lockImage(context) + if err != nil { + return err + } + defer lock.unlock() err = i.formatPartition(p, *context) if err != nil { return err } + lock.unlock() + devicePath := i.getPartitionDevice(p.number, *context) context.ImagePartitions = append(context.ImagePartitions, debos.Partition{p.Name, devicePath}) } @@ -547,15 +563,22 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { return strings.Count(mntA, "/") < strings.Count(mntB, "/") }) + lock, err := lockImage(context) + if err != nil { + return err + } + defer lock.unlock() + for _, m := range i.Mountpoints { dev := i.getPartitionDevice(m.part.number, *context) mntpath := path.Join(context.ImageMntDir, m.Mountpoint) os.MkdirAll(mntpath, 0755) - err := syscall.Mount(dev, mntpath, m.part.FS, 0, "") + err = syscall.Mount(dev, mntpath, m.part.FS, 0, "") if err != nil { return fmt.Errorf("%s mount failed: %v", m.part.Name, err) } } + lock.unlock() err = i.generateFSTab(context) if err != nil { @@ -567,6 +590,10 @@ func (i ImagePartitionAction) Run(context *debos.DebosContext) error { return err } + /* Now that all partitions are created (re)trigger all udev events for + * the image file to make sure everything is in a reasonable state + */ + i.triggerDeviceNodes(context) return nil } @@ -763,7 +790,7 @@ func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error { // binary units are multiples of 1024 - KiB, MiB, GiB, TiB, PiB // decimal units are multiples of 1000 - KB, MB, GB, TB, PB var getSizeValueFunc func(size string) (int64, error) - if regexp.MustCompile(`^[0-9]+[kmgtp]ib+$`).MatchString(strings.ToLower(i.ImageSize)) { + if regexp.MustCompile(`^[0-9.]+[kmgtp]ib+$`).MatchString(strings.ToLower(i.ImageSize)) { getSizeValueFunc = units.RAMInBytes } else { getSizeValueFunc = units.FromHumanSize diff --git a/actions/ostree_commit_action.go b/actions/ostree_commit_action.go index 85b3fdac..64a279e2 100644 --- a/actions/ostree_commit_action.go +++ b/actions/ostree_commit_action.go @@ -3,7 +3,7 @@ OstreeCommit Action Create OSTree commit from rootfs. -Yaml syntax: + # Yaml syntax: - action: ostree-commit repository: repository name branch: branch name diff --git a/actions/ostree_deploy_action.go b/actions/ostree_deploy_action.go index 73536914..e4249e4a 100644 --- a/actions/ostree_deploy_action.go +++ b/actions/ostree_deploy_action.go @@ -7,7 +7,7 @@ during this step. Action 'image-partition' must be called prior to OSTree deploy. -Yaml syntax: + # Yaml syntax: - action: ostree-deploy repository: repository name remote_repository: URL diff --git a/actions/overlay_action.go b/actions/overlay_action.go index 6a50a4b0..5c995511 100644 --- a/actions/overlay_action.go +++ b/actions/overlay_action.go @@ -3,7 +3,7 @@ Overlay Action Recursive copy of directory or file to target filesystem. -Yaml syntax: + # Yaml syntax: - action: overlay origin: name source: directory @@ -26,6 +26,7 @@ package actions import ( "fmt" + "log" "path" "github.com/go-debos/debos" @@ -63,5 +64,6 @@ func (overlay *OverlayAction) Run(context *debos.DebosContext) error { return err } + log.Printf("Overlaying %s on %s", sourcedir, destination) return debos.CopyTree(sourcedir, destination) } diff --git a/actions/pack_action.go b/actions/pack_action.go index b85f3188..fbe9a10b 100644 --- a/actions/pack_action.go +++ b/actions/pack_action.go @@ -3,7 +3,7 @@ Pack Action Create tarball with filesystem. -Yaml syntax: + # Yaml syntax: - action: pack file: filename.ext compression: gz @@ -26,6 +26,7 @@ import ( "log" "path" "strings" + "os/exec" "github.com/go-debos/debos" ) @@ -68,11 +69,29 @@ func (pf *PackAction) Verify(context *debos.DebosContext) error { func (pf *PackAction) Run(context *debos.DebosContext) error { pf.LogStart() + usePigz := false + if pf.Compression == "gz" { + if _,err := exec.LookPath("pigz"); err == nil { + usePigz = true + } + } outfile := path.Join(context.Artifactdir, pf.File) - var tarOpt = "cf" + tarOpts[pf.Compression] + command := []string{"tar"} + if usePigz == true { + command = append(command, "cf") + } else { + command = append(command, "cf" + tarOpts[pf.Compression]) + } + command = append(command, outfile) + command = append(command, "--xattrs") + command = append(command, "--xattrs-include=*.*") + if usePigz == true { + command = append(command, "--use-compress-program=pigz") + } + command = append(command, "-C", context.Rootdir) + command = append(command, ".") + log.Printf("Compressing to %s\n", outfile) - return debos.Command{}.Run("Packing", "tar", tarOpt, outfile, - "--xattrs", "--xattrs-include=*.*", - "-C", context.Rootdir, ".") + return debos.Command{}.Run("Packing", command...) } diff --git a/actions/pacman_action.go b/actions/pacman_action.go new file mode 100644 index 00000000..9f390db2 --- /dev/null +++ b/actions/pacman_action.go @@ -0,0 +1,39 @@ +/* +Pacman Action + +Install packages and their dependencies to the target rootfs with 'pacman'. + + # Yaml syntax: + - action: pacman + packages: + - package1 + - package2 + +Mandatory properties: + +- packages -- list of packages to install +*/ +package actions + +import ( + "github.com/go-debos/debos" +) + +type PacmanAction struct { + debos.BaseAction `yaml:",inline"` + Packages []string +} + +func (p *PacmanAction) Run(context *debos.DebosContext) error { + p.LogStart() + + pacmanOptions := []string{"pacman", "-Syu", "--noconfirm"} + pacmanOptions = append(pacmanOptions, p.Packages...) + + c := debos.NewChrootCommandForContext(*context) + if err := c.Run("pacman", pacmanOptions...); err != nil { + return err + } + + return nil +} diff --git a/actions/pacstrap_action.go b/actions/pacstrap_action.go new file mode 100644 index 00000000..25c1624e --- /dev/null +++ b/actions/pacstrap_action.go @@ -0,0 +1,133 @@ +/* +Pacstrap Action + +Construct the target rootfs with pacstrap tool. + + # Yaml syntax: + - action: pacstrap + config: + mirror: + +Mandatory properties: + + - config -- the pacman.conf file which will be used through the process + - mirror -- the mirrorlist file which will be used through the process +*/ +package actions + +import ( + "fmt" + "io/ioutil" + "os" + "path" + + "github.com/go-debos/debos" + "github.com/go-debos/fakemachine" +) + +type PacstrapAction struct { + debos.BaseAction `yaml:",inline"` + Config string `yaml:"config"` + Mirror string `yaml:"mirror"` +} + +func (d *PacstrapAction) listOptionFiles(context *debos.DebosContext) ([]string, error) { + files := []string{} + + if d.Config == "" { + return nil, fmt.Errorf("No config file set") + } + d.Config = debos.CleanPathAt(d.Config, context.RecipeDir) + files = append(files, d.Config) + + if d.Mirror == "" { + return nil, fmt.Errorf("No mirror file set") + } + d.Mirror = debos.CleanPathAt(d.Mirror, context.RecipeDir) + files = append(files, d.Mirror) + + return files, nil +} + +func (d *PacstrapAction) Verify(context *debos.DebosContext) error { + files, err := d.listOptionFiles(context) + if err != nil { + return err + } + + // Check if all needed files exists + for _, f := range files { + if _, err := os.Stat(f); os.IsNotExist(err) { + return err + } + } + + return nil +} + +func (d *PacstrapAction) PreNoMachine(context *debos.DebosContext) error { + return fmt.Errorf("action requires fakemachine") +} + +func (d *PacstrapAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine, args *[]string) error { + mounts, err := d.listOptionFiles(context) + if err != nil { + return err + } + + // Mount configuration files outside of recipes directory + for _, mount := range mounts { + m.AddVolume(path.Dir(mount)) + } + + return nil +} + +func (d *PacstrapAction) Run(context *debos.DebosContext) error { + d.LogStart() + + files := map[string]string{ + "/etc/pacman.conf": d.Config, + "/etc/pacman.d/mirrorlist": d.Mirror, + } + + // Copy the config/mirrorlist files + for dest, src := range files { + if err := os.MkdirAll(path.Dir(dest), 0755); err != nil { + return err + } + + read, err := ioutil.ReadFile(src) + if err != nil { + return err + } + + if err = ioutil.WriteFile(dest, read, 0644); err != nil { + return err + } + } + + // Setup the local keychain, within the fakemachine instance, since we + // don't have access to the host one. + // Even if we did, blindly copying it might not be a good idea. + cmdline := []string{"pacman-key", "--init"} + if err := (debos.Command{}.Run("pacman-key", cmdline...)); err != nil { + return fmt.Errorf("Couldn't init pacman keyring: %v", err) + } + + // When there's no explicit keyring suite we populate all available + cmdline = []string{"pacman-key", "--populate"} + if err := (debos.Command{}.Run("pacman-key", cmdline...)); err != nil { + return fmt.Errorf("Couldn't populate pacman keyring: %v", err) + } + + // Run pacstrap + cmdline = []string{"pacstrap", context.Rootdir} + if err := (debos.Command{}.Run("pacstrap", cmdline...)); err != nil { + log := path.Join(context.Rootdir, "var/log/pacman.log") + _ = debos.Command{}.Run("pacstrap.log", "cat", log) + return err + } + + return nil +} diff --git a/actions/raw_action.go b/actions/raw_action.go index 013eeae8..0807a17f 100644 --- a/actions/raw_action.go +++ b/actions/raw_action.go @@ -4,7 +4,7 @@ Raw Action Directly write a file to the output image at a given offset. This is typically useful for bootloaders. -Yaml syntax: + # Yaml syntax: - action: raw origin: name source: filename diff --git a/actions/recipe.go b/actions/recipe.go index 36826312..ddc03c95 100644 --- a/actions/recipe.go +++ b/actions/recipe.go @@ -55,6 +55,10 @@ Supported actions - pack -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Pack_Action +- pacman -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Pacman_Action + +- pacstrap -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Pacstrap_Action + - raw -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Raw_Action - recipe -- https://godoc.org/github.com/go-debos/debos/actions#hdr-Recipe_Action @@ -100,6 +104,8 @@ func (y *YamlAction) UnmarshalYAML(unmarshal func(interface{}) error) error { switch aux.Action { case "debootstrap": y.Action = NewDebootstrapAction() + case "pacstrap": + y.Action = &PacstrapAction{} case "pack": y.Action = NewPackAction() case "unpack": @@ -108,6 +114,8 @@ func (y *YamlAction) UnmarshalYAML(unmarshal func(interface{}) error) error { y.Action = &RunAction{} case "apt": y.Action = NewAptAction() + case "pacman": + y.Action = &PacmanAction{} case "ostree-commit": y.Action = &OstreeCommitAction{} case "ostree-deploy": diff --git a/actions/recipe_action.go b/actions/recipe_action.go index 9fa3b1c9..860faa69 100644 --- a/actions/recipe_action.go +++ b/actions/recipe_action.go @@ -11,7 +11,7 @@ passed in the "architecture" template variable. Limitations of combined recipes are equivalent to limitations within a single recipe (e.g. there can only be one image partition action). -Yaml syntax: + # Yaml syntax: - action: recipe recipe: path to recipe variables: diff --git a/actions/run_action.go b/actions/run_action.go index 7520a97d..af7fc797 100644 --- a/actions/run_action.go +++ b/actions/run_action.go @@ -5,7 +5,7 @@ Allows to run any available command or script in the filesystem or in build process host environment: specifically inside the fakemachine created by Debos. -Yaml syntax: + # Yaml syntax: - action: run chroot: bool postprocess: bool diff --git a/actions/unpack_action.go b/actions/unpack_action.go index 65551ada..8c7eac44 100644 --- a/actions/unpack_action.go +++ b/actions/unpack_action.go @@ -6,7 +6,7 @@ Useful for creating target rootfs from saved tarball with prepared file structur Only (compressed) tar archives are supported currently. -Yaml syntax: + # Yaml syntax: - action: unpack origin: name file: file.ext diff --git a/archiver.go b/archiver.go index c54f89c0..5844df5d 100644 --- a/archiver.go +++ b/archiver.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "os/exec" ) type ArchiveType int @@ -92,6 +93,12 @@ func tarOptions(compression string) string { func (tar *ArchiveTar) Unpack(destination string) error { command := []string{"tar"} + usePigz := false + if compression, ok := tar.options["tarcompression"]; ok && compression == "gz" { + if _, err := exec.LookPath("pigz"); err == nil { + usePigz = true + } + } if options, ok := tar.options["taroptions"].([]string); ok { for _, option := range options { command = append(command, option) @@ -104,7 +111,11 @@ func (tar *ArchiveTar) Unpack(destination string) error { if compression, ok := tar.options["tarcompression"]; ok { if unpackTarOpt := tarOptions(compression.(string)); len(unpackTarOpt) > 0 { - command = append(command, unpackTarOpt) + if usePigz == true { + command = append(command, "--use-compress-program=pigz") + } else { + command = append(command, unpackTarOpt) + } } } command = append(command, "-f", tar.file) diff --git a/bors.toml b/bors.toml deleted file mode 100644 index 40f18820..00000000 --- a/bors.toml +++ /dev/null @@ -1 +0,0 @@ -status = [ "bors" ] diff --git a/doc/examples/example.yaml b/doc/examples/debian-example-ospack.yaml similarity index 52% rename from doc/examples/example.yaml rename to doc/examples/debian-example-ospack.yaml index 9375323e..18f72b65 100644 --- a/doc/examples/example.yaml +++ b/doc/examples/debian-example-ospack.yaml @@ -1,6 +1,6 @@ -{{- $architecture := or .architecture "arm64" -}} -{{- $suite := or .suite "buster" -}} -{{ $image := or .image (printf "debian-%s-%s.tgz" $suite $architecture) }} +{{ $architecture := or .architecture "arm64" }} +{{ $suite := or .suite "bullseye" }} +{{ $image := or .image (printf "debian-example-ospack-%s-%s.tar.gz" $suite $architecture) }} architecture: {{ $architecture }} @@ -9,25 +9,26 @@ actions: suite: {{ $suite }} components: - main - - contrib - - non-free mirror: https://deb.debian.org/debian variant: minbase - action: apt description: Install some packages - packages: [ sudo, openssh-server, adduser, systemd-sysv, firmware-linux ] + packages: + - sudo + - openssh-server + - adduser + - systemd-sysv - action: run chroot: true - script: setup-user.sh + script: scripts/setup-user.sh - action: overlay source: overlays/sudo - action: run - chroot: true - command: echo debian > /etc/hostname + command: echo debian > ${ROOTDIR}/etc/hostname - action: pack file: {{ $image }} diff --git a/doc/examples/setup-user.sh b/doc/examples/scripts/setup-user.sh similarity index 71% rename from doc/examples/setup-user.sh rename to doc/examples/scripts/setup-user.sh index da155d12..07b29ee9 100755 --- a/doc/examples/setup-user.sh +++ b/doc/examples/scripts/setup-user.sh @@ -3,7 +3,7 @@ set -e echo "I: create user" -adduser --gecos User user +adduser --gecos User --disabled-password user echo "I: set user password" echo "user:user" | chpasswd diff --git a/doc/man/debos.1 b/doc/man/debos.1 index 2592d600..de3fd27c 100644 --- a/doc/man/debos.1 +++ b/doc/man/debos.1 @@ -69,6 +69,10 @@ filesystem .IP \[bu] 2 pack: create a tarball with the target filesystem .IP \[bu] 2 +pacman: install packages and their dependencies with pacman +.IP \[bu] 2 +pacstrap: construct the target rootfs with pacstrap +.IP \[bu] 2 raw: directly write a file to the output image at a given offset .IP \[bu] 2 recipe: includes the recipe actions at the given path diff --git a/docker/Dockerfile b/docker/Dockerfile index ae68a583..972236b7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,6 +26,22 @@ COPY . $GOPATH/src/github.com/go-debos/debos WORKDIR $GOPATH/src/github.com/go-debos/debos/cmd/debos RUN go install ./... +# Pull the latest archlinux-keyring, since the one in Debian is outdated +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1026080 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + pkgconf \ + python3-all \ + sq \ + systemd && \ + rm -rf /var/lib/apt/lists/* + +RUN git clone https://gitlab.archlinux.org/archlinux/archlinux-keyring && \ + cd archlinux-keyring && \ + git checkout -B latest-release 20221213 && \ + make build && \ + make PREFIX=/usr KEYRING_TARGET_DIR=/usr/share/keyrings/ DESTDIR=/arch-keyring install + ### second stage - runner ### FROM debian:bullseye-slim as runner @@ -76,6 +92,7 @@ RUN apt-get update && \ pkg-config \ qemu-system-x86 \ qemu-user-static \ + qemu-utils \ rsync \ systemd \ systemd-container \ @@ -87,6 +104,21 @@ RUN apt-get update && \ zip && \ rm -rf /var/lib/apt/lists/* +# Enable backports for the Arch dependencies +RUN echo "deb http://ftp.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list + +# NOTE: Explicitly install arch-install-scripts from backports. The normal one +# lacks pactrap. +# Install Arch dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + makepkg \ + pacman-package-manager && \ + apt-get install -y --no-install-recommends \ + -t bullseye-backports \ + arch-install-scripts && \ + rm -rf /var/lib/apt/lists/* + # debian's qemu-user-static package no longer registers binfmts # if running inside a virtualmachine; dockerhub builds are inside a vm RUN for arch in aarch64 alpha arm armeb cris hexagon hppa m68k microblaze mips mips64 mips64el mipsel mipsn32 mipsn32el ppc ppc64 ppc64le riscv32 riscv64 s390x sh4 sh4eb sparc sparc32plus sparc64 xtensa xtensaeb; do \ @@ -95,4 +127,8 @@ RUN for arch in aarch64 alpha arm armeb cris hexagon hppa m68k microblaze mips m COPY --from=builder $GOPATH/bin/debos /usr/local/bin/debos +# Pull the latest archlinux-keyring, since the one in Debian is outdated +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1026080 +COPY --from=builder /arch-keyring/ / + ENTRYPOINT ["/usr/local/bin/debos"] diff --git a/filesystem.go b/filesystem.go index b5c41fa9..43cf8c82 100644 --- a/filesystem.go +++ b/filesystem.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "os" "path" "path/filepath" @@ -58,7 +57,6 @@ func CopyFile(src, dst string, mode os.FileMode) error { } func CopyTree(sourcetree, desttree string) error { - fmt.Printf("Overlaying %s on %s\n", sourcetree, desttree) walker := func(p string, info os.FileInfo, err error) error { if err != nil { @@ -71,18 +69,18 @@ func CopyTree(sourcetree, desttree string) error { case 0: err := CopyFile(p, target, info.Mode()) if err != nil { - log.Panicf("Failed to copy file %s: %v", p, err) + return fmt.Errorf("Failed to copy file %s: %w", p, err) } case os.ModeDir: os.Mkdir(target, info.Mode()) case os.ModeSymlink: link, err := os.Readlink(p) if err != nil { - log.Panicf("Failed to read symlink %s: %v", suffix, err) + return fmt.Errorf("Failed to read symlink %s: %w", suffix, err) } os.Symlink(link, target) default: - log.Panicf("Not handled /%s %v", suffix, info.Mode()) + return fmt.Errorf("File %s with mode %v not handled", p, info.Mode()) } return nil diff --git a/go.mod b/go.mod index 3fcb3a36..7e117820 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/go-debos/debos go 1.15 require ( - github.com/docker/go-units v0.4.0 - github.com/go-debos/fakemachine v0.0.2 + github.com/docker/go-units v0.5.0 + github.com/go-debos/fakemachine v0.0.4 github.com/google/uuid v1.3.0 github.com/jessevdk/go-flags v1.5.0 github.com/sjoerdsimons/ostree-go v0.0.0-20201014091107-8fae757256f8 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.2 gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 996dded0..93e780ea 100644 --- a/go.sum +++ b/go.sum @@ -1,28 +1,34 @@ github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/go-debos/fakemachine v0.0.2 h1:BTkzw+v5Qm7kqdo3xrwdDr058t5OV+LtOPR43CxLpbk= -github.com/go-debos/fakemachine v0.0.2/go.mod h1:bMLri+qSembsGcuXApNhaaBEEotnCMXVU2CfL2IUreY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/go-debos/fakemachine v0.0.4 h1:XZIDHjovqXS0wO5XNGZn0grqdFB+IJzRl9oodv/OmQU= +github.com/go-debos/fakemachine v0.0.4/go.mod h1:b2FvxH5PpsHHDpfaXDglLD/r7kyIAp2JslxSxb0thUU= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.3 h1:wmfu2iqj9q22SyMINp1uQ8C2/V4M1phJdmH9fG4nba0= +github.com/klauspost/compress v1.15.3/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sjoerdsimons/ostree-go v0.0.0-20201014091107-8fae757256f8 h1:fLxnJNJ++tkunS7BATed+mFqhA8KZYG7kT+WYEarYU4= github.com/sjoerdsimons/ostree-go v0.0.0-20201014091107-8fae757256f8/go.mod h1:f9gMvY6srFTBixsEIcm3zd3q7Y6btDDE4DKYtn8yDII= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/surma/gocpio v1.1.0 h1:RUWT+VqJ8GSodSv7Oh5xjIxy7r24CV1YvothHFfPxcQ= github.com/surma/gocpio v1.1.0/go.mod h1:zaLNaN+EDnfSnNdWPJJf9OZxWF817w5dt8JNzF9LCVI= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -32,5 +38,6 @@ gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124 h1:aPcd9iB gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124/go.mod h1:6LXpUYtVsrx91XiupFRJ8jVKOqLZf5PrbEVSGHta/84= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/apertis/apertis-archive-keyring.gpg b/tests/apertis/apertis-archive-keyring.gpg new file mode 100644 index 00000000..4a126de5 Binary files /dev/null and b/tests/apertis/apertis-archive-keyring.gpg differ diff --git a/tests/apertis/test.yaml b/tests/apertis/test.yaml new file mode 100644 index 00000000..4cde8606 --- /dev/null +++ b/tests/apertis/test.yaml @@ -0,0 +1,20 @@ +--- +# Test building a non-debian distribution such as apertis to ensure +# bootstrapping suites that debootstrap won't internally know about works +{{- $architecture := or .architecture "amd64"}} +architecture: {{$architecture}} + +actions: + - action: debootstrap + suite: v2022 + components: + - target + mirror: https://repositories.apertis.org/apertis/ + variant: minbase + keyring-package: apertis-archive-keyring + keyring-file: apertis-archive-keyring.gpg + + - action: apt + description: Install some base packages + packages: + - procps diff --git a/tests/arch/mirrorlist b/tests/arch/mirrorlist new file mode 100644 index 00000000..495d8838 --- /dev/null +++ b/tests/arch/mirrorlist @@ -0,0 +1,3 @@ +Server = https://geo.mirror.pkgbuild.com/$repo/os/$arch +Server = https://mirror.rackspace.com/archlinux/$repo/os/$arch +Server = https://mirror.leaseweb.net/archlinux/$repo/os/$arch diff --git a/tests/arch/pacman.conf b/tests/arch/pacman.conf new file mode 100644 index 00000000..15cff0fc --- /dev/null +++ b/tests/arch/pacman.conf @@ -0,0 +1,101 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +#NoProgressBar +CheckSpace +#VerbosePkgLists +#ParallelDownloads = 5 + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +#[testing] +#Include = /etc/pacman.d/mirrorlist + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra] +Include = /etc/pacman.d/mirrorlist + +#[community-testing] +#Include = /etc/pacman.d/mirrorlist + +[community] +Include = /etc/pacman.d/mirrorlist + +# If you want to run 32 bit applications on your x86_64 system, +# enable the multilib repositories as required here. + +#[multilib-testing] +#Include = /etc/pacman.d/mirrorlist + +#[multilib] +#Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Required DatabaseTrustedOnly +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs diff --git a/tests/arch/test.yaml b/tests/arch/test.yaml new file mode 100644 index 00000000..beae3312 --- /dev/null +++ b/tests/arch/test.yaml @@ -0,0 +1,13 @@ +--- +{{- $architecture := or .architecture "amd64"}} +architecture: {{$architecture}} + +actions: + - action: pacstrap + config: pacman.conf + mirror: mirrorlist + + - action: pacman + description: Install some base packages + packages: + - procps diff --git a/tests/overlay-non-existent-destination/overlay-non-existent-destination.yaml b/tests/overlay-non-existent-destination/overlay-non-existent-destination.yaml new file mode 100644 index 00000000..473a42da --- /dev/null +++ b/tests/overlay-non-existent-destination/overlay-non-existent-destination.yaml @@ -0,0 +1,17 @@ +architecture: amd64 + +actions: + # This overlay action is expected to error out here because the destination + # doesn't exist in the filesystem. + - action: overlay + description: Overlay file into a non-existent destination + source: overlay-non-existent-destination.yaml + destination: /this/path/does/not/exist/overlay-non-existent-destination.yaml + + - action: run + description: Check if path exists + command: "[ -e /this/path/does/not/exist/overlay-non-existent-destination.yaml ] || exit 1" + + - action: run + postprocess: true + command: echo Test diff --git a/tests/partitioning/test.yaml b/tests/partitioning/test.yaml index b416178d..3a03bfc2 100644 --- a/tests/partitioning/test.yaml +++ b/tests/partitioning/test.yaml @@ -26,4 +26,4 @@ actions: - action: run description: Compare expected and actual chroot: false - command: diff -u ${RECIPEDIR}/expected.json ${RECIPEDIR}/actual.json + command: bash -c 'diff -u <(jq . ${RECIPEDIR}/expected.json) <(jq . ${RECIPEDIR}/actual.json)'