From 25b27095952cac0f32c4bfd0fec73b9e74f3a085 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 18 Oct 2024 16:26:18 -0400 Subject: [PATCH 1/7] feat: add git to ubuntu-22.04 release --- slices/git.yaml | 177 ++++++++++++++++++ slices/libcurl3-gnutls.yaml | 28 +++ tests/spread/integration/git/task.yaml | 30 +++ .../integration/git/test-git_core-bins.sh | 11 ++ .../integration/git/test-git_net-bins.sh | 3 + .../integration/libcurl3-gnutls/task.yaml | 9 + 6 files changed, 258 insertions(+) create mode 100644 slices/git.yaml create mode 100644 slices/libcurl3-gnutls.yaml create mode 100644 tests/spread/integration/git/task.yaml create mode 100755 tests/spread/integration/git/test-git_core-bins.sh create mode 100755 tests/spread/integration/git/test-git_net-bins.sh create mode 100644 tests/spread/integration/libcurl3-gnutls/task.yaml diff --git a/slices/git.yaml b/slices/git.yaml new file mode 100644 index 000000000..b497245dd --- /dev/null +++ b/slices/git.yaml @@ -0,0 +1,177 @@ +package: git +# Note: These slices do not currently include gitweb. + +essential: + - git_copyright + +slices: + bins: + # Pseudo slice to just get all binaries. + essential: + - git_core-bins + - git_net-bins + + core-bins: + essential: + # libc6 libcurl3-gnutls libexpat1 libpcre2-8-0 zlib1g perl liberror-perl git-man + - libc6_libs + - libpcre2-8-0_libs + - zlib1g_libs + # While CA certificates aren't necessarily needed for git, installing + contents: + /usr/bin/git: + /usr/bin/git-receive-pack: + /usr/bin/git-shell: + /usr/bin/git-upload-archive: + /usr/bin/git-upload-pack: + /usr/lib/git-core/git: + /usr/lib/git-core/git-add*: + /usr/lib/git-core/git-annotate: + /usr/lib/git-core/git-apply: + /usr/lib/git-core/git-archive: + /usr/lib/git-core/git-bisect*: + /usr/lib/git-core/git-blame: + /usr/lib/git-core/git-branch: + /usr/lib/git-core/git-bugreport: + /usr/lib/git-core/git-bundle: + /usr/lib/git-core/git-cat-file: + /usr/lib/git-core/git-check-*: + /usr/lib/git-core/git-checkout*: + /usr/lib/git-core/git-cherry*: + /usr/lib/git-core/git-clean: + /usr/lib/git-core/git-clone: + /usr/lib/git-core/git-column: + /usr/lib/git-core/git-commit*: + /usr/lib/git-core/git-config: + /usr/lib/git-core/git-count-objects: + /usr/lib/git-core/git-credential*: + /usr/lib/git-core/git-describe: + /usr/lib/git-core/git-diff: + /usr/lib/git-core/git-diff-*: + /usr/lib/git-core/git-difftool: + /usr/lib/git-core/git-difftool--helper: + /usr/lib/git-core/git-env--helper: + /usr/lib/git-core/git-fast-*: + /usr/lib/git-core/git-fetch*: + /usr/lib/git-core/git-filter-branch: + /usr/lib/git-core/git-fmt-merge-msg: + /usr/lib/git-core/git-for-each-*: + /usr/lib/git-core/git-format-patch: + /usr/lib/git-core/git-fsck*: + /usr/lib/git-core/git-gc: + /usr/lib/git-core/git-get-tar-commit-id: + /usr/lib/git-core/git-grep: + /usr/lib/git-core/git-hash-object: + /usr/lib/git-core/git-help: + /usr/lib/git-core/git-index-pack: + /usr/lib/git-core/git-init: + /usr/lib/git-core/git-init-db: + /usr/lib/git-core/git-interpret-trailers: + /usr/lib/git-core/git-log: + /usr/lib/git-core/git-ls-*: + /usr/lib/git-core/git-mailinfo: + /usr/lib/git-core/git-mailsplit: + /usr/lib/git-core/git-maintenance: + /usr/lib/git-core/git-merge: + /usr/lib/git-core/git-merge-base: + /usr/lib/git-core/git-merge-file: + /usr/lib/git-core/git-merge-index: + /usr/lib/git-core/git-merge-octopus: + /usr/lib/git-core/git-merge-one-file: + /usr/lib/git-core/git-merge-ours: + /usr/lib/git-core/git-merge-recursive: + /usr/lib/git-core/git-merge-resolve: + /usr/lib/git-core/git-merge-subtree: + /usr/lib/git-core/git-merge-tree: + /usr/lib/git-core/git-mktag: + /usr/lib/git-core/git-mktree: + /usr/lib/git-core/git-multi-pack-index: + /usr/lib/git-core/git-mv: + /usr/lib/git-core/git-name-rev: + /usr/lib/git-core/git-notes: + /usr/lib/git-core/git-pack-*: + /usr/lib/git-core/git-patch-id: + /usr/lib/git-core/git-prune: + /usr/lib/git-core/git-prune-packed: + /usr/lib/git-core/git-pull: + /usr/lib/git-core/git-push: + /usr/lib/git-core/git-quiltimport: + /usr/lib/git-core/git-range-diff: + /usr/lib/git-core/git-read-tree: + /usr/lib/git-core/git-rebase: + /usr/lib/git-core/git-receive-pack: + /usr/lib/git-core/git-reflog: + /usr/lib/git-core/git-remote: + /usr/lib/git-core/git-remote-ext: + /usr/lib/git-core/git-remote-fd: + /usr/lib/git-core/git-repack: + /usr/lib/git-core/git-replace: + /usr/lib/git-core/git-request-pull: + /usr/lib/git-core/git-rerere: + /usr/lib/git-core/git-reset: + /usr/lib/git-core/git-restore: + /usr/lib/git-core/git-rev-list: + /usr/lib/git-core/git-rev-parse: + /usr/lib/git-core/git-rm: + /usr/lib/git-core/git-send-pack: + /usr/lib/git-core/git-sh-i18n: + /usr/lib/git-core/git-sh-i18n--envsubst: + /usr/lib/git-core/git-sh-prompt: + /usr/lib/git-core/git-sh-setup: + /usr/lib/git-core/git-shell: + /usr/lib/git-core/git-shortlog: + /usr/lib/git-core/git-show: + /usr/lib/git-core/git-show-branch: + /usr/lib/git-core/git-show-index: + /usr/lib/git-core/git-show-ref: + /usr/lib/git-core/git-sparse-checkout: + /usr/lib/git-core/git-stage: + /usr/lib/git-core/git-stash: + /usr/lib/git-core/git-status: + /usr/lib/git-core/git-stripspace: + /usr/lib/git-core/git-submodule: + /usr/lib/git-core/git-submodule--helper: + /usr/lib/git-core/git-subtree: + /usr/lib/git-core/git-switch: + /usr/lib/git-core/git-symbolic-ref: + /usr/lib/git-core/git-tag: + /usr/lib/git-core/git-unpack-file: + /usr/lib/git-core/git-unpack-objects: + /usr/lib/git-core/git-update-index: + /usr/lib/git-core/git-update-ref: + /usr/lib/git-core/git-update-server-info: + /usr/lib/git-core/git-upload-archive: + /usr/lib/git-core/git-upload-pack: + /usr/lib/git-core/git-var: + /usr/lib/git-core/git-verify-commit: + /usr/lib/git-core/git-verify-pack: + /usr/lib/git-core/git-verify-tag: + /usr/lib/git-core/git-whatchanged: + /usr/lib/git-core/git-worktree: + /usr/lib/git-core/git-write-tree: + + daemon: + essential: + - git_core-bins + contents: + /usr/lib/git-core/git-daemon: + + net-bins: + # Binaries used with networking. Not needed if git is only being used for local + # items. + essential: + - git_core-bins + - libcurl3-gnutls_libs + - libexpat1_libs + # While ca-certificates_data isn't strictly a dependency, it is generally needed if you use https. + contents: + /usr/bin/git-receive-pack: + /usr/bin/git-upload-archive: + /usr/bin/git-upload-pack: + /usr/lib/git-core/git-http-*: + /usr/lib/git-core/git-imap-send: + /usr/lib/git-core/git-remote-*: + + copyright: + contents: + /usr/share/doc/git/copyright: diff --git a/slices/libcurl3-gnutls.yaml b/slices/libcurl3-gnutls.yaml new file mode 100644 index 000000000..09ddddf49 --- /dev/null +++ b/slices/libcurl3-gnutls.yaml @@ -0,0 +1,28 @@ +package: libcurl3-gnutls + +essential: + - libcurl3-gnutls_copyright + +slices: + libs: + essential: + - libbrotli1_libs + - libc6_libs + - libgnutls30_libs + - libgssapi-krb5-2_libs + - libidn2-0_libs + - libldap-2.5-0_libs + - libnettle8_libs + - libnghttp2-14_libs + - libpsl5_libs + - librtmp1_libs + - libssh-4_libs + - libzstd1_libs + - zlib1g_libs + contents: + /usr/lib/*-linux-*/libcurl-gnutls.so.3: + /usr/lib/*-linux-*/libcurl-gnutls.so.4*: + + copyright: + contents: + /usr/share/doc/libcurl3-gnutls/copyright: diff --git a/tests/spread/integration/git/task.yaml b/tests/spread/integration/git/task.yaml new file mode 100644 index 000000000..390d674d5 --- /dev/null +++ b/tests/spread/integration/git/task.yaml @@ -0,0 +1,30 @@ +summary: Integration tests for git + +environment: + EXTRA_SLICES: "" # No extra slices by default. + SLICE/core_bins: git_core-bins + SLICE/net_bins: git_net-bins + EXTRA_SLICES/net_bins: ca-certificates_data + SLICE/gitweb: git_gitweb + EXTRA_SLICES/gitweb: python3.10_standard curl_bins + +execute: | + rootfs="$(install-slices $SLICE $EXTRA_SLICES)" + + # Basic configuration of extra things not in the packages. + # Git uses /dev/null + mkdir -p "${rootfs}/dev" + mount --bind /dev "${rootfs}/dev" + + # Make domain resolution work + mkdir -p "${rootfs}/etc" + cp /etc/resolv.conf "${rootfs}/etc/resolv.conf" + + # Give root a homedir in the fs (for git config) + mkdir -p "${rootfs}/root" + + # Run the test script for this slice. + "./test-${SLICE}.sh" "${rootfs}" + + # Clean up + umount "${rootfs}/dev" diff --git a/tests/spread/integration/git/test-git_core-bins.sh b/tests/spread/integration/git/test-git_core-bins.sh new file mode 100755 index 000000000..4b23ce094 --- /dev/null +++ b/tests/spread/integration/git/test-git_core-bins.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Basic tests for git core binaries. +rootfs="${1}" + +chroot "${rootfs}" git init +echo "Test" > "${rootfs}/test.txt" +chroot "${rootfs}" git add test.txt +chroot "${rootfs}" git config --global user.email "root@localhost" +chroot "${rootfs}" git config --global user.name "Test Runner" +chroot "${rootfs}" git commit -m test +[[ $(chroot "${rootfs}" git ls-files) == "test.txt" ]] diff --git a/tests/spread/integration/git/test-git_net-bins.sh b/tests/spread/integration/git/test-git_net-bins.sh new file mode 100755 index 000000000..91c44c8b7 --- /dev/null +++ b/tests/spread/integration/git/test-git_net-bins.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Test for network activity. +chroot "${1}" git clone --branch ubuntu-22.04 --depth 1 https://github.com/canonical/chisel-releases diff --git a/tests/spread/integration/libcurl3-gnutls/task.yaml b/tests/spread/integration/libcurl3-gnutls/task.yaml new file mode 100644 index 000000000..d2e37692b --- /dev/null +++ b/tests/spread/integration/libcurl3-gnutls/task.yaml @@ -0,0 +1,9 @@ +summary: Integration tests for libcurl3-gnutls + +execute: | + rootfs="$(install-slices libcurl3-gnutls_libs)" + + arch=$(uname -m) + + test -f ${rootfs}/usr/lib/${arch}-linux-gnu/libcurl-gnutls.so.3 + test -f ${rootfs}/usr/lib/${arch}-linux-gnu/libcurl-gnutls.so.4 From a7cd9212db8bc4750c88b5309568e5ea5c9ce2a7 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 14 Nov 2024 10:12:49 -0500 Subject: [PATCH 2/7] fix: sync to current state of #282 Synchronises this branch to the current state of the in-progress Ubuntu 24.04 slice PR. This is incomplete because there are currently questions about the tests. --- slices/git.yaml | 177 +++--------------- slices/libcurl3-gnutls.yaml | 3 +- slices/python3-pyftpdlib.yaml | 15 ++ tests/spread/integration/git/task.yaml | 166 +++++++++++++--- .../integration/git/test-git_core-bins.sh | 11 -- .../integration/git/test-git_net-bins.sh | 3 - 6 files changed, 187 insertions(+), 188 deletions(-) create mode 100644 slices/python3-pyftpdlib.yaml delete mode 100755 tests/spread/integration/git/test-git_core-bins.sh delete mode 100755 tests/spread/integration/git/test-git_net-bins.sh diff --git a/slices/git.yaml b/slices/git.yaml index b497245dd..08ab26c10 100644 --- a/slices/git.yaml +++ b/slices/git.yaml @@ -1,176 +1,53 @@ package: git -# Note: These slices do not currently include gitweb. essential: - git_copyright slices: bins: - # Pseudo slice to just get all binaries. + # basic functionality including: + # - init + # - config + # - add + # - commit + # - status + # - log + # - checkout + # - blame + # - diff + # - merge essential: - - git_core-bins - - git_net-bins - - core-bins: - essential: - # libc6 libcurl3-gnutls libexpat1 libpcre2-8-0 zlib1g perl liberror-perl git-man - libc6_libs - libpcre2-8-0_libs - zlib1g_libs - # While CA certificates aren't necessarily needed for git, installing contents: - /usr/bin/git: - /usr/bin/git-receive-pack: - /usr/bin/git-shell: - /usr/bin/git-upload-archive: - /usr/bin/git-upload-pack: + /usr/bin/git: {symlink: /usr/lib/git-core/git} /usr/lib/git-core/git: - /usr/lib/git-core/git-add*: - /usr/lib/git-core/git-annotate: - /usr/lib/git-core/git-apply: - /usr/lib/git-core/git-archive: - /usr/lib/git-core/git-bisect*: - /usr/lib/git-core/git-blame: - /usr/lib/git-core/git-branch: - /usr/lib/git-core/git-bugreport: - /usr/lib/git-core/git-bundle: - /usr/lib/git-core/git-cat-file: - /usr/lib/git-core/git-check-*: - /usr/lib/git-core/git-checkout*: - /usr/lib/git-core/git-cherry*: - /usr/lib/git-core/git-clean: - /usr/lib/git-core/git-clone: - /usr/lib/git-core/git-column: - /usr/lib/git-core/git-commit*: - /usr/lib/git-core/git-config: - /usr/lib/git-core/git-count-objects: - /usr/lib/git-core/git-credential*: - /usr/lib/git-core/git-describe: - /usr/lib/git-core/git-diff: - /usr/lib/git-core/git-diff-*: - /usr/lib/git-core/git-difftool: - /usr/lib/git-core/git-difftool--helper: - /usr/lib/git-core/git-env--helper: - /usr/lib/git-core/git-fast-*: - /usr/lib/git-core/git-fetch*: - /usr/lib/git-core/git-filter-branch: - /usr/lib/git-core/git-fmt-merge-msg: - /usr/lib/git-core/git-for-each-*: - /usr/lib/git-core/git-format-patch: - /usr/lib/git-core/git-fsck*: - /usr/lib/git-core/git-gc: - /usr/lib/git-core/git-get-tar-commit-id: - /usr/lib/git-core/git-grep: - /usr/lib/git-core/git-hash-object: - /usr/lib/git-core/git-help: - /usr/lib/git-core/git-index-pack: - /usr/lib/git-core/git-init: - /usr/lib/git-core/git-init-db: - /usr/lib/git-core/git-interpret-trailers: - /usr/lib/git-core/git-log: - /usr/lib/git-core/git-ls-*: - /usr/lib/git-core/git-mailinfo: - /usr/lib/git-core/git-mailsplit: - /usr/lib/git-core/git-maintenance: - /usr/lib/git-core/git-merge: - /usr/lib/git-core/git-merge-base: - /usr/lib/git-core/git-merge-file: - /usr/lib/git-core/git-merge-index: - /usr/lib/git-core/git-merge-octopus: - /usr/lib/git-core/git-merge-one-file: - /usr/lib/git-core/git-merge-ours: - /usr/lib/git-core/git-merge-recursive: - /usr/lib/git-core/git-merge-resolve: - /usr/lib/git-core/git-merge-subtree: - /usr/lib/git-core/git-merge-tree: - /usr/lib/git-core/git-mktag: - /usr/lib/git-core/git-mktree: - /usr/lib/git-core/git-multi-pack-index: - /usr/lib/git-core/git-mv: - /usr/lib/git-core/git-name-rev: - /usr/lib/git-core/git-notes: - /usr/lib/git-core/git-pack-*: - /usr/lib/git-core/git-patch-id: - /usr/lib/git-core/git-prune: - /usr/lib/git-core/git-prune-packed: - /usr/lib/git-core/git-pull: - /usr/lib/git-core/git-push: - /usr/lib/git-core/git-quiltimport: - /usr/lib/git-core/git-range-diff: - /usr/lib/git-core/git-read-tree: - /usr/lib/git-core/git-rebase: - /usr/lib/git-core/git-receive-pack: - /usr/lib/git-core/git-reflog: - /usr/lib/git-core/git-remote: - /usr/lib/git-core/git-remote-ext: - /usr/lib/git-core/git-remote-fd: - /usr/lib/git-core/git-repack: - /usr/lib/git-core/git-replace: - /usr/lib/git-core/git-request-pull: - /usr/lib/git-core/git-rerere: - /usr/lib/git-core/git-reset: - /usr/lib/git-core/git-restore: - /usr/lib/git-core/git-rev-list: - /usr/lib/git-core/git-rev-parse: - /usr/lib/git-core/git-rm: - /usr/lib/git-core/git-send-pack: - /usr/lib/git-core/git-sh-i18n: - /usr/lib/git-core/git-sh-i18n--envsubst: - /usr/lib/git-core/git-sh-prompt: - /usr/lib/git-core/git-sh-setup: - /usr/lib/git-core/git-shell: - /usr/lib/git-core/git-shortlog: - /usr/lib/git-core/git-show: - /usr/lib/git-core/git-show-branch: - /usr/lib/git-core/git-show-index: - /usr/lib/git-core/git-show-ref: - /usr/lib/git-core/git-sparse-checkout: - /usr/lib/git-core/git-stage: - /usr/lib/git-core/git-stash: - /usr/lib/git-core/git-status: - /usr/lib/git-core/git-stripspace: - /usr/lib/git-core/git-submodule: - /usr/lib/git-core/git-submodule--helper: - /usr/lib/git-core/git-subtree: - /usr/lib/git-core/git-switch: - /usr/lib/git-core/git-symbolic-ref: - /usr/lib/git-core/git-tag: - /usr/lib/git-core/git-unpack-file: - /usr/lib/git-core/git-unpack-objects: - /usr/lib/git-core/git-update-index: - /usr/lib/git-core/git-update-ref: - /usr/lib/git-core/git-update-server-info: - /usr/lib/git-core/git-upload-archive: - /usr/lib/git-core/git-upload-pack: - /usr/lib/git-core/git-var: - /usr/lib/git-core/git-verify-commit: - /usr/lib/git-core/git-verify-pack: - /usr/lib/git-core/git-verify-tag: - /usr/lib/git-core/git-whatchanged: - /usr/lib/git-core/git-worktree: - /usr/lib/git-core/git-write-tree: daemon: + # support for git daemon essential: - - git_core-bins + - git_bins contents: /usr/lib/git-core/git-daemon: - net-bins: - # Binaries used with networking. Not needed if git is only being used for local - # items. + http-clone: + # requirements for cloning over http `git clone https://`: essential: - - git_core-bins + - git_bins - libcurl3-gnutls_libs - libexpat1_libs - # While ca-certificates_data isn't strictly a dependency, it is generally needed if you use https. + - libpcre2-8-0_libs + contents: + /usr/lib/git-core/git-remote-http: + /usr/lib/git-core/git-remote-https: # link pointing to git-remote-http + + ftp-clone: + # requirements for cloning over ftp `git clone ftp://`: + essential: + - git_http-clone contents: - /usr/bin/git-receive-pack: - /usr/bin/git-upload-archive: - /usr/bin/git-upload-pack: - /usr/lib/git-core/git-http-*: - /usr/lib/git-core/git-imap-send: - /usr/lib/git-core/git-remote-*: + /usr/lib/git-core/git-remote-ftp: copyright: contents: diff --git a/slices/libcurl3-gnutls.yaml b/slices/libcurl3-gnutls.yaml index 09ddddf49..d7001b90c 100644 --- a/slices/libcurl3-gnutls.yaml +++ b/slices/libcurl3-gnutls.yaml @@ -20,8 +20,7 @@ slices: - libzstd1_libs - zlib1g_libs contents: - /usr/lib/*-linux-*/libcurl-gnutls.so.3: - /usr/lib/*-linux-*/libcurl-gnutls.so.4*: + /usr/lib/*-linux-*/libcurl-gnutls.so.*: copyright: contents: diff --git a/slices/python3-pyftpdlib.yaml b/slices/python3-pyftpdlib.yaml new file mode 100644 index 000000000..54e71952e --- /dev/null +++ b/slices/python3-pyftpdlib.yaml @@ -0,0 +1,15 @@ +package: python3-pyftpdlib + +essential: + - python3-pyftpdlib_copyright + +slices: + libs: + essential: + - python3.10_standard + contents: + /usr/lib/python3/dist-packages/pyftpdlib/*.py: + + copyright: + contents: + /usr/share/doc/python3-pyftpdlib/copyright: diff --git a/tests/spread/integration/git/task.yaml b/tests/spread/integration/git/task.yaml index 390d674d5..d6dc9a019 100644 --- a/tests/spread/integration/git/task.yaml +++ b/tests/spread/integration/git/task.yaml @@ -1,30 +1,152 @@ summary: Integration tests for git -environment: - EXTRA_SLICES: "" # No extra slices by default. - SLICE/core_bins: git_core-bins - SLICE/net_bins: git_net-bins - EXTRA_SLICES/net_bins: ca-certificates_data - SLICE/gitweb: git_gitweb - EXTRA_SLICES/gitweb: python3.10_standard curl_bins - execute: | - rootfs="$(install-slices $SLICE $EXTRA_SLICES)" - # Basic configuration of extra things not in the packages. - # Git uses /dev/null - mkdir -p "${rootfs}/dev" - mount --bind /dev "${rootfs}/dev" + function setup(){ + + mkdir -p "$1/dev/" + mkdir -p "$1/etc/" + + # fake essentials + touch "$1/dev/null" + head -c 500 /dev/urandom > "$1/dev/urandom" + cp /etc/resolv.conf $1/etc/ + + # link busybox + pushd "$1/usr/bin/" + for TARGET in ash mkdir cat sleep nc + do + ln -sf busybox $TARGET + done + popd + + } + + rootfs_bins=$(install-slices git_bins busybox_bins) + setup "$rootfs_bins" + + chroot "$rootfs_bins" ash << EOF + set -ex + + mkdir repo + cd repo + + git init + + git config user.email "you@example.com" + git config user.name "Your Name" + git config core.pager cat + + echo hello_world > test_file + git add test_file + git commit -m "hello hello_world" + + git status + git log + + git blame -L 1,1 test_file + + git checkout -b test + + echo foo_bar >> test_file + git add test_file + git commit -m "hello foo_bar" + + git diff master + + git branch + git checkout master + git merge test + + EOF + + + # Testing for http-clone + rootfs_http=$(install-slices git_http-clone busybox_bins ca-certificates_data) + setup "$rootfs_http" + + chroot "$rootfs_http" ash << EOF + set -ex + + # TODO: should we replace this with locally hosted server? + git clone https://github.com/cmatsuoka/figlet.git "$rootfs_bins/figlet" + + EOF + + # Testing for daemon clone + rootfs_daemon=$(install-slices git_daemon busybox_bins) + setup "$rootfs_daemon" + + chroot "$rootfs_daemon" ash << EOF + set -xe + + # create a test repo + repo_name=daemon-src + repo_path=/srv/git/\$repo_name + mkdir -p \$repo_path + cd \$repo_path + + git init . + echo foo_bar >> test_file + git add test_file + git config user.email "you@example.com" + git config user.name "Your Name" + git commit -m "hello foo_bar" + + # start the daemon in the background + cd .. + /usr/lib/git-core/git-daemon \\ + --base-path=. \\ + --reuseaddr \\ + --listen=127.0.0.1 \\ + --verbose \\ + --export-all & + daemon_pid=\$! + + # wait for server port to become available + while ! echo -n | nc 127.0.0.1 9418 + do + sleep 1 + done + + # clone the repo + git clone git://localhost/\$repo_name daemon-dst + + # cleanup + kill \$daemon_pid + + EOF + + # Testing for ftp-clone + rootfs_ftp=$(install-slices git_ftp-clone busybox_bins python3-pyftpdlib_libs) + setup "$rootfs_ftp" + + chroot "$rootfs_ftp" ash << EOF + set -xe + + # create a test repo + repo_name=ftp-src + repo_path=/srv/git/\$repo_name + mkdir -p \$repo_path + cd \$repo_path + + git init --bare + git update-server-info + + cd .. + /usr/bin/python3.12 -m pyftpdlib -V & + pyftpdlib_pid=\$! - # Make domain resolution work - mkdir -p "${rootfs}/etc" - cp /etc/resolv.conf "${rootfs}/etc/resolv.conf" + # wait for server port to become available + while ! echo -n | nc 127.0.0.1 2121 + do + sleep 1 + done - # Give root a homedir in the fs (for git config) - mkdir -p "${rootfs}/root" + # clone the repo + git clone ftp://localhost:2121/\$repo_name ftp-dst - # Run the test script for this slice. - "./test-${SLICE}.sh" "${rootfs}" + # cleanup + kill \$pyftpdlib_pid - # Clean up - umount "${rootfs}/dev" + EOF diff --git a/tests/spread/integration/git/test-git_core-bins.sh b/tests/spread/integration/git/test-git_core-bins.sh deleted file mode 100755 index 4b23ce094..000000000 --- a/tests/spread/integration/git/test-git_core-bins.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Basic tests for git core binaries. -rootfs="${1}" - -chroot "${rootfs}" git init -echo "Test" > "${rootfs}/test.txt" -chroot "${rootfs}" git add test.txt -chroot "${rootfs}" git config --global user.email "root@localhost" -chroot "${rootfs}" git config --global user.name "Test Runner" -chroot "${rootfs}" git commit -m test -[[ $(chroot "${rootfs}" git ls-files) == "test.txt" ]] diff --git a/tests/spread/integration/git/test-git_net-bins.sh b/tests/spread/integration/git/test-git_net-bins.sh deleted file mode 100755 index 91c44c8b7..000000000 --- a/tests/spread/integration/git/test-git_net-bins.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -# Test for network activity. -chroot "${1}" git clone --branch ubuntu-22.04 --depth 1 https://github.com/canonical/chisel-releases From 59a2bdbeefc6877381e7385d3ff116fd737d3e86 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 14 Nov 2024 11:46:36 -0500 Subject: [PATCH 3/7] fix: reorganise git tests --- tests/spread/integration/git/task.yaml | 200 +++++++++++-------------- 1 file changed, 85 insertions(+), 115 deletions(-) diff --git a/tests/spread/integration/git/task.yaml b/tests/spread/integration/git/task.yaml index d6dc9a019..e16af29de 100644 --- a/tests/spread/integration/git/task.yaml +++ b/tests/spread/integration/git/task.yaml @@ -1,9 +1,19 @@ summary: Integration tests for git -execute: | +environment: + EXTRA_SLICES: "" # No extra slices by default. + RUN/bins: test_bins + SLICE/bins: git_bins + RUN/http_clone: test_http_clone + SLICE/http_clone: git_http-clone + EXTRA_SLICES/http_clone: ca-certificates_data + RUN/ftp_clone: test_ftp_clone + SLICE/ftp_clone: git_ftp-clone + RUN/daemon_clone: test_daemon_clone + SLICE/daemon_clone: git_daemon +execute: | function setup(){ - mkdir -p "$1/dev/" mkdir -p "$1/etc/" @@ -11,142 +21,102 @@ execute: | touch "$1/dev/null" head -c 500 /dev/urandom > "$1/dev/urandom" cp /etc/resolv.conf $1/etc/ + mkdir "$1/root" - # link busybox - pushd "$1/usr/bin/" - for TARGET in ash mkdir cat sleep nc - do - ln -sf busybox $TARGET - done - popd - + # Configure git inside the chroot. + chroot "$1" git config --global user.email "root@localhost" + chroot "$1" git config --global user.name "Test Runner" } - rootfs_bins=$(install-slices git_bins busybox_bins) - setup "$rootfs_bins" - - chroot "$rootfs_bins" ash << EOF - set -ex - - mkdir repo - cd repo - - git init - - git config user.email "you@example.com" - git config user.name "Your Name" - git config core.pager cat - - echo hello_world > test_file - git add test_file - git commit -m "hello hello_world" - - git status - git log - - git blame -L 1,1 test_file - - git checkout -b test - - echo foo_bar >> test_file - git add test_file - git commit -m "hello foo_bar" - - git diff master - - git branch - git checkout master - git merge test - - EOF + function test_bins(){ + chroot "$1" git init + echo "Test" > "$1/test.txt" + chroot "$1" git add test.txt + chroot "$1" git commit -m 'test commit' + chroot "$1" git --no-pager status + chroot "$1" git --no-pager log - # Testing for http-clone - rootfs_http=$(install-slices git_http-clone busybox_bins ca-certificates_data) - setup "$rootfs_http" + chroot "$1" git blame -L 1,1 test.txt + [[ $(chroot "$1" git ls-files) == "test.txt" ]] - chroot "$rootfs_http" ash << EOF - set -ex + chroot "$1" git checkout -b test - # TODO: should we replace this with locally hosted server? - git clone https://github.com/cmatsuoka/figlet.git "$rootfs_bins/figlet" - - EOF - - # Testing for daemon clone - rootfs_daemon=$(install-slices git_daemon busybox_bins) - setup "$rootfs_daemon" + chroot "$1" git --no-pager diff master + chroot "$1" git branch + chroot "$1" git checkout master + chroot "$1" git merge test + } - chroot "$rootfs_daemon" ash << EOF - set -xe + function test_http_clone(){ + chroot "$1" git clone --depth 1 https://git.launchpad.net/ubuntu/+source/hello + } - # create a test repo - repo_name=daemon-src - repo_path=/srv/git/\$repo_name - mkdir -p \$repo_path - cd \$repo_path + function test_ftp_clone(){ + ftp_root=$(install-slices python3-pyftpdlib_libs busybox_bins) + mkdir $ftp_root/dev + mount --bind /dev "${ftp_root}/dev" + mkdir -p "$1/etc/" + cp /etc/resolv.conf $1/etc/ - git init . - echo foo_bar >> test_file - git add test_file - git config user.email "you@example.com" - git config user.name "Your Name" - git commit -m "hello foo_bar" + mkdir -p $ftp_root/srv/git/ftp-src - # start the daemon in the background - cd .. - /usr/lib/git-core/git-daemon \\ - --base-path=. \\ - --reuseaddr \\ - --listen=127.0.0.1 \\ - --verbose \\ - --export-all & - daemon_pid=\$! + pushd $ftp_root/srv/git/ftp-src + git init --bare + git update-server-info + popd + pushd $ftp_root/srv/git + chroot $ftp_root /usr/bin/python3.10 -m pyftpdlib -V -p 2121 -d /srv/git & + export pyftpdlib_pid=$! # wait for server port to become available - while ! echo -n | nc 127.0.0.1 9418 + while ! nc -z 127.0.0.1 2121 do + test -d /proc/$pyftpdlib_pid || (echo "pyftpdlib exited early"; exit 1) sleep 1 done - # clone the repo - git clone git://localhost/\$repo_name daemon-dst + # Test cloning the repo + chroot $1 git clone ftp://localhost:2121/ftp-src ftp-dst # cleanup - kill \$daemon_pid - - EOF - - # Testing for ftp-clone - rootfs_ftp=$(install-slices git_ftp-clone busybox_bins python3-pyftpdlib_libs) - setup "$rootfs_ftp" - - chroot "$rootfs_ftp" ash << EOF - set -xe + kill $pyftpdlib_pid + } - # create a test repo - repo_name=ftp-src - repo_path=/srv/git/\$repo_name - mkdir -p \$repo_path - cd \$repo_path + function test_daemon_clone(){ + mkdir -p $1/srv/git/daemon-src - git init --bare - git update-server-info + pushd $1/srv/git/daemon-src + git init + touch test-file + git add test-file + git config user.email "root@localhost" + git config user.name "Test Runner" + git commit -m "Hello daemon" + popd - cd .. - /usr/bin/python3.12 -m pyftpdlib -V & - pyftpdlib_pid=\$! + chroot "$1" /usr/lib/git-core/git-daemon \ + --base-path=/srv/git \ + --reuseaddr \ + --listen=127.0.0.1 \ + --verbose \ + --export-all & + daemon_pid=$! - # wait for server port to become available - while ! echo -n | nc 127.0.0.1 2121 - do - sleep 1 - done + # Wait for daemon to start + while ! nc -z 127.0.0.1 9418 + do + test -d /proc/$daemon_pid || (echo "daemon exited early"; exit 1) + sleep 1 + done - # clone the repo - git clone ftp://localhost:2121/\$repo_name ftp-dst + # Test cloning the repo + chroot $1 git clone git://localhost/daemon-src daemon-dst - # cleanup - kill \$pyftpdlib_pid + # Cleanup + kill $daemon_pid + } - EOF + rootfs=$(install-slices $SLICE $EXTRA_SLICES) + setup $rootfs + $RUN $rootfs From 69df699582094ffe7a5832a44b07ae5217ad54d1 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 15 Nov 2024 11:31:04 -0500 Subject: [PATCH 4/7] feat: integration test for python3-pyftpdlib --- .../integration/python3-pyftpdlib/task.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/spread/integration/python3-pyftpdlib/task.yaml diff --git a/tests/spread/integration/python3-pyftpdlib/task.yaml b/tests/spread/integration/python3-pyftpdlib/task.yaml new file mode 100644 index 000000000..7bdd4904a --- /dev/null +++ b/tests/spread/integration/python3-pyftpdlib/task.yaml @@ -0,0 +1,64 @@ +summary: Integration tests for python3-pyftpdlib + +execute: | + function wait_for_ftp(){ + sleep 0.3 + while ! nc -z 127.0.0.1 2121 + do + test -d /proc/$pyftpdlib_pid || (echo "pyftpdlib exited early"; exit 1) + sleep 1 + done + } + function cleanup(){ + while [[ -d /proc/$pyftpdlib_pid ]]; do + kill $pyftpdlib_pid || true + sleep 0.1 + done + } + # Still run the cleanup even on test failure. + trap cleanup EXIT + + rootfs="$(install-slices python3-pyftpdlib_libs)" + mkdir "${rootfs}/dev" + mount --bind /dev "${rootfs}/dev" + mkdir -p "${rootfs}"/srv/subdir + echo "Test file" > "${rootfs}"/srv/subdir/test-file + + # Basic install smoke test + + chroot "${rootfs}" python3.10 -m pyftpdlib --help + + # Anonymous hosting + + chroot "${rootfs}" /usr/bin/python3.10 -m pyftpdlib \ + --verbose \ + --interface=127.0.0.1 \ + --port=2121 \ + --directory=/srv & + export pyftpdlib_pid=$! + wait_for_ftp + + [[ "$(curl ftp://127.0.0.1:2121/subdir/test-file)" == "Test file" ]] + + cleanup + + # With login and file upload. + + chroot "${rootfs}" /usr/bin/python3.10 -m pyftpdlib \ + --verbose \ + --interface=127.0.0.1 \ + --port=2121 \ + --directory=/srv \ + --write \ + --username=testy \ + --password=testy & + export pyftpdlib_pid=$! + wait_for_ftp + + [[ "$(curl ftp://testy:testy@127.0.0.1:2121/subdir/test-file)" == "Test file" ]] + + echo "Test upload" > test-upload + curl -T test-upload ftp://testy:testy@127.0.0.1:2121/test-upload + [[ "$(cat "${rootfs}/srv/test-upload")" == "$(cat test-upload)" ]] + + cleanup From d9ba673af464ff667ea73183c0594aedea45c089 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 11 Dec 2024 19:31:52 -0500 Subject: [PATCH 5/7] fix: match the final version of the 24.04 slices --- slices/git.yaml | 27 ++--- tests/spread/integration/git/task.yaml | 139 ++++++++++++++++--------- 2 files changed, 102 insertions(+), 64 deletions(-) diff --git a/slices/git.yaml b/slices/git.yaml index 08ab26c10..d0cd3358f 100644 --- a/slices/git.yaml +++ b/slices/git.yaml @@ -5,17 +5,20 @@ essential: slices: bins: - # basic functionality including: - # - init - # - config + # Basic functionality including: # - add - # - commit - # - status - # - log - # - checkout # - blame + # - branch + # - checkout + # - commit + # - config # - diff + # - fetch + # - init + # - log # - merge + # - rev-list + # - status essential: - libc6_libs - libpcre2-8-0_libs @@ -31,8 +34,8 @@ slices: contents: /usr/lib/git-core/git-daemon: - http-clone: - # requirements for cloning over http `git clone https://`: + http-support: + # Adds support for cloning over HTTP e.g. git clone https://... essential: - git_bins - libcurl3-gnutls_libs @@ -42,10 +45,10 @@ slices: /usr/lib/git-core/git-remote-http: /usr/lib/git-core/git-remote-https: # link pointing to git-remote-http - ftp-clone: - # requirements for cloning over ftp `git clone ftp://`: + ftp-support: + # Adds support for cloning over FTP e.g. git clone ftp://... essential: - - git_http-clone + - git_http-support contents: /usr/lib/git-core/git-remote-ftp: diff --git a/tests/spread/integration/git/task.yaml b/tests/spread/integration/git/task.yaml index e16af29de..97950dbce 100644 --- a/tests/spread/integration/git/task.yaml +++ b/tests/spread/integration/git/task.yaml @@ -1,18 +1,31 @@ summary: Integration tests for git environment: - EXTRA_SLICES: "" # No extra slices by default. + EXTRA_SLICES: "" # No extra slices by default. + # test git_bins RUN/bins: test_bins SLICE/bins: git_bins - RUN/http_clone: test_http_clone - SLICE/http_clone: git_http-clone - EXTRA_SLICES/http_clone: ca-certificates_data - RUN/ftp_clone: test_ftp_clone - SLICE/ftp_clone: git_ftp-clone - RUN/daemon_clone: test_daemon_clone - SLICE/daemon_clone: git_daemon + # test git_http-support + RUN/http_support: test_http_support + SLICE/http_support: git_http-support + EXTRA_SLICES/http_support: ca-certificates_data + # test git_ftp-support + RUN/ftp_support: test_ftp_support + SLICE/ftp_support: git_ftp-support + # test git_daemon + RUN/daemon_support: test_daemon_support + SLICE/daemon_support: git_daemon execute: | + + git_rootfs="/" + function chroot_git(){ + # execute git within chroot. We use -C to cd since + # we have no $SHELL to navigate with. + repo="$(realpath -e --relative-to="$git_rootfs" "$PWD")" + chroot "$git_rootfs" /usr/bin/git -C "$repo" "$@" + } + function setup(){ mkdir -p "$1/dev/" mkdir -p "$1/etc/" @@ -23,79 +36,95 @@ execute: | cp /etc/resolv.conf $1/etc/ mkdir "$1/root" - # Configure git inside the chroot. - chroot "$1" git config --global user.email "root@localhost" - chroot "$1" git config --global user.name "Test Runner" + # configure + git_rootfs="$1" + pushd "$1" + + } + + function cleanup(){ + popd } function test_bins(){ - chroot "$1" git init + chroot_git init + chroot_git config user.email "root@localhost" + chroot_git config user.name "Test Runner" + echo "Test" > "$1/test.txt" - chroot "$1" git add test.txt - chroot "$1" git commit -m 'test commit' + chroot_git add test.txt + chroot_git commit -m 'test commit' - chroot "$1" git --no-pager status - chroot "$1" git --no-pager log + chroot_git --no-pager status + chroot_git --no-pager log - chroot "$1" git blame -L 1,1 test.txt - [[ $(chroot "$1" git ls-files) == "test.txt" ]] + chroot_git blame -L 1,1 test.txt + [[ $(chroot_git ls-files) == "test.txt" ]] - chroot "$1" git checkout -b test + chroot_git checkout -b test - chroot "$1" git --no-pager diff master - chroot "$1" git branch - chroot "$1" git checkout master - chroot "$1" git merge test + chroot_git --no-pager diff master + chroot_git branch + chroot_git checkout master + chroot_git merge test } - function test_http_clone(){ - chroot "$1" git clone --depth 1 https://git.launchpad.net/ubuntu/+source/hello - } + function test_http_support(){ + chroot_git clone --depth 1 https://git.launchpad.net/ubuntu/+source/hello hello + cd $_ - function test_ftp_clone(){ - ftp_root=$(install-slices python3-pyftpdlib_libs busybox_bins) - mkdir $ftp_root/dev - mount --bind /dev "${ftp_root}/dev" - mkdir -p "$1/etc/" - cp /etc/resolv.conf $1/etc/ + chroot_git --no-pager log + chroot_git fetch origin + chroot_git --no-pager branch --remotes --contains $(chroot_git rev-list -n 1 HEAD) + } - mkdir -p $ftp_root/srv/git/ftp-src + function test_ftp_support(){ + ftp_root=$(install-slices python3-pyftpdlib_libs) - pushd $ftp_root/srv/git/ftp-src - git init --bare - git update-server-info + mkdir -p srv/git/ftp-src + pushd $_ + chroot_git init --bare + chroot_git update-server-info popd + + cp -r $1/srv $ftp_root + pushd $ftp_root/srv/git chroot $ftp_root /usr/bin/python3.10 -m pyftpdlib -V -p 2121 -d /srv/git & - export pyftpdlib_pid=$! + pyftpdlib_pid=$! + popd # wait for server port to become available - while ! nc -z 127.0.0.1 2121 + while ! fuser 2121/tcp > /dev/null 2>&1 do - test -d /proc/$pyftpdlib_pid || (echo "pyftpdlib exited early"; exit 1) + if ! test -d /proc/$pyftpdlib_pid + then + echo "pyftpdlib exited early" + exit 1 + fi sleep 1 done # Test cloning the repo - chroot $1 git clone ftp://localhost:2121/ftp-src ftp-dst + chroot_git clone ftp://localhost:2121/ftp-src ftp-dst # cleanup - kill $pyftpdlib_pid + kill -TERM $(fuser 2121/tcp 2>/dev/null) } - function test_daemon_clone(){ + function test_daemon_support(){ mkdir -p $1/srv/git/daemon-src pushd $1/srv/git/daemon-src git init touch test-file - git add test-file - git config user.email "root@localhost" - git config user.name "Test Runner" - git commit -m "Hello daemon" + chroot_git add test-file + chroot_git config user.email "root@localhost" + chroot_git config user.name "Test Runner" + chroot_git commit -m "Hello daemon" popd - chroot "$1" /usr/lib/git-core/git-daemon \ + chroot_git daemon \ --base-path=/srv/git \ --reuseaddr \ --listen=127.0.0.1 \ @@ -103,20 +132,26 @@ execute: | --export-all & daemon_pid=$! + # Wait for daemon to start - while ! nc -z 127.0.0.1 9418 + while ! fuser 9418/tcp > /dev/null 2>&1 do - test -d /proc/$daemon_pid || (echo "daemon exited early"; exit 1) + if ! test -d /proc/$daemon_pid + then + echo "daemon exited early" + exit 1 + fi sleep 1 done # Test cloning the repo - chroot $1 git clone git://localhost/daemon-src daemon-dst + chroot_git clone git://localhost/daemon-src daemon-dst - # Cleanup - kill $daemon_pid + # Cleanup. TERM or KILL cause git to return an en error. + kill -INT $(fuser 9418/tcp 2>/dev/null) } rootfs=$(install-slices $SLICE $EXTRA_SLICES) setup $rootfs $RUN $rootfs + cleanup From 6309e4a0c707885efd999fa004e2e450c6b78c0f Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 11 Dec 2024 19:48:44 -0500 Subject: [PATCH 6/7] fix: bind mount /dev in ftp_root Needed because pyftpdlib needs a file in /dev --- tests/spread/integration/git/task.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/spread/integration/git/task.yaml b/tests/spread/integration/git/task.yaml index 97950dbce..e4ff26cc4 100644 --- a/tests/spread/integration/git/task.yaml +++ b/tests/spread/integration/git/task.yaml @@ -80,6 +80,8 @@ execute: | function test_ftp_support(){ ftp_root=$(install-slices python3-pyftpdlib_libs) + mkdir $ftp_root/dev + mount --bind /dev "${ftp_root}/dev" mkdir -p srv/git/ftp-src pushd $_ From 2512633cb8c9202dbdb167340d7bda4c79f2ea89 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Mon, 16 Dec 2024 17:40:13 -0500 Subject: [PATCH 7/7] fix: pr suggestion --- slices/git.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/slices/git.yaml b/slices/git.yaml index d0cd3358f..029bf04f7 100644 --- a/slices/git.yaml +++ b/slices/git.yaml @@ -24,6 +24,9 @@ slices: - libpcre2-8-0_libs - zlib1g_libs contents: + # /usr/bin/git is a regular file in the package, but shares + # the same hash of /usr/lib/git-core/git. That's why this + # is added as a symlink to reduce space. /usr/bin/git: {symlink: /usr/lib/git-core/git} /usr/lib/git-core/git: