From 6418c54042cf05808a521b94950262e6d2b7dd8c Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 15 Nov 2024 17:00:05 +0100 Subject: [PATCH 1/2] feat: support Pro archives (#167) In chisel.yaml, archive definitions can now use the "pro" value to specify Ubuntu Pro archives. The `archives..pro` value currently accepts the following values: "fips", "fips-updates", "esm-apps" and "esm-infra". Any other values are ignored. By default, Chisel will look for credentials in the `/etc/apt/auth.conf.d/` directory, unless the environment variable `CHISEL_AUTH_DIR` is set. In which case, it will look for configuration files in that directory. The configuration files may only have the ".conf" extensions or no extensions, the format is described in https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html. Co-authored-by: Rafid Bin Mostofa --- .github/workflows/pro_tests.yaml | 81 ++++ .github/workflows/spread.yml | 2 +- .github/workflows/tests.yaml | 15 + README.md | 42 ++ cmd/chisel/cmd_cut.go | 5 + cmd/chisel/main.go | 1 + internal/archive/archive.go | 109 ++++- internal/archive/archive_test.go | 417 ++++++++++++++---- internal/archive/export_test.go | 2 + internal/setup/setup.go | 10 + internal/setup/setup_test.go | 111 +++++ internal/slicer/slicer_test.go | 24 + internal/testutil/pgpkeys.go | 117 +++++ snap/snapcraft.yaml | 10 + spread.yaml | 1 + .../pro-archives/chisel-releases/chisel.yaml | 45 ++ .../chisel-releases/slices/hello.yaml | 13 + tests/pro-archives/task.yaml | 24 + 18 files changed, 938 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/pro_tests.yaml create mode 100644 tests/pro-archives/chisel-releases/chisel.yaml create mode 100644 tests/pro-archives/chisel-releases/slices/hello.yaml create mode 100644 tests/pro-archives/task.yaml diff --git a/.github/workflows/pro_tests.yaml b/.github/workflows/pro_tests.yaml new file mode 100644 index 00000000..af52a20d --- /dev/null +++ b/.github/workflows/pro_tests.yaml @@ -0,0 +1,81 @@ +name: Pro Tests + +on: + workflow_dispatch: + push: + paths-ignore: + - '**.md' + schedule: + - cron: "0 0 */2 * *" + workflow_run: + workflows: ["CLA check"] + types: + - completed + +jobs: + real-archive-tests: + name: Real Archive Tests + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + # Do not change to newer releases as "fips" may not be available there. + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run real archive tests + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + set -ex + + detach() { + sudo pro detach --assume-yes || true + sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage + } + trap detach EXIT + + # Attach pro token and enable services + sudo pro attach ${PRO_TOKEN} --no-auto-enable + + # Cannot enable fips and fips-updates at the same time. + # Hack: enable fips, copy the credentials and then after enabling + # other services, add the credentials back. + sudo pro enable fips --assume-yes + sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + # This will disable the fips service. + sudo pro enable fips-updates esm-apps esm-infra --assume-yes + # Add the fips credentials back. + sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage' + sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + + # Make apt credentials accessible to USER. + sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage + + # Run tests on Pro real archives. + go test ./internal/archive/ --real-pro-archive + + spread-tests: + name: Spread tests + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 + with: + repository: snapcore/spread + path: _spread + + - uses: actions/setup-go@v3 + with: + go-version: '>=1.17.0' + + - name: Build and run spread + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + (cd _spread/cmd/spread && go build) + _spread/cmd/spread/spread -v tests/pro-archives diff --git a/.github/workflows/spread.yml b/.github/workflows/spread.yml index 190a9061..3ac906f6 100644 --- a/.github/workflows/spread.yml +++ b/.github/workflows/spread.yml @@ -29,4 +29,4 @@ jobs: - name: Build and run spread run: | (cd _spread/cmd/spread && go build) - _spread/cmd/spread/spread -v focal jammy mantic noble + _spread/cmd/spread/spread -v diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7579f22d..0f2e0425 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -43,3 +43,18 @@ jobs: with: name: chisel-test-coverage.html path: ./*.html + + real-archive-tests: + # Do not change to newer releases as "fips" may not be available there. + runs-on: ubuntu-20.04 + name: Real Archive Tests + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run real archive tests + run: | + go test ./internal/archive/ --real-archive diff --git a/README.md b/README.md index 4e448125..14ce897f 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,48 @@ provided packages and install only the desired slices into the *myrootfs* folder, according to the slice definitions available in the ["ubuntu-22.04" chisel-releases branch](). +## Support for Pro archives +> [!IMPORTANT] +> To chisel a Pro package you need to have a Pro-enabled host. + +To fetch and install slices from Ubuntu Pro packages, the Pro archive has to be +defined with the `archives..pro` field in `chisel.yaml`: + + +```yaml +# chisel.yaml +format: v1 +archives: + : + pro: + ... +... +``` + +The following Pro archives are currently supported: + +| `pro` value | Archive URL | +|--------------|--------------------------------------------| +| fips | https://esm.ubuntu.com/fips/ubuntu | +| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | +| esm-apps | https://esm.ubuntu.com/apps/ubuntu | +| esm-infra | https://esm.ubuntu.com/infra/ubuntu | + +If the system is using the [Pro client](https://ubuntu.com/pro/tutorial), and the +services are enabled, the credentials will be automatically picked up from +`/etc/apt/auth.conf.d/`. However, the default permissions of the credentials file +need to be changed so that Chisel can read it. Example: +```shell +sudo pro enable esm-infra + +sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage +# or, alternatively, +sudo chmod u+r /etc/apt/auth.conf.d/90ubuntu-advantage +``` + +The location of the credentials can be configured using the environment variable +`CHISEL_AUTH_DIR`. + ## Reference ### Chisel releases diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index e5ed42bc..5c3088b3 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -70,10 +70,15 @@ func (cmd *cmdCut) Execute(args []string) error { Arch: cmd.Arch, Suites: archiveInfo.Suites, Components: archiveInfo.Components, + Pro: archiveInfo.Pro, CacheDir: cache.DefaultDir("chisel"), PubKeys: archiveInfo.PubKeys, }) if err != nil { + if err == archive.ErrCredentialsNotFound { + logf("Archive %q ignored: credentials not found", archiveName) + continue + } return err } archives[archiveName] = openArchive diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index e8f925a2..50792839 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -327,6 +327,7 @@ func run() error { deb.SetLogger(log.Default()) setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) + SetLogger(log.Default()) parser := Parser() xtra, err := parser.Parse() diff --git a/internal/archive/archive.go b/internal/archive/archive.go index c09fbdfb..170e3bd5 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -36,6 +36,7 @@ type Options struct { Arch string Suites []string Components []string + Pro string CacheDir string PubKeys []*packet.PublicKey } @@ -77,6 +78,8 @@ type ubuntuArchive struct { indexes []*ubuntuIndex cache *cache.Cache pubKeys []*packet.PublicKey + baseURL string + creds *credentials } type ubuntuIndex struct { @@ -147,6 +150,54 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" +const ( + ProFIPS = "fips" + ProFIPSUpdates = "fips-updates" + ProApps = "esm-apps" + ProInfra = "esm-infra" +) + +var proArchiveInfo = map[string]struct { + BaseURL, Label string +}{ + ProFIPS: { + BaseURL: "https://esm.ubuntu.com/fips/ubuntu/", + Label: "UbuntuFIPS", + }, + ProFIPSUpdates: { + BaseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", + Label: "UbuntuFIPSUpdates", + }, + ProApps: { + BaseURL: "https://esm.ubuntu.com/apps/ubuntu/", + Label: "UbuntuESMApps", + }, + ProInfra: { + BaseURL: "https://esm.ubuntu.com/infra/ubuntu/", + Label: "UbuntuESM", + }, +} + +func archiveURL(pro, arch string) (string, *credentials, error) { + if pro != "" { + archiveInfo, ok := proArchiveInfo[pro] + if !ok { + return "", nil, fmt.Errorf("invalid pro value: %q", pro) + } + url := archiveInfo.BaseURL + creds, err := findCredentials(url) + if err != nil { + return "", nil, err + } + return url, creds, nil + } + + if arch == "amd64" || arch == "i386" { + return ubuntuURL, nil, nil + } + return ubuntuPortsURL, nil, nil +} + func openUbuntu(options *Options) (Archive, error) { if len(options.Components) == 0 { return nil, fmt.Errorf("archive options missing components") @@ -158,12 +209,19 @@ func openUbuntu(options *Options) (Archive, error) { return nil, fmt.Errorf("archive options missing version") } + baseURL, creds, err := archiveURL(options.Pro, options.Arch) + if err != nil { + return nil, err + } + archive := &ubuntuArchive{ options: *options, cache: &cache.Cache{ Dir: options.CacheDir, }, pubKeys: options.PubKeys, + baseURL: baseURL, + creds: creds, } for _, suite := range options.Suites { @@ -184,6 +242,11 @@ func openUbuntu(options *Options) (Archive, error) { return nil, err } release = index.release + if !index.supportsArch(options.Arch) { + // Release does not support the specified architecture, do + // not add any of its indexes. + break + } err = index.checkComponents(options.Components) if err != nil { return nil, err @@ -201,7 +264,7 @@ func openUbuntu(options *Options) (Archive, error) { } func (index *ubuntuIndex) fetchRelease() error { - logf("Fetching %s %s %s suite details...", index.label, index.version, index.suite) + logf("Fetching %s %s %s suite details...", index.displayName(), index.version, index.suite) reader, err := index.fetch("InRelease", "", fetchDefault) if err != nil { return err @@ -235,12 +298,14 @@ func (index *ubuntuIndex) fetchRelease() error { if err != nil { return fmt.Errorf("cannot parse InRelease file: %v", err) } - section := ctrl.Section("Ubuntu") + // Parse the appropriate section for the type of archive. + label := "Ubuntu" + if index.archive.options.Pro != "" { + label = proArchiveInfo[index.archive.options.Pro].Label + } + section := ctrl.Section(label) if section == nil { - section = ctrl.Section("UbuntuProFIPS") - if section == nil { - return fmt.Errorf("corrupted archive InRelease file: no Ubuntu section") - } + return fmt.Errorf("corrupted archive InRelease file: no %s section", label) } logf("Release date: %s", section.Get("Date")) @@ -256,7 +321,7 @@ func (index *ubuntuIndex) fetchIndex() error { return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component) } - logf("Fetching index for %s %s %s %s component...", index.label, index.version, index.suite, index.component) + logf("Fetching index for %s %s %s %s component...", index.displayName(), index.version, index.suite, index.component) reader, err := index.fetch(packagesPath+".gz", digest, fetchBulk) if err != nil { return err @@ -270,6 +335,17 @@ func (index *ubuntuIndex) fetchIndex() error { return nil } +// supportsArch returns true if the Architectures field in the index release +// contains "arch". Per the Debian wiki [1], index release files should list the +// supported architectures in the "Architectures" field. +// The "ubuntuURL" archive only supports the amd64 and i386 architectures +// whereas the "ubuntuPortsURL" one supports the rest. But each of them +// (faultly) specifies all those architectures in their InRelease files. +// Reference: [1] https://wiki.debian.org/DebianRepository/Format#Architectures +func (index *ubuntuIndex) supportsArch(arch string) bool { + return strings.Contains(index.release.Get("Architectures"), arch) +} + func (index *ubuntuIndex) checkComponents(components []string) error { releaseComponents := strings.Fields(index.release.Get("Components")) for _, c1 := range components { @@ -295,10 +371,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return nil, err } - baseURL := ubuntuURL - if index.arch != "amd64" && index.arch != "i386" { - baseURL = ubuntuPortsURL - } + baseURL, creds := index.archive.baseURL, index.archive.creds var url string if strings.HasPrefix(suffix, "pool/") { @@ -311,6 +384,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } + if creds != nil && !creds.Empty() { + req.SetBasicAuth(creds.Username, creds.Password) + } var resp *http.Response if flags&fetchBulk != 0 { resp, err = bulkDo(req) @@ -325,7 +401,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea switch resp.StatusCode { case 200: // ok - case 401, 404: + case 401: + return nil, fmt.Errorf("cannot fetch from %q: unauthorized", index.label) + case 404: return nil, fmt.Errorf("cannot find archive data") default: return nil, fmt.Errorf("error from archive: %v", resp.Status) @@ -363,3 +441,10 @@ func sectionPackageInfo(section control.Section) *PackageInfo { SHA256: section.Get("SHA256"), } } + +func (index *ubuntuIndex) displayName() string { + if index.archive.options.Pro == "" { + return index.label + } + return index.label + " " + index.archive.options.Pro + " (pro)" +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 703148fb..d8a243d2 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -40,9 +40,12 @@ type httpSuite struct { var _ = Suite(&httpSuite{}) var ( - key1 = testutil.PGPKeys["key1"] - key2 = testutil.PGPKeys["key2"] - keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + key1 = testutil.PGPKeys["key1"] + key2 = testutil.PGPKeys["key2"] + keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + keyUbuntuFIPSv1 = testutil.PGPKeys["key-ubuntu-fips-v1"] + keyUbuntuApps = testutil.PGPKeys["key-ubuntu-apps"] + keyUbuntuESMv2 = testutil.PGPKeys["key-ubuntu-esm-v2"] ) func (s *httpSuite) SetUpTest(c *C) { @@ -178,6 +181,16 @@ var optionErrorTests = []optionErrorTest{{ Components: []string{"main", "other"}, }, error: `invalid package architecture: foo`, +}, { + options: archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "other"}, + Pro: "invalid", + }, + error: `invalid pro value: "invalid"`, }} func (s *httpSuite) TestOptionErrors(c *C) { @@ -328,44 +341,112 @@ func (s *httpSuite) TestArchiveLabels(c *C) { } } - s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + tests := []struct { + summary string + label string + err string + }{{ + summary: "Ubuntu label", + label: "Ubuntu", + }, { + summary: "Unknown label", + label: "Unknown", + err: "corrupted archive InRelease file: no Ubuntu section", + }} + + for _, test := range tests { + c.Logf("Summary: %s", test.summary) - options := archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + var adjust func(*testarchive.Release) + if test.label != "" { + adjust = setLabel(test.label) + } + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, adjust) + + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err := archive.Open(&options) + if test.err != "" { + c.Assert(err, ErrorMatches, test.err) + } else { + c.Assert(err, IsNil) + } } +} - _, err := archive.Open(&options) +func (s *httpSuite) TestProArchives(c *C) { + setLabel := func(label string) func(*testarchive.Release) { + return func(r *testarchive.Release) { + r.Label = label + } + } + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "" + for _, info := range archive.ProArchiveInfo { + contents += fmt.Sprintf("machine %s login foo password bar\n", info.BaseURL) + } + err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("Ubuntu")) + do := func(req *http.Request) (*http.Response, error) { + auth, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + c.Assert(auth, DeepEquals, []string{"Basic Zm9vOmJhcg=="}) + return s.Do(req) + } + restoreDo := archive.FakeDo(do) + defer restoreDo() - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err = archive.Open(&options) + c.Assert(err, IsNil) } - _, err = archive.Open(&options) - c.Assert(err, IsNil) + // Test non-pro archives. + do = func(req *http.Request) (*http.Response, error) { + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, false, Commentf("Non-pro archives should not have any authorization header")) + return s.Do(req) + } + restoreDo = archive.FakeDo(do) + defer restoreDo() - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("UbuntuProFIPS")) + s.base = "http://archive.ubuntu.com/ubuntu/" + s.prepareArchive("focal", "20.04", "amd64", []string{"main"}) - options = archive.Options{ + options := archive.Options{ Label: "ubuntu", - Version: "22.04", + Version: "20.04", Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, + Suites: []string{"focal"}, + Components: []string{"main"}, CacheDir: c.MkDir(), PubKeys: []*packet.PublicKey{s.pubKey}, } @@ -373,20 +454,41 @@ func (s *httpSuite) TestArchiveLabels(c *C) { _, err = archive.Open(&options) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("ThirdParty")) - - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + // Test Pro archives with bad credentials. + do = func(req *http.Request) (*http.Response, error) { + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + if strings.Contains(req.URL.String(), "/pool/") { + s.status = 401 + } else { + s.status = 200 + } + return s.Do(req) } + restoreDo = archive.FakeDo(do) + defer restoreDo() - _, err = archive.Open(&options) - c.Assert(err, ErrorMatches, `.*\bno Ubuntu section`) + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) + + _, _, err = testArchive.Fetch("mypkg1") + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) + } } type verifyArchiveReleaseTest struct { @@ -491,45 +593,177 @@ func read(r io.Reader) string { } // ---------------------------------------------------------------------------------------- -// Real archive tests, only enabled via --real-archive. +// Real archive tests, only enabled via: +// 1. --real-archive for non-Pro archives (e.g. standard jammy archive), +// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). +// +// To run the tests for Ubuntu Pro archives, the host machine must be Pro +// enabled and relevant Pro services must be enabled. The following commands +// might help: +// sudo pro attach --no-auto-enable +// sudo pro enable fips-updates esm-apps esm-infra --assume-yes var realArchiveFlag = flag.Bool("real-archive", false, "Perform tests against real archive") +var proArchiveFlag = flag.Bool("real-pro-archive", false, "Perform tests against real Ubuntu Pro archive") func (s *S) TestRealArchive(c *C) { if !*realArchiveFlag { c.Skip("--real-archive not provided") } - for _, release := range ubuntuReleases { - for _, arch := range elfToDebArch { - s.testOpenArchiveArch(c, release, arch) + s.runRealArchiveTests(c, realArchiveTests) +} + +func (s *S) TestRealProArchives(c *C) { + if !*proArchiveFlag { + c.Skip("--real-pro-archive not provided") + } + s.runRealArchiveTests(c, proArchiveTests) + s.testRealProArchiveBadCreds(c) +} + +func (s *S) runRealArchiveTests(c *C, tests []realArchiveTest) { + allArch := make([]string, 0, len(elfToDebArch)) + for _, arch := range elfToDebArch { + allArch = append(allArch, arch) + } + for _, test := range tests { + if len(test.archs) == 0 { + test.archs = allArch + } + for _, arch := range test.archs { + s.testOpenArchiveArch(c, test, arch) } } } -type ubuntuRelease struct { +type realArchiveTest struct { name string version string + suites []string + components []string + pro string archivePubKeys []*packet.PublicKey + archs []string + pkg string + path string } -var ubuntuReleases = []ubuntuRelease{{ - name: "focal", - version: "20.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, +var realArchiveTests = []realArchiveTest{{ + name: "focal", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/bin/hostname", }, { - name: "jammy", - version: "22.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, + name: "jammy", + version: "22.04", + suites: []string{"jammy"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/bin/hostname", }, { - name: "noble", - version: "24.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, + name: "noble", + version: "24.04", + suites: []string{"noble"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/usr/bin/hostname", +}} + +var proArchiveTests = []realArchiveTest{{ + name: "focal-fips", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main"}, + pro: "fips", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "focal-fips-updates", + version: "20.04", + suites: []string{"focal-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "focal-esm-apps", + version: "20.04", + suites: []string{"focal-apps-security", "focal-apps-updates"}, + components: []string{"main"}, + pro: "esm-apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "focal-esm-infra", + version: "20.04", + suites: []string{"focal-infra-security", "focal-infra-updates"}, + components: []string{"main"}, + pro: "esm-infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "jammy-fips-updates", + version: "22.04", + suites: []string{"jammy-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "jammy-esm-apps", + version: "22.04", + suites: []string{"jammy-apps-security", "jammy-apps-updates"}, + components: []string{"main"}, + pro: "esm-apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "jammy-esm-infra", + version: "22.04", + suites: []string{"jammy-infra-security", "jammy-infra-updates"}, + components: []string{"main"}, + pro: "esm-infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "noble-esm-apps", + version: "24.04", + suites: []string{"noble-apps-security", "noble-apps-updates"}, + components: []string{"main"}, + pro: "esm-apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "noble-esm-infra", + version: "24.04", + suites: []string{"noble-infra-security", "noble-infra-updates"}, + components: []string{"main"}, + pro: "esm-infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", }} var elfToDebArch = map[elf.Machine]string{ @@ -551,17 +785,18 @@ func (s *S) checkArchitecture(c *C, arch string, binaryPath string) { c.Assert(binaryArch, Equals, arch) } -func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { - c.Logf("Checking ubuntu archive %s %s...", release.name, arch) +func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { + c.Logf("Checking ubuntu archive %s %s...", test.name, arch) options := archive.Options{ Label: "ubuntu", - Version: release.version, + Version: test.version, Arch: arch, - Suites: []string{release.name}, - Components: []string{"main", "universe"}, + Suites: test.suites, + Components: test.components, CacheDir: c.MkDir(), - PubKeys: release.archivePubKeys, + Pro: test.pro, + PubKeys: test.archivePubKeys, } testArchive, err := archive.Open(&options) @@ -569,30 +804,56 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { extractDir := c.MkDir() - pkg, info, err := testArchive.Fetch("hostname") + pkg, info, err := testArchive.Fetch(test.pkg) c.Assert(err, IsNil) - c.Assert(info.Name, DeepEquals, "hostname") + c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) err = deb.Extract(pkg, &deb.ExtractOptions{ - Package: "hostname", + Package: test.pkg, TargetDir: extractDir, Extract: map[string][]deb.ExtractInfo{ - "/usr/share/doc/hostname/copyright": { + fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, }, - "/bin/hostname": { - {Path: "/hostname"}, + test.path: { + {Path: "/binary"}, }, }, }) c.Assert(err, IsNil) - data, err := os.ReadFile(filepath.Join(extractDir, "copyright")) + s.checkArchitecture(c, arch, filepath.Join(extractDir, "binary")) +} + +func (s *S) testRealProArchiveBadCreds(c *C) { + c.Logf("Cannot fetch Pro packages with bad credentials") + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "machine esm.ubuntu.com/fips/ubuntu/ login bearer password invalid" + err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) - copyrightTop := "This package was written by Peter Tobias " - c.Assert(strings.Contains(string(data), copyrightTop), Equals, true) + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: "fips", + PubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + } + + // The archive can be "opened" without any credentials since the dists/ path + // containing InRelease files, does not require any credentials. + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) - s.checkArchitecture(c, arch, filepath.Join(extractDir, "hostname")) + _, _, err = testArchive.Fetch("openssh-client") + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) } diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index cd75d5f4..3569a699 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -19,3 +19,5 @@ type Credentials = credentials var FindCredentials = findCredentials var FindCredentialsInDir = findCredentialsInDir + +var ProArchiveInfo = proArchiveInfo diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 00337bb6..eb8114f5 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -13,6 +13,7 @@ import ( "golang.org/x/crypto/openpgp/packet" "gopkg.in/yaml.v3" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" "github.com/canonical/chisel/internal/strdist" @@ -33,6 +34,7 @@ type Archive struct { Suites []string Components []string Priority int + Pro string PubKeys []*packet.PublicKey } @@ -399,6 +401,7 @@ type yamlArchive struct { Suites []string `yaml:"suites"` Components []string `yaml:"components"` Priority *int `yaml:"priority"` + Pro string `yaml:"pro"` Default bool `yaml:"default"` PubKeys []string `yaml:"public-keys"` } @@ -553,6 +556,12 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if len(details.Components) == 0 { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } + switch details.Pro { + case "", archive.ProApps, archive.ProFIPS, archive.ProFIPSUpdates, archive.ProInfra: + default: + logf("Archive %q ignored: invalid pro value: %q", archiveName, details.Pro) + continue + } if details.Default && defaultArchive != "" { if archiveName < defaultArchive { archiveName, defaultArchive = defaultArchive, archiveName @@ -591,6 +600,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Version: details.Version, Suites: details.Suites, Components: details.Components, + Pro: details.Pro, Priority: priority, PubKeys: archiveKeys, } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index d8d2f564..32e81428 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1739,6 +1739,117 @@ var setupTests = []setupTest{{ }, }, }, +}, { + summary: "Pro values in archives", + input: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + ubuntu: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + public-keys: [test-key] + fips: + version: 20.04 + components: [main] + suites: [focal] + pro: fips + priority: 20 + public-keys: [test-key] + fips-updates: + version: 20.04 + components: [main] + suites: [focal-updates] + pro: fips-updates + priority: 21 + public-keys: [test-key] + esm-apps: + version: 20.04 + components: [main] + suites: [focal-apps-security] + pro: esm-apps + priority: 16 + public-keys: [test-key] + esm-infra: + version: 20.04 + components: [main] + suites: [focal-infra-security] + pro: esm-infra + priority: 15 + public-keys: [test-key] + ignored: + version: 20.04 + components: [main] + suites: [foo] + pro: unknown-value + priority: 10 + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Priority: 10, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips": { + Name: "fips", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Pro: "fips", + Priority: 20, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips-updates": { + Name: "fips-updates", + Version: "20.04", + Suites: []string{"focal-updates"}, + Components: []string{"main"}, + Pro: "fips-updates", + Priority: 21, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "esm-apps": { + Name: "esm-apps", + Version: "20.04", + Suites: []string{"focal-apps-security"}, + Components: []string{"main"}, + Pro: "esm-apps", + Priority: 16, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "esm-infra": { + Name: "esm-infra", + Version: "20.04", + Suites: []string{"focal-infra-security"}, + Components: []string{"main"}, + Pro: "esm-infra", + Priority: 15, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, }, { summary: "Default is ignored", input: map[string]string{ diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 6cdb8f01..4974d623 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1445,6 +1445,29 @@ var slicerTests = []slicerTest{{ contents: `, }, +}, { + summary: "No valid archives defined due to invalid pro value", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + invalid: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + public-keys: [test-key] + pro: unknown-value + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, }} var defaultChiselYaml = ` @@ -1537,6 +1560,7 @@ func (s *S) TestRun(c *C) { Version: setupArchive.Version, Suites: setupArchive.Suites, Components: setupArchive.Components, + Pro: setupArchive.Pro, Arch: test.arch, }, Packages: pkgs, diff --git a/internal/testutil/pgpkeys.go b/internal/testutil/pgpkeys.go index e999a9a3..98eb6b05 100644 --- a/internal/testutil/pgpkeys.go +++ b/internal/testutil/pgpkeys.go @@ -21,6 +21,18 @@ var PGPKeys = map[string]*PGPKeyData{ ID: "871920D1991BC93C", PubKeyArmor: pubKeyUbuntu2018Armor, }, + "key-ubuntu-fips-v1": { + ID: "C1997C40EDE22758", + PubKeyArmor: pubKeyUbuntuFIPSv1Armor, + }, + "key-ubuntu-apps": { + ID: "AB01A101DB53907B", + PubKeyArmor: pubKeyUbuntuAppsArmor, + }, + "key-ubuntu-esm-v2": { + ID: "4067E40313CB4B13", + PubKeyArmor: pubKeyUbuntuESMv2Armor, + }, "key1": { ID: "854BAF1AA9D76600", PubKeyArmor: pubKey1Armor, @@ -87,6 +99,111 @@ uOgcXny1UlwtCUzlrSaP -----END PGP PUBLIC KEY BLOCK----- ` +// Ubuntu Federal Information Processing Standards Automatic Signing Key V1 . +// ID: C1997C40EDE22758. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuFIPSv1Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFzZxGABEADSWmX0+K//0cosKPyr5m1ewmwWKjRo/KBPTyR8icHhbBWfFd8T +DtYggvQHPU0YnKRcWits0et8JqSgZttNa28s7SaSUTBzfgzFJZgULAi/4i8u8TUj ++KH2zSoUX55NKC9aozba1cR66jM6O/BHXK5YoZzTpmiY1AHlIWAJ9s6cCClhnYMR +zwxSZVbefcYFbVPX/dQw/FMvJVeZ2aQ18NDgMQciu786aYklMFowxWNs/eLLTqum +cDHaw9UpKyhgfL/mkaIXuhYy6YRByYq1oOnJ5XffAOtovvCti1MvsPc0NDhPiGLf +9Fd/GtnqHxzVDqZmtUXX50mGu4LnJoHgWRjml3mapDPStzFr7Xgbb0NnyflmxnfN +kQcu2lFyXFfndWwg/RAOFdBPxBQhRK52uZiCfydKD7zCXz9YGm9xEK541EG0FrwA +6Vk1xaFol/jI8MQdP1o3JySX0Pqva3IHF7FHWHmxrIPaJLIHi0IrFG6Fgmk4sQ2w +XSc8kbxR+wYYKqIhBUZP0eb1jkFfvRVS6YvAy18xtw5pFD+VURdA0Uu5cotESfyz +oHsQ5R7wzg76oV/mYukHGC0x8peqxiPwbyhGFAhG8eUR66iYZgGbzmNI+OJz2EUi +UZJJXt4rnI1RVJLbhK9RjeobkOjf58Cm8RExlqJU16gy9saCMSiAqHx8swARAQAB +tFxVYnVudHUgRmVkZXJhbCBJbmZvcm1hdGlvbiBQcm9jZXNzaW5nIFN0YW5kYXJk +cyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgVjEgPGVzbUBjYW5vbmljYWwuY29tPokC +OAQTAQIAIgUCXNnEYAIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQwZl8 +QO3iJ1j4Vw//SawfmZi1GW+EUnuPqSz+zcmIKdx6AWZTe9/vSj6jgq4SYt//LAiD +NQz3dn2m0m5AaCucza2BCixUBrNhMh66m+lXfTqymUtTIpWpu4L1WLUbPjQ+s3Ad +xuF7S5wJtQrYmPvmZduZgg1wcb8eaqVltRJREpOP6sxcuqtvcfv4v4QYZ+iYd7eJ +8fxPOiyJEOTQPTdPZahYTaUOIloN5pT6uVg03u59Kh4aHCYxlRorvuRBabdctCfA +EBgomk4Us20Tv31dqlvMAiGKJqf1wdjhzlUmk4g/fOiRSNETKSC/VeUGH0fSbizl +Gs7Mg60jChPKpwzB6Rb5Nv2/Aw/FlSkfFhMdCdfKjl8IWOMPmElTVJFyVx1mmURi +3LgsloDFmJfebXefSFA7S8KLyBGlZJ/APaym64Ls12PUOjfh1Glie3E8KO66AGLo +ID1dQnzRizuHxW80ET03dSjzTXHLSi+iFycmNAxo6gB3GyOQ8tlIHjo1FfDfNYDf +qKic3Q0B9TvF6hqVRIcyePK4lN5YtRpVRdVj/jv8AqbzaIaVCP4k4nNrbaVx5zQf +BWq2E9IH+vLZfPyiP+hwxswfrlU3mrXBpPStIxq41yXFwQiDnqgkhEVAcrYPjBnS +T6s3+b+4HbAW6mbp4jEHUd/F1+iXz90T2WArrNIkMbmpChMuSyRN8Hc= +=DWhM +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Apps Automatic Signing Key . +// ID: AB01A101DB53907B. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuAppsArmor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF3WVA4BEAC7MDr8HClfKptSd4VeB12Vy+Ao/4NpY2ITdkRed4vfh/4eBWWn +3+in6So2ekweifACSxScB/M9zVObsI1cab7QPMkIiATNUfIyOEP7iNWLX4+AytM1 +LP3bZo8OpghnLZNstCGbiRUO4CDNmCI04DOPCu9EVEO4WWNuWIMRwCLShDSf7Cid +J2fn2TT/7vsmA4eI3YnAne+u8g4X2zMHQFkHANhylB0lPyThXo5jaxHImzm4wf/2 +LF8f1Y1nRQObS2jcvYc3fm9B7iOGpyNAw3h6hrPKH5T9tY/ZoMtFHqn66J1CBSHb +hDkEvA46X50su4yAHeSiEG/hMYG7SoHzmAsjEXnvkTIE41WhmxlidQnRs2uWy34U +7VmOpaidWn3R99fNHYOtSOB6bpIvls8snWSQ63jcFXnt05nVZsp/Ixzl0Oqitynx +DFwoxEwt3ZuCHwxbx2vZ+FiZXVFN7I0IyBDOEL6XS27FNaMCZ7Q/6z/ckdWto55E +264OWf9lnw31bXFXHWSusRXWzD6FK8dqWgjtrWwRxlvF4jm688lqpjac6fFES3UK +BhjyHXFGL/+HHZ9CNxlLYF5QnXq1mGR0Ykw975u8KoOFSLBqsx+1a21m6dfzujY7 +2Gq6Sju+9Yo1aOF+CNvTMYdRBoDL4sFj6VAmUsszMA5aAb+82pOCaDvGJQARAQAB +tDVVYnVudHUgQXBwcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGVzbUBjYW5vbmlj +YWwuY29tPokCOAQTAQIAIgUCXdZUDgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC +F4AACgkQqwGhAdtTkHuTOw/8Czv42TSpwHz+eNtl3ZFyxta9rR/qWC3h+vMu0R/l +5KU3aQQOygWOoUcr1QTPSSg3v/H+v/8vqVq2UuUxSIfpMxBj2kIX2vqskv6Roez7 +xR8lVDa0a47z/NYMfKpxrEJxOLh/c7I6aAsa597bTqDHtucHL/22BvfUJJqw6jq1 +7SswP5lqKPBFz7x+E2hgfJE7Vn7h0ICm29FkWnOeTKfj8VwTAeKXKUI9Hw6+aqr9 +29Y2NdLsYZ57mpivRLNM9sBZoF3avP1pUC2k0IwP3dwh4AxUMXjRRPh173iXBfR2 +yAf1lWET/5+8dSBrfFIZSo+FF/EEBmqIVtJpHkq8+YxUbCLbkoikRi2kwrgyXLEn +FqxSU2Ab0xurFHiHcJoCGVD38xjznO5cQl7H4K9+B/rFpTTowOHbOcFpKAzpYqB5 +8rnR1yRSsB33zac8xesUIfzYWRtLc5/VIb5mOkWlb62d8emILx2XuRFVjKq6mKki +oGckhDUOuEFrjW1cQq+PWBBxyJoXcy6wGSoPJ/ELeaf9zg8SF0jwuN6BPHVBeJ/E +W53zR5iV0N9fRT+M2JN5tc5HenO92xLgPAh+GPWLYmPdTmHu+kFozqsHx/NUw2iP +PBL6Q1VZytt2Uf6qLPUx7GpYMKf42Vldb0feFo/YA/lzOgPlY29pDLKXbse6o+Sr +kmk= +=AEEr +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Extended Security Maintenance Automatic Signing Key v2 . +// ID: 4067E40313CB4B13. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuESMv2Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY +680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs +8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t +YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 +t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI +OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG +JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 +rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 +K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT +Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE +6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB +tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj +IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 +kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C +NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB +m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S +nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 +1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F +EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC +puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG +tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C +DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS +89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 +RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi +JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc +=KWLs +-----END PGP PUBLIC KEY BLOCK----- +` + // Test-purpose RSA 2048 bits signing key-pairs without a passphrase. // ID: 854BAF1AA9D76600. User: "foo-bar ". const pubKey1Armor = ` diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ee1c284c..958a5970 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -41,6 +41,8 @@ parts: override-build: | go generate ./cmd craftctl default + stage: + - -bin/chrorder chisel-release-data: plugin: nil @@ -56,9 +58,17 @@ parts: craftctl set grade="$grade" after: [chisel] +plugs: + pro-credentials: + interface: system-files + read: + - /etc/apt/auth.conf.d + - /etc/apt/auth.conf.d/90ubuntu-advantage + apps: chisel: command: bin/chisel plugs: - network - home + - pro-credentials diff --git a/spread.yaml b/spread.yaml index a8de3d36..274bbd4b 100644 --- a/spread.yaml +++ b/spread.yaml @@ -4,6 +4,7 @@ path: /chisel environment: OS: ubuntu + PRO_TOKEN: $(HOST:echo $PRO_TOKEN) backends: # Cannot use LXD backend due to https://github.com/snapcore/spread/issues/154 diff --git a/tests/pro-archives/chisel-releases/chisel.yaml b/tests/pro-archives/chisel-releases/chisel.yaml new file mode 100644 index 00000000..5beca3a9 --- /dev/null +++ b/tests/pro-archives/chisel-releases/chisel.yaml @@ -0,0 +1,45 @@ +format: v1 + +archives: + ubuntu: + version: 24.04 + pro: esm-infra + components: [main] + suites: [noble-infra-security, noble-infra-updates] + public-keys: [ubuntu-esm-key-v2] + +public-keys: + # Ubuntu Extended Security Maintenance Automatic Signing Key v2 + # rsa4096/56f7650a24c9e9ecf87c4d8d4067e40313cb4b13 2019-04-17T02:33:33Z + ubuntu-esm-key-v2: + id: "4067E40313CB4B13" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY + 680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs + 8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t + YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 + t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI + OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG + JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 + rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 + K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT + Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE + 6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB + tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj + IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 + kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C + NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB + m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S + nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 + 1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F + EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC + puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG + tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C + DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS + 89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 + RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi + JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc + =KWLs + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/pro-archives/chisel-releases/slices/hello.yaml b/tests/pro-archives/chisel-releases/slices/hello.yaml new file mode 100644 index 00000000..7bd1a393 --- /dev/null +++ b/tests/pro-archives/chisel-releases/slices/hello.yaml @@ -0,0 +1,13 @@ +package: hello + +essential: + - hello_copyright + +slices: + bins: + contents: + /usr/bin/hello: + + copyright: + contents: + /usr/share/doc/hello/copyright: diff --git a/tests/pro-archives/task.yaml b/tests/pro-archives/task.yaml new file mode 100644 index 00000000..21963548 --- /dev/null +++ b/tests/pro-archives/task.yaml @@ -0,0 +1,24 @@ +summary: Chisel can fetch packages from Ubuntu Pro archives + +manual: true + +variants: + - noble + +environment: + ROOTFS: rootfs + +prepare: | + apt update && apt install -y ubuntu-pro-client + pro attach ${PRO_TOKEN} --no-auto-enable + pro enable esm-infra --assume-yes + mkdir ${ROOTFS} + +restore: | + pro detach --assume-yes + rm -r ${ROOTFS} + +execute: | + chisel cut --release ./chisel-releases/ --root ${ROOTFS} hello_bins + test -f ${ROOTFS}/usr/bin/hello + test -f ${ROOTFS}/usr/share/doc/hello/copyright From ccfe87a0e76aecf9135fab20ebacb0e532a91ee2 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 15 Nov 2024 17:05:19 +0100 Subject: [PATCH 2/2] fix: explicit parents override implicit (#166) This commits introduces a new flag for fsutil.Create called OverrideMode which updates the mode of existing entries. It is used to ensure that the explicit folder overrides the permissions of the implicit ones. --- internal/deb/extract.go | 11 +++-- internal/deb/extract_test.go | 27 +++++++++-- internal/fsutil/create.go | 14 +++++- internal/fsutil/create_test.go | 87 ++++++++++++++++++++++++++++++++++ internal/slicer/slicer_test.go | 44 +++++++++++------ 5 files changed, 160 insertions(+), 23 deletions(-) diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 07f219bd..d9e84875 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -247,11 +247,12 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { } // Create the entry itself. createOptions := &fsutil.CreateOptions{ - Path: filepath.Join(options.TargetDir, targetPath), - Mode: tarHeader.FileInfo().Mode(), - Data: pathReader, - Link: tarHeader.Linkname, - MakeParents: true, + Path: filepath.Join(options.TargetDir, targetPath), + Mode: tarHeader.FileInfo().Mode(), + Data: pathReader, + Link: tarHeader.Linkname, + MakeParents: true, + OverrideMode: true, } err := options.Create(extractInfos, createOptions) if err != nil { diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go index 22a1fd18..db73df86 100644 --- a/internal/deb/extract_test.go +++ b/internal/deb/extract_test.go @@ -2,6 +2,8 @@ package deb_test import ( "bytes" + "os" + "path" "path/filepath" "sort" "strings" @@ -17,7 +19,7 @@ type extractTest struct { summary string pkgdata []byte options deb.ExtractOptions - hackopt func(o *deb.ExtractOptions) + hackopt func(c *C, o *deb.ExtractOptions) result map[string]string // paths which the extractor did not create explicitly. notCreated []string @@ -98,7 +100,7 @@ var extractTests = []extractTest{{ "/dir/several/levels/deep/file": "file 0644 6bc26dff", "/other-dir/": "dir 0755", }, - hackopt: func(o *deb.ExtractOptions) { + hackopt: func(c *C, o *deb.ExtractOptions) { o.Create = nil }, }, { @@ -352,6 +354,25 @@ var extractTests = []extractTest{{ }, }, error: `cannot extract from package "test-package": path /dir/ requested twice with diverging mode: 0777 != 0000`, +}, { + summary: "Explicit extraction overrides existing file", + pkgdata: testutil.PackageData["test-package"], + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/dir/": []deb.ExtractInfo{{ + Path: "/dir/", + Mode: 0777, + }}, + }, + }, + hackopt: func(c *C, o *deb.ExtractOptions) { + err := os.Mkdir(path.Join(o.TargetDir, "/dir"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/dir/": "dir 0777", + }, + notCreated: []string{}, }} func (s *S) TestExtract(c *C) { @@ -374,7 +395,7 @@ func (s *S) TestExtract(c *C) { } if test.hackopt != nil { - test.hackopt(&options) + test.hackopt(c, &options) } err := deb.Extract(bytes.NewBuffer(test.pkgdata), &options) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index f76271f1..76561b77 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -19,6 +19,9 @@ type CreateOptions struct { // If MakeParents is true, missing parent directories of Path are // created with permissions 0755. MakeParents bool + // If OverrideMode is true and entry already exists, update the mode. Does + // not affect symlinks. + OverrideMode bool } type Entry struct { @@ -65,9 +68,18 @@ func Create(options *CreateOptions) (*Entry, error) { if err != nil { return nil, err } + mode := s.Mode() + if o.OverrideMode && mode != o.Mode && o.Mode&fs.ModeSymlink == 0 { + err := os.Chmod(o.Path, o.Mode) + if err != nil { + return nil, err + } + mode = o.Mode + } + entry := &Entry{ Path: o.Path, - Mode: s.Mode(), + Mode: mode, SHA256: hash, Size: rp.size, Link: o.Link, diff --git a/internal/fsutil/create_test.go b/internal/fsutil/create_test.go index d41bbf5a..be086ea5 100644 --- a/internal/fsutil/create_test.go +++ b/internal/fsutil/create_test.go @@ -93,6 +93,93 @@ var createTests = []createTest{{ // mode is not updated. "/foo": "file 0666 d67e2e94", }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Mode: fs.ModeDir | 0775, + OverrideMode: true, + }, + hackdir: func(c *C, dir string) { + c.Assert(os.Mkdir(filepath.Join(dir, "foo/"), fs.ModeDir|0765), IsNil) + }, + result: map[string]string{ + // mode is updated. + "/foo/": "dir 0775", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Mode: 0775, + Data: bytes.NewBufferString("whatever"), + OverrideMode: true, + }, + hackdir: func(c *C, dir string) { + err := os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + // mode is updated. + "/foo": "file 0775 85738f8f", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Link: "./bar", + Mode: 0666 | fs.ModeSymlink, + }, + hackdir: func(c *C, dir string) { + err := os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/foo": "symlink ./bar", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Link: "./bar", + Mode: 0776 | fs.ModeSymlink, + OverrideMode: true, + }, + hackdir: func(c *C, dir string) { + err := os.WriteFile(filepath.Join(dir, "bar"), []byte("data"), 0666) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/foo": "symlink ./bar", + // mode is not updated. + "/bar": "file 0666 3a6eb079", + }, +}, { + options: fsutil.CreateOptions{ + Path: "bar", + // Existing link with different target. + Link: "other", + Mode: 0666 | fs.ModeSymlink, + }, + hackdir: func(c *C, dir string) { + err := os.Symlink("foo", filepath.Join(dir, "bar")) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/bar": "symlink other", + }, +}, { + options: fsutil.CreateOptions{ + Path: "bar", + // Existing link with same target. + Link: "foo", + Mode: 0666 | fs.ModeSymlink, + }, + hackdir: func(c *C, dir string) { + err := os.Symlink("foo", filepath.Join(dir, "bar")) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/bar": "symlink foo", + }, }} func (s *S) TestCreate(c *C) { diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 4974d623..9e232fae 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -321,43 +321,59 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, explicit path has preference over implicit parent", slices: []setup.SliceKey{ - {"implicit-parent", "myslice"}, - {"explicit-dir", "myslice"}}, + {"a-implicit-parent", "myslice"}, + {"b-explicit-dir", "myslice"}, + {"c-implicit-parent", "myslice"}}, pkgs: []*testutil.TestPackage{{ - Name: "implicit-parent", + Name: "a-implicit-parent", Data: testutil.MustMakeDeb([]testutil.TarEntry{ testutil.Dir(0755, "./dir/"), - testutil.Reg(0644, "./dir/file", "random"), + testutil.Reg(0644, "./dir/file-1", "random"), }), }, { - Name: "explicit-dir", + Name: "b-explicit-dir", Data: testutil.MustMakeDeb([]testutil.TarEntry{ testutil.Dir(01777, "./dir/"), }), + }, { + Name: "c-implicit-parent", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0755, "./dir/"), + testutil.Reg(0644, "./dir/file-2", "random"), + }), }}, release: map[string]string{ - "slices/mydir/implicit-parent.yaml": ` - package: implicit-parent + "slices/mydir/a-implicit-parent.yaml": ` + package: a-implicit-parent slices: myslice: contents: - /dir/file: + /dir/file-1: `, - "slices/mydir/explicit-dir.yaml": ` - package: explicit-dir + "slices/mydir/b-explicit-dir.yaml": ` + package: b-explicit-dir slices: myslice: contents: /dir/: `, + "slices/mydir/c-implicit-parent.yaml": ` + package: c-implicit-parent + slices: + myslice: + contents: + /dir/file-2: + `, }, filesystem: map[string]string{ - "/dir/": "dir 01777", - "/dir/file": "file 0644 a441b15f", + "/dir/": "dir 01777", + "/dir/file-1": "file 0644 a441b15f", + "/dir/file-2": "file 0644 a441b15f", }, manifestPaths: map[string]string{ - "/dir/": "dir 01777 {explicit-dir_myslice}", - "/dir/file": "file 0644 a441b15f {implicit-parent_myslice}", + "/dir/": "dir 01777 {b-explicit-dir_myslice}", + "/dir/file-1": "file 0644 a441b15f {a-implicit-parent_myslice}", + "/dir/file-2": "file 0644 a441b15f {c-implicit-parent_myslice}", }, }, { summary: "Valid same file in two slices in different packages",