From a801a4a59d30ab8d912a9707e0664aa7e580352f Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Fri, 18 Oct 2024 15:03:39 +0600 Subject: [PATCH 01/13] feat: support "Pro" archives In chisel.yaml, archive definitions can now use the "pro" value to specify Ubuntu Pro archives. The `archives..pro` value currently accepts the following values: "fips", "fips-updates", "apps" and "infra". Any other values are ignored. By default, Chisel will look for credentials in the `/etc/apt/auth.conf.d/` directory, unless the environment variable `CHISEL_AUTH_DIR` is set. In which case, it will look for configuration files in that directory. The configuration files may only have the ".conf" extensions or no extensions, the format is described in https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html. --- .github/workflows/spread.yml | 2 + .github/workflows/tests.yaml | 43 ++ README.md | 42 ++ cmd/chisel/cmd_cut.go | 5 + cmd/chisel/main.go | 1 + internal/archive/archive.go | 110 ++++- internal/archive/archive_test.go | 417 ++++++++++++++---- internal/archive/export_test.go | 2 + internal/setup/setup.go | 18 +- internal/setup/setup_test.go | 111 +++++ internal/slicer/slicer_test.go | 24 + internal/testutil/pgpkeys.go | 117 +++++ snap/snapcraft.yaml | 10 + spread.yaml | 1 + .../pro-archives/chisel-releases/chisel.yaml | 45 ++ .../chisel-releases/slices/hello.yaml | 13 + tests/pro-archives/task.yaml | 22 + 17 files changed, 892 insertions(+), 91 deletions(-) create mode 100644 tests/pro-archives/chisel-releases/chisel.yaml create mode 100644 tests/pro-archives/chisel-releases/slices/hello.yaml create mode 100644 tests/pro-archives/task.yaml diff --git a/.github/workflows/spread.yml b/.github/workflows/spread.yml index 190a9061..ae95b7a1 100644 --- a/.github/workflows/spread.yml +++ b/.github/workflows/spread.yml @@ -27,6 +27,8 @@ jobs: go-version: '>=1.17.0' - name: Build and run spread + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} run: | (cd _spread/cmd/spread && go build) _spread/cmd/spread/spread -v focal jammy mantic noble diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7579f22d..7510cfc8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -43,3 +43,46 @@ jobs: with: name: chisel-test-coverage.html path: ./*.html + + real-archive-tests: + # Do not change to newer releases as "fips" may not be available there. + runs-on: ubuntu-20.04 + name: Real Archive Tests + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run real archive tests + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + set -ex + + detach() { + sudo pro detach --assume-yes || true + sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage + } + trap detach EXIT + + # Attach pro token and enable services + sudo pro attach ${PRO_TOKEN} --no-auto-enable + + # Cannot enable fips and fips-updates at the same time. + # Hack: enable fips, copy the credentials and then after enabling + # other services, add the credentials back. + sudo pro enable fips --assume-yes + sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + # This will disable the fips service. + sudo pro enable fips-updates esm-apps esm-infra --assume-yes + # Add the fips credentials back. + sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage' + sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + + # Make apt credentials accessible to USER. + sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage + + # Run tests on Pro and non-Pro real archives. + go test ./internal/archive/ -v --real-archive --real-pro-archive diff --git a/README.md b/README.md index 4e448125..cf5b5f8b 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,48 @@ provided packages and install only the desired slices into the *myrootfs* folder, according to the slice definitions available in the ["ubuntu-22.04" chisel-releases branch](). +## Chisel support for Pro archives + +Chisel can also fetch and install packages from Ubuntu Pro archives. For this, +the archive has to be defined with the `archives..pro` field in +chisel.yaml and its credentials have to be made available to Chisel. + + +```yaml +# chisel.yaml +format: v1 +archives: + : + pro: + ... +... +``` + +Chisel currently supports the following Pro archives: + +| `pro` value | Archive URL | Related Ubuntu Pro service | +| - | - | - | +| fips | https://esm.ubuntu.com/fips/ubuntu | fips | +| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | fips-updates | +| apps | https://esm.ubuntu.com/apps/ubuntu | esm-apps | +| infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | + +Authentication to Pro archives requires that the host is Pro or it is equipped +with the Pro credentials. By default, Chisel will support using credentials +from the `/etc/apt/auth.conf.d/` directory, but this location can be configured +using the environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have +read permission for the necessary credentials files. + +The format of the files is documented in the +[apt_auth.conf(5)](https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html) +man page. Below is a snippet of the `/etc/apt/auth.conf.d/90ubuntu-advantage` +file from a host with the `fips-updates` and `infra` archives enabled: + +``` +machine esm.ubuntu.com/infra/ubuntu/ login bearer password +machine esm.ubuntu.com/fips-updates/ubuntu/ login bearer password +``` + ## Reference ### Chisel releases diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index e5ed42bc..80b9f9eb 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -70,10 +70,15 @@ func (cmd *cmdCut) Execute(args []string) error { Arch: cmd.Arch, Suites: archiveInfo.Suites, Components: archiveInfo.Components, + Pro: archiveInfo.Pro, CacheDir: cache.DefaultDir("chisel"), PubKeys: archiveInfo.PubKeys, }) if err != nil { + if err == archive.ErrCredentialsNotFound { + logf("Ignoring archive %q (credentials not found)...", archiveName) + continue + } return err } archives[archiveName] = openArchive diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index e8f925a2..50792839 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -327,6 +327,7 @@ func run() error { deb.SetLogger(log.Default()) setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) + SetLogger(log.Default()) parser := Parser() xtra, err := parser.Parse() diff --git a/internal/archive/archive.go b/internal/archive/archive.go index c09fbdfb..9b81bda6 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -14,6 +14,7 @@ import ( "github.com/canonical/chisel/internal/control" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" + "github.com/canonical/chisel/internal/setup" ) type Archive interface { @@ -36,6 +37,7 @@ type Options struct { Arch string Suites []string Components []string + Pro string CacheDir string PubKeys []*packet.PublicKey } @@ -77,6 +79,8 @@ type ubuntuArchive struct { indexes []*ubuntuIndex cache *cache.Cache pubKeys []*packet.PublicKey + baseURL string + creds *credentials } type ubuntuIndex struct { @@ -147,6 +151,39 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" +var proArchiveInfo = map[string]struct { + BaseURL, Label string +}{ + setup.ProFIPS: { + BaseURL: "https://esm.ubuntu.com/fips/ubuntu/", + Label: "UbuntuFIPS", + }, + setup.ProFIPSUpdates: { + BaseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", + Label: "UbuntuFIPSUpdates", + }, + setup.ProApps: { + BaseURL: "https://esm.ubuntu.com/apps/ubuntu/", + Label: "UbuntuESMApps", + }, + setup.ProInfra: { + BaseURL: "https://esm.ubuntu.com/infra/ubuntu/", + Label: "UbuntuESM", + }, +} + +// archiveURL returns the archive base URL depending on the "pro" value and +// selected architecture "arch". +func archiveURL(pro, arch string) string { + if pro != "" { + return proArchiveInfo[pro].BaseURL + } + if arch == "amd64" || arch == "i386" { + return ubuntuURL + } + return ubuntuPortsURL +} + func openUbuntu(options *Options) (Archive, error) { if len(options.Components) == 0 { return nil, fmt.Errorf("archive options missing components") @@ -157,6 +194,21 @@ func openUbuntu(options *Options) (Archive, error) { if len(options.Version) == 0 { return nil, fmt.Errorf("archive options missing version") } + if options.Pro != "" { + if _, ok := proArchiveInfo[options.Pro]; !ok { + return nil, fmt.Errorf("invalid pro value: %q", options.Pro) + } + } + + baseURL := archiveURL(options.Pro, options.Arch) + var creds *credentials + if options.Pro != "" { + var err error + creds, err = findCredentials(baseURL) + if err != nil { + return nil, err + } + } archive := &ubuntuArchive{ options: *options, @@ -164,6 +216,8 @@ func openUbuntu(options *Options) (Archive, error) { Dir: options.CacheDir, }, pubKeys: options.PubKeys, + baseURL: baseURL, + creds: creds, } for _, suite := range options.Suites { @@ -184,6 +238,13 @@ func openUbuntu(options *Options) (Archive, error) { return nil, err } release = index.release + if !index.supportsArch(options.Arch) { + // Release does not support the specified architecture, do + // not add any of its indexes. + logf("Warning: ignoring %s %s %s suite (unsupported arch %s)...", + index.proSuffixedLabel(), index.version, index.suite, options.Arch) + break + } err = index.checkComponents(options.Components) if err != nil { return nil, err @@ -201,7 +262,7 @@ func openUbuntu(options *Options) (Archive, error) { } func (index *ubuntuIndex) fetchRelease() error { - logf("Fetching %s %s %s suite details...", index.label, index.version, index.suite) + logf("Fetching %s %s %s suite details...", index.proSuffixedLabel(), index.version, index.suite) reader, err := index.fetch("InRelease", "", fetchDefault) if err != nil { return err @@ -235,12 +296,14 @@ func (index *ubuntuIndex) fetchRelease() error { if err != nil { return fmt.Errorf("cannot parse InRelease file: %v", err) } - section := ctrl.Section("Ubuntu") + // Parse the appropriate section for the type of archive. + label := "Ubuntu" + if index.archive.options.Pro != "" { + label = proArchiveInfo[index.archive.options.Pro].Label + } + section := ctrl.Section(label) if section == nil { - section = ctrl.Section("UbuntuProFIPS") - if section == nil { - return fmt.Errorf("corrupted archive InRelease file: no Ubuntu section") - } + return fmt.Errorf("corrupted archive InRelease file: no %s section", label) } logf("Release date: %s", section.Get("Date")) @@ -256,7 +319,7 @@ func (index *ubuntuIndex) fetchIndex() error { return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component) } - logf("Fetching index for %s %s %s %s component...", index.label, index.version, index.suite, index.component) + logf("Fetching index for %s %s %s %s component...", index.proSuffixedLabel(), index.version, index.suite, index.component) reader, err := index.fetch(packagesPath+".gz", digest, fetchBulk) if err != nil { return err @@ -270,6 +333,17 @@ func (index *ubuntuIndex) fetchIndex() error { return nil } +// supportsArch returns true if the Architectures field in the index release +// contains "arch". Per the Debian wiki [1], index release files should list the +// supported architectures in the "Architectures" field. +// The "ubuntuURL" archive only supports the amd64 and i386 architectures +// whereas the "ubuntuPortsURL" one supports the rest. But each of them +// (faultly) specifies all those architectures in their InRelease files. +// Reference: [1] https://wiki.debian.org/DebianRepository/Format#Architectures +func (index *ubuntuIndex) supportsArch(arch string) bool { + return strings.Contains(index.release.Get("Architectures"), arch) +} + func (index *ubuntuIndex) checkComponents(components []string) error { releaseComponents := strings.Fields(index.release.Get("Components")) for _, c1 := range components { @@ -295,10 +369,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return nil, err } - baseURL := ubuntuURL - if index.arch != "amd64" && index.arch != "i386" { - baseURL = ubuntuPortsURL - } + baseURL, creds := index.archive.baseURL, index.archive.creds var url string if strings.HasPrefix(suffix, "pool/") { @@ -311,6 +382,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } + if creds != nil && !creds.Empty() { + req.SetBasicAuth(creds.Username, creds.Password) + } var resp *http.Response if flags&fetchBulk != 0 { resp, err = bulkDo(req) @@ -325,7 +399,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea switch resp.StatusCode { case 200: // ok - case 401, 404: + case 401: + return nil, fmt.Errorf("cannot fetch from %q: unauthorized", index.label) + case 404: return nil, fmt.Errorf("cannot find archive data") default: return nil, fmt.Errorf("error from archive: %v", resp.Status) @@ -363,3 +439,13 @@ func sectionPackageInfo(section control.Section) *PackageInfo { SHA256: section.Get("SHA256"), } } + +// proSuffixedLabel adds " (pro)" suffix to the label and returns it +// if the archive is specified with pro value. Otherwise, it returns the +// original label. +func (index *ubuntuIndex) proSuffixedLabel() string { + if index.archive.options.Pro == "" { + return index.label + } + return index.label + " " + index.archive.options.Pro + " (pro)" +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 703148fb..cf586bab 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -40,9 +40,12 @@ type httpSuite struct { var _ = Suite(&httpSuite{}) var ( - key1 = testutil.PGPKeys["key1"] - key2 = testutil.PGPKeys["key2"] - keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + key1 = testutil.PGPKeys["key1"] + key2 = testutil.PGPKeys["key2"] + keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + keyUbuntuFIPSv1 = testutil.PGPKeys["key-ubuntu-fips-v1"] + keyUbuntuApps = testutil.PGPKeys["key-ubuntu-apps"] + keyUbuntuESMv2 = testutil.PGPKeys["key-ubuntu-esm-v2"] ) func (s *httpSuite) SetUpTest(c *C) { @@ -178,6 +181,16 @@ var optionErrorTests = []optionErrorTest{{ Components: []string{"main", "other"}, }, error: `invalid package architecture: foo`, +}, { + options: archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "other"}, + Pro: "invalid", + }, + error: `invalid pro value: "invalid"`, }} func (s *httpSuite) TestOptionErrors(c *C) { @@ -328,44 +341,112 @@ func (s *httpSuite) TestArchiveLabels(c *C) { } } - s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + tests := []struct { + summary string + label string + err string + }{{ + summary: "Ubuntu label", + label: "Ubuntu", + }, { + summary: "Unknown label", + label: "Unknown", + err: "corrupted archive InRelease file: no Ubuntu section", + }} + + for _, test := range tests { + c.Logf("Summary: %s", test.summary) - options := archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + var adjust func(*testarchive.Release) + if test.label != "" { + adjust = setLabel(test.label) + } + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, adjust) + + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err := archive.Open(&options) + if test.err != "" { + c.Assert(err, ErrorMatches, test.err) + } else { + c.Assert(err, IsNil) + } } +} - _, err := archive.Open(&options) +func (s *httpSuite) TestProArchives(c *C) { + setLabel := func(label string) func(*testarchive.Release) { + return func(r *testarchive.Release) { + r.Label = label + } + } + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "" + for _, info := range archive.ProArchiveInfo { + contents += fmt.Sprintf("machine %s login foo password bar\n", info.BaseURL) + } + err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("Ubuntu")) + do := func(req *http.Request) (*http.Response, error) { + auth, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + c.Assert(auth, DeepEquals, []string{"Basic Zm9vOmJhcg=="}) + return s.Do(req) + } + restoreDo := archive.FakeDo(do) + defer restoreDo() - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err = archive.Open(&options) + c.Assert(err, IsNil) } - _, err = archive.Open(&options) - c.Assert(err, IsNil) + // Test non-pro archives. + do = func(req *http.Request) (*http.Response, error) { + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, false, Commentf("Non-pro archives should not have any authorization header")) + return s.Do(req) + } + restoreDo = archive.FakeDo(do) + defer restoreDo() - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("UbuntuProFIPS")) + s.base = "http://archive.ubuntu.com/ubuntu/" + s.prepareArchive("focal", "20.04", "amd64", []string{"main"}) - options = archive.Options{ + options := archive.Options{ Label: "ubuntu", - Version: "22.04", + Version: "20.04", Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, + Suites: []string{"focal"}, + Components: []string{"main"}, CacheDir: c.MkDir(), PubKeys: []*packet.PublicKey{s.pubKey}, } @@ -373,20 +454,41 @@ func (s *httpSuite) TestArchiveLabels(c *C) { _, err = archive.Open(&options) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("ThirdParty")) - - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + // Test Pro archives with bad credentials. + do = func(req *http.Request) (*http.Response, error) { + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + if strings.Contains(req.URL.String(), "/pool/") { + s.status = 401 + } else { + s.status = 200 + } + return s.Do(req) } + restoreDo = archive.FakeDo(do) + defer restoreDo() - _, err = archive.Open(&options) - c.Assert(err, ErrorMatches, `.*\bno Ubuntu section`) + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) + + _, _, err = testArchive.Fetch("mypkg1") + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) + } } type verifyArchiveReleaseTest struct { @@ -491,45 +593,177 @@ func read(r io.Reader) string { } // ---------------------------------------------------------------------------------------- -// Real archive tests, only enabled via --real-archive. +// Real archive tests, only enabled via: +// 1. --real-archive for non-Pro archives (e.g. standard jammy archive), +// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). +// +// To run the tests for Ubuntu Pro archives, the host machine must be Pro +// enabled and relevant Pro services must be enabled. The following commands +// might help: +// sudo pro attach --no-auto-enable +// sudo pro enable fips-updates esm-apps esm-infra --assume-yes var realArchiveFlag = flag.Bool("real-archive", false, "Perform tests against real archive") +var proArchiveFlag = flag.Bool("real-pro-archive", false, "Perform tests against real Ubuntu Pro archive") func (s *S) TestRealArchive(c *C) { if !*realArchiveFlag { c.Skip("--real-archive not provided") } - for _, release := range ubuntuReleases { - for _, arch := range elfToDebArch { - s.testOpenArchiveArch(c, release, arch) + s.runRealArchiveTests(c, realArchiveTests) +} + +func (s *S) TestRealProArchives(c *C) { + if !*proArchiveFlag { + c.Skip("--real-pro-archive not provided") + } + s.runRealArchiveTests(c, proArchiveTests) + s.testRealProArchiveBadCreds(c) +} + +func (s *S) runRealArchiveTests(c *C, tests []realArchiveTest) { + allArch := make([]string, 0, len(elfToDebArch)) + for _, arch := range elfToDebArch { + allArch = append(allArch, arch) + } + for _, test := range tests { + if len(test.archs) == 0 { + test.archs = allArch + } + for _, arch := range test.archs { + s.testOpenArchiveArch(c, test, arch) } } } -type ubuntuRelease struct { +type realArchiveTest struct { name string version string + suites []string + components []string + pro string archivePubKeys []*packet.PublicKey + archs []string + pkg string + path string } -var ubuntuReleases = []ubuntuRelease{{ - name: "focal", - version: "20.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, +var realArchiveTests = []realArchiveTest{{ + name: "focal", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/bin/hostname", }, { - name: "jammy", - version: "22.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, + name: "jammy", + version: "22.04", + suites: []string{"jammy"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/bin/hostname", }, { - name: "noble", - version: "24.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, + name: "noble", + version: "24.04", + suites: []string{"noble"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/usr/bin/hostname", +}} + +var proArchiveTests = []realArchiveTest{{ + name: "focal-fips", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main"}, + pro: "fips", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "focal-fips-updates", + version: "20.04", + suites: []string{"focal-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "focal-esm-apps", + version: "20.04", + suites: []string{"focal-apps-security", "focal-apps-updates"}, + components: []string{"main"}, + pro: "apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "focal-esm-infra", + version: "20.04", + suites: []string{"focal-infra-security", "focal-infra-updates"}, + components: []string{"main"}, + pro: "infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "jammy-fips-updates", + version: "22.04", + suites: []string{"jammy-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "jammy-esm-apps", + version: "22.04", + suites: []string{"jammy-apps-security", "jammy-apps-updates"}, + components: []string{"main"}, + pro: "apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "jammy-esm-infra", + version: "22.04", + suites: []string{"jammy-infra-security", "jammy-infra-updates"}, + components: []string{"main"}, + pro: "infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "noble-esm-apps", + version: "24.04", + suites: []string{"noble-apps-security", "noble-apps-updates"}, + components: []string{"main"}, + pro: "apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "noble-esm-infra", + version: "24.04", + suites: []string{"noble-infra-security", "noble-infra-updates"}, + components: []string{"main"}, + pro: "infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", }} var elfToDebArch = map[elf.Machine]string{ @@ -551,17 +785,18 @@ func (s *S) checkArchitecture(c *C, arch string, binaryPath string) { c.Assert(binaryArch, Equals, arch) } -func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { - c.Logf("Checking ubuntu archive %s %s...", release.name, arch) +func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { + c.Logf("Checking ubuntu archive %s %s...", test.name, arch) options := archive.Options{ Label: "ubuntu", - Version: release.version, + Version: test.version, Arch: arch, - Suites: []string{release.name}, - Components: []string{"main", "universe"}, + Suites: test.suites, + Components: test.components, CacheDir: c.MkDir(), - PubKeys: release.archivePubKeys, + Pro: test.pro, + PubKeys: test.archivePubKeys, } testArchive, err := archive.Open(&options) @@ -569,30 +804,56 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { extractDir := c.MkDir() - pkg, info, err := testArchive.Fetch("hostname") + pkg, info, err := testArchive.Fetch(test.pkg) c.Assert(err, IsNil) - c.Assert(info.Name, DeepEquals, "hostname") + c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) err = deb.Extract(pkg, &deb.ExtractOptions{ - Package: "hostname", + Package: test.pkg, TargetDir: extractDir, Extract: map[string][]deb.ExtractInfo{ - "/usr/share/doc/hostname/copyright": { + fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, }, - "/bin/hostname": { - {Path: "/hostname"}, + test.path: { + {Path: "/binary"}, }, }, }) c.Assert(err, IsNil) - data, err := os.ReadFile(filepath.Join(extractDir, "copyright")) + s.checkArchitecture(c, arch, filepath.Join(extractDir, "binary")) +} + +func (s *S) testRealProArchiveBadCreds(c *C) { + c.Logf("Cannot fetch Pro packages with bad credentials") + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "machine esm.ubuntu.com/fips/ubuntu/ login bearer password invalid" + err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) - copyrightTop := "This package was written by Peter Tobias " - c.Assert(strings.Contains(string(data), copyrightTop), Equals, true) + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: "fips", + PubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + } + + // The archive can be "opened" without any credentials since the dists/ path + // containing InRelease files, does not require any credentials. + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) - s.checkArchitecture(c, arch, filepath.Join(extractDir, "hostname")) + _, _, err = testArchive.Fetch("openssh-client") + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) } diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index cd75d5f4..3569a699 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -19,3 +19,5 @@ type Credentials = credentials var FindCredentials = findCredentials var FindCredentialsInDir = findCredentialsInDir + +var ProArchiveInfo = proArchiveInfo diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 00337bb6..ca9b2111 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -26,6 +26,13 @@ type Release struct { Archives map[string]*Archive } +const ( + ProFIPS = "fips" + ProFIPSUpdates = "fips-updates" + ProApps = "apps" + ProInfra = "infra" +) + // Archive is the location from which binary packages are obtained. type Archive struct { Name string @@ -33,6 +40,7 @@ type Archive struct { Suites []string Components []string Priority int + Pro string PubKeys []*packet.PublicKey } @@ -398,7 +406,8 @@ type yamlArchive struct { Version string `yaml:"version"` Suites []string `yaml:"suites"` Components []string `yaml:"components"` - Priority *int `yaml:"priority"` + Priority *int `yaml:"priority"` + Pro string `yaml:"pro"` Default bool `yaml:"default"` PubKeys []string `yaml:"public-keys"` } @@ -553,6 +562,12 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if len(details.Components) == 0 { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } + switch details.Pro { + case "", ProApps, ProFIPS, ProFIPSUpdates, ProInfra: + default: + logf("Ignoring archive %q (invalid pro value: %q)...", archiveName, details.Pro) + continue + } if details.Default && defaultArchive != "" { if archiveName < defaultArchive { archiveName, defaultArchive = defaultArchive, archiveName @@ -591,6 +606,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Version: details.Version, Suites: details.Suites, Components: details.Components, + Pro: details.Pro, Priority: priority, PubKeys: archiveKeys, } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index d8d2f564..3866da12 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1739,6 +1739,117 @@ var setupTests = []setupTest{{ }, }, }, +}, { + summary: "Pro values in archives", + input: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + ubuntu: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + public-keys: [test-key] + fips: + version: 20.04 + components: [main] + suites: [focal] + pro: fips + priority: 20 + public-keys: [test-key] + fips-updates: + version: 20.04 + components: [main] + suites: [focal-updates] + pro: fips-updates + priority: 21 + public-keys: [test-key] + apps: + version: 20.04 + components: [main] + suites: [focal-apps-security] + pro: apps + priority: 16 + public-keys: [test-key] + infra: + version: 20.04 + components: [main] + suites: [focal-infra-security] + pro: infra + priority: 15 + public-keys: [test-key] + ignored: + version: 20.04 + components: [main] + suites: [foo] + pro: unknown-value + priority: 10 + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Priority: 10, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips": { + Name: "fips", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Pro: "fips", + Priority: 20, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips-updates": { + Name: "fips-updates", + Version: "20.04", + Suites: []string{"focal-updates"}, + Components: []string{"main"}, + Pro: "fips-updates", + Priority: 21, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "apps": { + Name: "apps", + Version: "20.04", + Suites: []string{"focal-apps-security"}, + Components: []string{"main"}, + Pro: "apps", + Priority: 16, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "infra": { + Name: "infra", + Version: "20.04", + Suites: []string{"focal-infra-security"}, + Components: []string{"main"}, + Pro: "infra", + Priority: 15, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, }, { summary: "Default is ignored", input: map[string]string{ diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 6cdb8f01..4974d623 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1445,6 +1445,29 @@ var slicerTests = []slicerTest{{ contents: `, }, +}, { + summary: "No valid archives defined due to invalid pro value", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + invalid: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + public-keys: [test-key] + pro: unknown-value + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, }} var defaultChiselYaml = ` @@ -1537,6 +1560,7 @@ func (s *S) TestRun(c *C) { Version: setupArchive.Version, Suites: setupArchive.Suites, Components: setupArchive.Components, + Pro: setupArchive.Pro, Arch: test.arch, }, Packages: pkgs, diff --git a/internal/testutil/pgpkeys.go b/internal/testutil/pgpkeys.go index e999a9a3..98eb6b05 100644 --- a/internal/testutil/pgpkeys.go +++ b/internal/testutil/pgpkeys.go @@ -21,6 +21,18 @@ var PGPKeys = map[string]*PGPKeyData{ ID: "871920D1991BC93C", PubKeyArmor: pubKeyUbuntu2018Armor, }, + "key-ubuntu-fips-v1": { + ID: "C1997C40EDE22758", + PubKeyArmor: pubKeyUbuntuFIPSv1Armor, + }, + "key-ubuntu-apps": { + ID: "AB01A101DB53907B", + PubKeyArmor: pubKeyUbuntuAppsArmor, + }, + "key-ubuntu-esm-v2": { + ID: "4067E40313CB4B13", + PubKeyArmor: pubKeyUbuntuESMv2Armor, + }, "key1": { ID: "854BAF1AA9D76600", PubKeyArmor: pubKey1Armor, @@ -87,6 +99,111 @@ uOgcXny1UlwtCUzlrSaP -----END PGP PUBLIC KEY BLOCK----- ` +// Ubuntu Federal Information Processing Standards Automatic Signing Key V1 . +// ID: C1997C40EDE22758. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuFIPSv1Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFzZxGABEADSWmX0+K//0cosKPyr5m1ewmwWKjRo/KBPTyR8icHhbBWfFd8T +DtYggvQHPU0YnKRcWits0et8JqSgZttNa28s7SaSUTBzfgzFJZgULAi/4i8u8TUj ++KH2zSoUX55NKC9aozba1cR66jM6O/BHXK5YoZzTpmiY1AHlIWAJ9s6cCClhnYMR +zwxSZVbefcYFbVPX/dQw/FMvJVeZ2aQ18NDgMQciu786aYklMFowxWNs/eLLTqum +cDHaw9UpKyhgfL/mkaIXuhYy6YRByYq1oOnJ5XffAOtovvCti1MvsPc0NDhPiGLf +9Fd/GtnqHxzVDqZmtUXX50mGu4LnJoHgWRjml3mapDPStzFr7Xgbb0NnyflmxnfN +kQcu2lFyXFfndWwg/RAOFdBPxBQhRK52uZiCfydKD7zCXz9YGm9xEK541EG0FrwA +6Vk1xaFol/jI8MQdP1o3JySX0Pqva3IHF7FHWHmxrIPaJLIHi0IrFG6Fgmk4sQ2w +XSc8kbxR+wYYKqIhBUZP0eb1jkFfvRVS6YvAy18xtw5pFD+VURdA0Uu5cotESfyz +oHsQ5R7wzg76oV/mYukHGC0x8peqxiPwbyhGFAhG8eUR66iYZgGbzmNI+OJz2EUi +UZJJXt4rnI1RVJLbhK9RjeobkOjf58Cm8RExlqJU16gy9saCMSiAqHx8swARAQAB +tFxVYnVudHUgRmVkZXJhbCBJbmZvcm1hdGlvbiBQcm9jZXNzaW5nIFN0YW5kYXJk +cyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgVjEgPGVzbUBjYW5vbmljYWwuY29tPokC +OAQTAQIAIgUCXNnEYAIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQwZl8 +QO3iJ1j4Vw//SawfmZi1GW+EUnuPqSz+zcmIKdx6AWZTe9/vSj6jgq4SYt//LAiD +NQz3dn2m0m5AaCucza2BCixUBrNhMh66m+lXfTqymUtTIpWpu4L1WLUbPjQ+s3Ad +xuF7S5wJtQrYmPvmZduZgg1wcb8eaqVltRJREpOP6sxcuqtvcfv4v4QYZ+iYd7eJ +8fxPOiyJEOTQPTdPZahYTaUOIloN5pT6uVg03u59Kh4aHCYxlRorvuRBabdctCfA +EBgomk4Us20Tv31dqlvMAiGKJqf1wdjhzlUmk4g/fOiRSNETKSC/VeUGH0fSbizl +Gs7Mg60jChPKpwzB6Rb5Nv2/Aw/FlSkfFhMdCdfKjl8IWOMPmElTVJFyVx1mmURi +3LgsloDFmJfebXefSFA7S8KLyBGlZJ/APaym64Ls12PUOjfh1Glie3E8KO66AGLo +ID1dQnzRizuHxW80ET03dSjzTXHLSi+iFycmNAxo6gB3GyOQ8tlIHjo1FfDfNYDf +qKic3Q0B9TvF6hqVRIcyePK4lN5YtRpVRdVj/jv8AqbzaIaVCP4k4nNrbaVx5zQf +BWq2E9IH+vLZfPyiP+hwxswfrlU3mrXBpPStIxq41yXFwQiDnqgkhEVAcrYPjBnS +T6s3+b+4HbAW6mbp4jEHUd/F1+iXz90T2WArrNIkMbmpChMuSyRN8Hc= +=DWhM +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Apps Automatic Signing Key . +// ID: AB01A101DB53907B. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuAppsArmor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF3WVA4BEAC7MDr8HClfKptSd4VeB12Vy+Ao/4NpY2ITdkRed4vfh/4eBWWn +3+in6So2ekweifACSxScB/M9zVObsI1cab7QPMkIiATNUfIyOEP7iNWLX4+AytM1 +LP3bZo8OpghnLZNstCGbiRUO4CDNmCI04DOPCu9EVEO4WWNuWIMRwCLShDSf7Cid +J2fn2TT/7vsmA4eI3YnAne+u8g4X2zMHQFkHANhylB0lPyThXo5jaxHImzm4wf/2 +LF8f1Y1nRQObS2jcvYc3fm9B7iOGpyNAw3h6hrPKH5T9tY/ZoMtFHqn66J1CBSHb +hDkEvA46X50su4yAHeSiEG/hMYG7SoHzmAsjEXnvkTIE41WhmxlidQnRs2uWy34U +7VmOpaidWn3R99fNHYOtSOB6bpIvls8snWSQ63jcFXnt05nVZsp/Ixzl0Oqitynx +DFwoxEwt3ZuCHwxbx2vZ+FiZXVFN7I0IyBDOEL6XS27FNaMCZ7Q/6z/ckdWto55E +264OWf9lnw31bXFXHWSusRXWzD6FK8dqWgjtrWwRxlvF4jm688lqpjac6fFES3UK +BhjyHXFGL/+HHZ9CNxlLYF5QnXq1mGR0Ykw975u8KoOFSLBqsx+1a21m6dfzujY7 +2Gq6Sju+9Yo1aOF+CNvTMYdRBoDL4sFj6VAmUsszMA5aAb+82pOCaDvGJQARAQAB +tDVVYnVudHUgQXBwcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGVzbUBjYW5vbmlj +YWwuY29tPokCOAQTAQIAIgUCXdZUDgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC +F4AACgkQqwGhAdtTkHuTOw/8Czv42TSpwHz+eNtl3ZFyxta9rR/qWC3h+vMu0R/l +5KU3aQQOygWOoUcr1QTPSSg3v/H+v/8vqVq2UuUxSIfpMxBj2kIX2vqskv6Roez7 +xR8lVDa0a47z/NYMfKpxrEJxOLh/c7I6aAsa597bTqDHtucHL/22BvfUJJqw6jq1 +7SswP5lqKPBFz7x+E2hgfJE7Vn7h0ICm29FkWnOeTKfj8VwTAeKXKUI9Hw6+aqr9 +29Y2NdLsYZ57mpivRLNM9sBZoF3avP1pUC2k0IwP3dwh4AxUMXjRRPh173iXBfR2 +yAf1lWET/5+8dSBrfFIZSo+FF/EEBmqIVtJpHkq8+YxUbCLbkoikRi2kwrgyXLEn +FqxSU2Ab0xurFHiHcJoCGVD38xjznO5cQl7H4K9+B/rFpTTowOHbOcFpKAzpYqB5 +8rnR1yRSsB33zac8xesUIfzYWRtLc5/VIb5mOkWlb62d8emILx2XuRFVjKq6mKki +oGckhDUOuEFrjW1cQq+PWBBxyJoXcy6wGSoPJ/ELeaf9zg8SF0jwuN6BPHVBeJ/E +W53zR5iV0N9fRT+M2JN5tc5HenO92xLgPAh+GPWLYmPdTmHu+kFozqsHx/NUw2iP +PBL6Q1VZytt2Uf6qLPUx7GpYMKf42Vldb0feFo/YA/lzOgPlY29pDLKXbse6o+Sr +kmk= +=AEEr +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Extended Security Maintenance Automatic Signing Key v2 . +// ID: 4067E40313CB4B13. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuESMv2Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY +680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs +8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t +YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 +t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI +OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG +JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 +rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 +K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT +Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE +6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB +tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj +IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 +kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C +NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB +m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S +nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 +1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F +EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC +puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG +tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C +DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS +89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 +RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi +JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc +=KWLs +-----END PGP PUBLIC KEY BLOCK----- +` + // Test-purpose RSA 2048 bits signing key-pairs without a passphrase. // ID: 854BAF1AA9D76600. User: "foo-bar ". const pubKey1Armor = ` diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ee1c284c..958a5970 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -41,6 +41,8 @@ parts: override-build: | go generate ./cmd craftctl default + stage: + - -bin/chrorder chisel-release-data: plugin: nil @@ -56,9 +58,17 @@ parts: craftctl set grade="$grade" after: [chisel] +plugs: + pro-credentials: + interface: system-files + read: + - /etc/apt/auth.conf.d + - /etc/apt/auth.conf.d/90ubuntu-advantage + apps: chisel: command: bin/chisel plugs: - network - home + - pro-credentials diff --git a/spread.yaml b/spread.yaml index a8de3d36..274bbd4b 100644 --- a/spread.yaml +++ b/spread.yaml @@ -4,6 +4,7 @@ path: /chisel environment: OS: ubuntu + PRO_TOKEN: $(HOST:echo $PRO_TOKEN) backends: # Cannot use LXD backend due to https://github.com/snapcore/spread/issues/154 diff --git a/tests/pro-archives/chisel-releases/chisel.yaml b/tests/pro-archives/chisel-releases/chisel.yaml new file mode 100644 index 00000000..7de9d8c5 --- /dev/null +++ b/tests/pro-archives/chisel-releases/chisel.yaml @@ -0,0 +1,45 @@ +format: v1 + +archives: + ubuntu: + version: 24.04 + pro: infra + components: [main] + suites: [noble-infra-security, noble-infra-updates] + public-keys: [ubuntu-esm-key-v2] + +public-keys: + # Ubuntu Extended Security Maintenance Automatic Signing Key v2 + # rsa4096/56f7650a24c9e9ecf87c4d8d4067e40313cb4b13 2019-04-17T02:33:33Z + ubuntu-esm-key-v2: + id: "4067E40313CB4B13" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY + 680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs + 8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t + YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 + t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI + OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG + JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 + rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 + K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT + Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE + 6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB + tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj + IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 + kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C + NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB + m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S + nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 + 1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F + EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC + puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG + tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C + DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS + 89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 + RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi + JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc + =KWLs + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/pro-archives/chisel-releases/slices/hello.yaml b/tests/pro-archives/chisel-releases/slices/hello.yaml new file mode 100644 index 00000000..7bd1a393 --- /dev/null +++ b/tests/pro-archives/chisel-releases/slices/hello.yaml @@ -0,0 +1,13 @@ +package: hello + +essential: + - hello_copyright + +slices: + bins: + contents: + /usr/bin/hello: + + copyright: + contents: + /usr/share/doc/hello/copyright: diff --git a/tests/pro-archives/task.yaml b/tests/pro-archives/task.yaml new file mode 100644 index 00000000..262baab9 --- /dev/null +++ b/tests/pro-archives/task.yaml @@ -0,0 +1,22 @@ +summary: Chisel can fetch packages from Ubuntu Pro archives + +variants: + - noble + +environment: + ROOTFS: rootfs + +prepare: | + apt update && apt install -y ubuntu-pro-client + pro attach ${PRO_TOKEN} --no-auto-enable + pro enable esm-infra --assume-yes + mkdir ${ROOTFS} + +restore: | + pro detach --assume-yes + rm -r ${ROOTFS} + +execute: | + chisel cut --release ./chisel-releases/ --root ${ROOTFS} hello_bins + test -f ${ROOTFS}/usr/bin/hello + test -f ${ROOTFS}/usr/share/doc/hello/copyright From 2ac8cd814d120c2b2694a11587d48c02b2ce9f27 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 18 Oct 2024 17:45:17 +0200 Subject: [PATCH 02/13] lint: extra space --- internal/setup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index ca9b2111..0309b538 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -406,7 +406,7 @@ type yamlArchive struct { Version string `yaml:"version"` Suites []string `yaml:"suites"` Components []string `yaml:"components"` - Priority *int `yaml:"priority"` + Priority *int `yaml:"priority"` Pro string `yaml:"pro"` Default bool `yaml:"default"` PubKeys []string `yaml:"public-keys"` From d872d169d7b092656b1a1382f305f5d029034129 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 12 Nov 2024 14:11:07 +0100 Subject: [PATCH 03/13] README, new version, first draft --- README.md | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index cf5b5f8b..b72049ee 100644 --- a/README.md +++ b/README.md @@ -67,24 +67,24 @@ provided packages and install only the desired slices into the *myrootfs* folder, according to the slice definitions available in the ["ubuntu-22.04" chisel-releases branch](). -## Chisel support for Pro archives +## Support for Pro archives -Chisel can also fetch and install packages from Ubuntu Pro archives. For this, -the archive has to be defined with the `archives..pro` field in -chisel.yaml and its credentials have to be made available to Chisel. +To fetch and install packages from Ubuntu Pro archives, +the archive has to be defined with the `archives..pro` field in +`chisel.yaml`: ```yaml # chisel.yaml format: v1 archives: - : + : pro: ... ... ``` -Chisel currently supports the following Pro archives: +The following Pro archives are currently supported: | `pro` value | Archive URL | Related Ubuntu Pro service | | - | - | - | @@ -93,22 +93,21 @@ Chisel currently supports the following Pro archives: | apps | https://esm.ubuntu.com/apps/ubuntu | esm-apps | | infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | -Authentication to Pro archives requires that the host is Pro or it is equipped -with the Pro credentials. By default, Chisel will support using credentials -from the `/etc/apt/auth.conf.d/` directory, but this location can be configured -using the environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have -read permission for the necessary credentials files. +If the system is using the [Pro client](https://ubuntu.com/pro/tutorial), and the +services are enabled, the credentials will be automatically picked up from +`/etc/apt/auth.conf.d/`. However, the default permissions of the credentials need +to be changed for Chisel to be able to read them. Example workflow: +```shell +sudo pro enable fips -The format of the files is documented in the -[apt_auth.conf(5)](https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html) -man page. Below is a snippet of the `/etc/apt/auth.conf.d/90ubuntu-advantage` -file from a host with the `fips-updates` and `infra` archives enabled: - -``` -machine esm.ubuntu.com/infra/ubuntu/ login bearer password -machine esm.ubuntu.com/fips-updates/ubuntu/ login bearer password +sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage +# or, alternatively, +sudo chmod u+r /etc/apt/auth.conf.d/90ubuntu-advantage ``` +Alternatively, the location of the credentials can be configured using the environment variable +`CHISEL_AUTH_DIR`. + ## Reference ### Chisel releases From 3adec2115f52d7c843696754c451811025656d48 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 12 Nov 2024 16:08:31 +0100 Subject: [PATCH 04/13] Change log to avoid using gerunds --- cmd/chisel/cmd_cut.go | 2 +- internal/setup/setup.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 80b9f9eb..5c3088b3 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -76,7 +76,7 @@ func (cmd *cmdCut) Execute(args []string) error { }) if err != nil { if err == archive.ErrCredentialsNotFound { - logf("Ignoring archive %q (credentials not found)...", archiveName) + logf("Archive %q ignored: credentials not found", archiveName) continue } return err diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 0309b538..0bec1529 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -565,7 +565,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { switch details.Pro { case "", ProApps, ProFIPS, ProFIPSUpdates, ProInfra: default: - logf("Ignoring archive %q (invalid pro value: %q)...", archiveName, details.Pro) + logf("Archive %q ignored: invalid pro value: %q", archiveName, details.Pro) continue } if details.Default && defaultArchive != "" { From 3d457a9f712677332fc7fec6d878962671d4ed3d Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 12 Nov 2024 16:18:28 +0100 Subject: [PATCH 05/13] refactor archiveURL to return error and creds --- internal/archive/archive.go | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 9b81bda6..3bb490c3 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -172,16 +172,24 @@ var proArchiveInfo = map[string]struct { }, } -// archiveURL returns the archive base URL depending on the "pro" value and -// selected architecture "arch". -func archiveURL(pro, arch string) string { +func archiveURL(pro, arch string) (string, *credentials, error) { if pro != "" { - return proArchiveInfo[pro].BaseURL + archiveInfo, ok := proArchiveInfo[pro] + if !ok { + return "", nil, fmt.Errorf("invalid pro value: %q", pro) + } + url := archiveInfo.BaseURL + creds, err := findCredentials(url) + if err != nil { + return "", nil, err + } + return url, creds, nil } + if arch == "amd64" || arch == "i386" { - return ubuntuURL + return ubuntuURL, nil, nil } - return ubuntuPortsURL + return ubuntuPortsURL, nil, nil } func openUbuntu(options *Options) (Archive, error) { @@ -194,20 +202,10 @@ func openUbuntu(options *Options) (Archive, error) { if len(options.Version) == 0 { return nil, fmt.Errorf("archive options missing version") } - if options.Pro != "" { - if _, ok := proArchiveInfo[options.Pro]; !ok { - return nil, fmt.Errorf("invalid pro value: %q", options.Pro) - } - } - baseURL := archiveURL(options.Pro, options.Arch) - var creds *credentials - if options.Pro != "" { - var err error - creds, err = findCredentials(baseURL) - if err != nil { - return nil, err - } + baseURL, creds, err := archiveURL(options.Pro, options.Arch) + if err != nil { + return nil, err } archive := &ubuntuArchive{ From b163f3530707653459d26fb1a6e65788939d86bd Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 12 Nov 2024 16:20:07 +0100 Subject: [PATCH 06/13] remove noisy warning --- internal/archive/archive.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 3bb490c3..f067d218 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -239,8 +239,6 @@ func openUbuntu(options *Options) (Archive, error) { if !index.supportsArch(options.Arch) { // Release does not support the specified architecture, do // not add any of its indexes. - logf("Warning: ignoring %s %s %s suite (unsupported arch %s)...", - index.proSuffixedLabel(), index.version, index.suite, options.Arch) break } err = index.checkComponents(options.Components) From d1986468ced3757e8f48a616aed5ce2fdc854886 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 12 Nov 2024 16:21:00 +0100 Subject: [PATCH 07/13] rename to displayName --- internal/archive/archive.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index f067d218..26455f0c 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -258,7 +258,7 @@ func openUbuntu(options *Options) (Archive, error) { } func (index *ubuntuIndex) fetchRelease() error { - logf("Fetching %s %s %s suite details...", index.proSuffixedLabel(), index.version, index.suite) + logf("Fetching %s %s %s suite details...", index.displayName(), index.version, index.suite) reader, err := index.fetch("InRelease", "", fetchDefault) if err != nil { return err @@ -315,7 +315,7 @@ func (index *ubuntuIndex) fetchIndex() error { return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component) } - logf("Fetching index for %s %s %s %s component...", index.proSuffixedLabel(), index.version, index.suite, index.component) + logf("Fetching index for %s %s %s %s component...", index.displayName(), index.version, index.suite, index.component) reader, err := index.fetch(packagesPath+".gz", digest, fetchBulk) if err != nil { return err @@ -436,10 +436,7 @@ func sectionPackageInfo(section control.Section) *PackageInfo { } } -// proSuffixedLabel adds " (pro)" suffix to the label and returns it -// if the archive is specified with pro value. Otherwise, it returns the -// original label. -func (index *ubuntuIndex) proSuffixedLabel() string { +func (index *ubuntuIndex) displayName() string { if index.archive.options.Pro == "" { return index.label } From e8be8301bed4c96cb4e44b648790c7df091479e8 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 12 Nov 2024 16:23:39 +0100 Subject: [PATCH 08/13] reverse the dependency archive->setup --- internal/archive/archive.go | 16 +++++++++++----- internal/setup/setup.go | 10 ++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 26455f0c..0a2890fe 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -14,7 +14,6 @@ import ( "github.com/canonical/chisel/internal/control" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" - "github.com/canonical/chisel/internal/setup" ) type Archive interface { @@ -151,22 +150,29 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" +const ( + ProFIPS = "fips" + ProFIPSUpdates = "fips-updates" + ProApps = "apps" + ProInfra = "infra" +) + var proArchiveInfo = map[string]struct { BaseURL, Label string }{ - setup.ProFIPS: { + ProFIPS: { BaseURL: "https://esm.ubuntu.com/fips/ubuntu/", Label: "UbuntuFIPS", }, - setup.ProFIPSUpdates: { + ProFIPSUpdates: { BaseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", Label: "UbuntuFIPSUpdates", }, - setup.ProApps: { + ProApps: { BaseURL: "https://esm.ubuntu.com/apps/ubuntu/", Label: "UbuntuESMApps", }, - setup.ProInfra: { + ProInfra: { BaseURL: "https://esm.ubuntu.com/infra/ubuntu/", Label: "UbuntuESM", }, diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 0bec1529..eb8114f5 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -13,6 +13,7 @@ import ( "golang.org/x/crypto/openpgp/packet" "gopkg.in/yaml.v3" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" "github.com/canonical/chisel/internal/strdist" @@ -26,13 +27,6 @@ type Release struct { Archives map[string]*Archive } -const ( - ProFIPS = "fips" - ProFIPSUpdates = "fips-updates" - ProApps = "apps" - ProInfra = "infra" -) - // Archive is the location from which binary packages are obtained. type Archive struct { Name string @@ -563,7 +557,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } switch details.Pro { - case "", ProApps, ProFIPS, ProFIPSUpdates, ProInfra: + case "", archive.ProApps, archive.ProFIPS, archive.ProFIPSUpdates, archive.ProInfra: default: logf("Archive %q ignored: invalid pro value: %q", archiveName, details.Pro) continue From 2b0a13e1ae6aefa89ceaaa6d679af32ee38e59b8 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 12 Nov 2024 16:30:00 +0100 Subject: [PATCH 09/13] address Cris comments on the README --- README.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b72049ee..32f65240 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,11 @@ folder, according to the slice definitions available in the ["ubuntu-22.04" chisel-releases branch](). ## Support for Pro archives +> [!IMPORTANT] +> To chisel a Pro package you need to have a Pro-enabled host. -To fetch and install packages from Ubuntu Pro archives, -the archive has to be defined with the `archives..pro` field in -`chisel.yaml`: +To fetch and install slices from Ubuntu Pro packages, the Pro archive has to be +defined with the `archives..pro` field in `chisel.yaml`: ```yaml @@ -86,17 +87,17 @@ archives: The following Pro archives are currently supported: -| `pro` value | Archive URL | Related Ubuntu Pro service | -| - | - | - | -| fips | https://esm.ubuntu.com/fips/ubuntu | fips | -| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | fips-updates | -| apps | https://esm.ubuntu.com/apps/ubuntu | esm-apps | -| infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | +| `pro` value | Archive URL | Corresponding Ubuntu Pro service | +| - | - |----------------------------------| +| fips | https://esm.ubuntu.com/fips/ubuntu | fips | +| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | fips-updates | +| apps | https://esm.ubuntu.com/apps/ubuntu | esm-apps | +| infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | If the system is using the [Pro client](https://ubuntu.com/pro/tutorial), and the services are enabled, the credentials will be automatically picked up from -`/etc/apt/auth.conf.d/`. However, the default permissions of the credentials need -to be changed for Chisel to be able to read them. Example workflow: +`/etc/apt/auth.conf.d/`. However, the default permissions of the credentials file +need to be changed so that Chisel can read it. Example: ```shell sudo pro enable fips @@ -105,7 +106,7 @@ sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage sudo chmod u+r /etc/apt/auth.conf.d/90ubuntu-advantage ``` -Alternatively, the location of the credentials can be configured using the environment variable +The location of the credentials can be configured using the environment variable `CHISEL_AUTH_DIR`. ## Reference From 9d77322ebec21c395504743d31fe94f9a0391122 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Wed, 13 Nov 2024 14:49:53 +0100 Subject: [PATCH 10/13] feat: changes to the CI to use secrets properly --- .github/workflows/pro_tests.yaml | 81 ++++++++++++++++++++++++++++++++ .github/workflows/spread.yml | 4 +- .github/workflows/tests.yaml | 30 +----------- 3 files changed, 83 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/pro_tests.yaml diff --git a/.github/workflows/pro_tests.yaml b/.github/workflows/pro_tests.yaml new file mode 100644 index 00000000..af52a20d --- /dev/null +++ b/.github/workflows/pro_tests.yaml @@ -0,0 +1,81 @@ +name: Pro Tests + +on: + workflow_dispatch: + push: + paths-ignore: + - '**.md' + schedule: + - cron: "0 0 */2 * *" + workflow_run: + workflows: ["CLA check"] + types: + - completed + +jobs: + real-archive-tests: + name: Real Archive Tests + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + # Do not change to newer releases as "fips" may not be available there. + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run real archive tests + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + set -ex + + detach() { + sudo pro detach --assume-yes || true + sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage + } + trap detach EXIT + + # Attach pro token and enable services + sudo pro attach ${PRO_TOKEN} --no-auto-enable + + # Cannot enable fips and fips-updates at the same time. + # Hack: enable fips, copy the credentials and then after enabling + # other services, add the credentials back. + sudo pro enable fips --assume-yes + sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + # This will disable the fips service. + sudo pro enable fips-updates esm-apps esm-infra --assume-yes + # Add the fips credentials back. + sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage' + sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + + # Make apt credentials accessible to USER. + sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage + + # Run tests on Pro real archives. + go test ./internal/archive/ --real-pro-archive + + spread-tests: + name: Spread tests + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 + with: + repository: snapcore/spread + path: _spread + + - uses: actions/setup-go@v3 + with: + go-version: '>=1.17.0' + + - name: Build and run spread + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + (cd _spread/cmd/spread && go build) + _spread/cmd/spread/spread -v tests/pro-archives diff --git a/.github/workflows/spread.yml b/.github/workflows/spread.yml index ae95b7a1..3ac906f6 100644 --- a/.github/workflows/spread.yml +++ b/.github/workflows/spread.yml @@ -27,8 +27,6 @@ jobs: go-version: '>=1.17.0' - name: Build and run spread - env: - PRO_TOKEN: ${{ secrets.PRO_TOKEN }} run: | (cd _spread/cmd/spread && go build) - _spread/cmd/spread/spread -v focal jammy mantic noble + _spread/cmd/spread/spread -v diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7510cfc8..0f2e0425 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -56,33 +56,5 @@ jobs: go-version-file: 'go.mod' - name: Run real archive tests - env: - PRO_TOKEN: ${{ secrets.PRO_TOKEN }} run: | - set -ex - - detach() { - sudo pro detach --assume-yes || true - sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage - } - trap detach EXIT - - # Attach pro token and enable services - sudo pro attach ${PRO_TOKEN} --no-auto-enable - - # Cannot enable fips and fips-updates at the same time. - # Hack: enable fips, copy the credentials and then after enabling - # other services, add the credentials back. - sudo pro enable fips --assume-yes - sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds - # This will disable the fips service. - sudo pro enable fips-updates esm-apps esm-infra --assume-yes - # Add the fips credentials back. - sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage' - sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds - - # Make apt credentials accessible to USER. - sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage - - # Run tests on Pro and non-Pro real archives. - go test ./internal/archive/ -v --real-archive --real-pro-archive + go test ./internal/archive/ --real-archive From 23c369ff60f13a00fc2c2434a6fa4e6cc6d0389b Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Wed, 13 Nov 2024 14:51:41 +0100 Subject: [PATCH 11/13] make pro spread tests manual --- tests/pro-archives/task.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pro-archives/task.yaml b/tests/pro-archives/task.yaml index 262baab9..21963548 100644 --- a/tests/pro-archives/task.yaml +++ b/tests/pro-archives/task.yaml @@ -1,5 +1,7 @@ summary: Chisel can fetch packages from Ubuntu Pro archives +manual: true + variants: - noble From fb7dc7f1529ca00b7087f68a01e85c2e0c823e48 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Thu, 14 Nov 2024 18:10:00 +0100 Subject: [PATCH 12/13] fix: add esm-* prefix for consistency --- README.md | 12 +++++------ internal/archive/archive.go | 4 ++-- internal/archive/archive_test.go | 12 +++++------ internal/setup/setup_test.go | 20 +++++++++---------- .../pro-archives/chisel-releases/chisel.yaml | 2 +- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 32f65240..eea794f1 100644 --- a/README.md +++ b/README.md @@ -87,12 +87,12 @@ archives: The following Pro archives are currently supported: -| `pro` value | Archive URL | Corresponding Ubuntu Pro service | -| - | - |----------------------------------| -| fips | https://esm.ubuntu.com/fips/ubuntu | fips | -| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | fips-updates | -| apps | https://esm.ubuntu.com/apps/ubuntu | esm-apps | -| infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | +| `pro` value | Archive URL | +|--------------|--------------------------------------------| +| fips | https://esm.ubuntu.com/fips/ubuntu | +| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | +| esm-apps | https://esm.ubuntu.com/apps/ubuntu | +| esm-infra | https://esm.ubuntu.com/infra/ubuntu | If the system is using the [Pro client](https://ubuntu.com/pro/tutorial), and the services are enabled, the credentials will be automatically picked up from diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 0a2890fe..170e3bd5 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -153,8 +153,8 @@ const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" const ( ProFIPS = "fips" ProFIPSUpdates = "fips-updates" - ProApps = "apps" - ProInfra = "infra" + ProApps = "esm-apps" + ProInfra = "esm-infra" ) var proArchiveInfo = map[string]struct { diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index cf586bab..d8a243d2 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -699,7 +699,7 @@ var proArchiveTests = []realArchiveTest{{ version: "20.04", suites: []string{"focal-apps-security", "focal-apps-updates"}, components: []string{"main"}, - pro: "apps", + pro: "esm-apps", archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, archs: []string{"amd64"}, pkg: "hello", @@ -709,7 +709,7 @@ var proArchiveTests = []realArchiveTest{{ version: "20.04", suites: []string{"focal-infra-security", "focal-infra-updates"}, components: []string{"main"}, - pro: "infra", + pro: "esm-infra", archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, archs: []string{"amd64"}, pkg: "hello", @@ -729,7 +729,7 @@ var proArchiveTests = []realArchiveTest{{ version: "22.04", suites: []string{"jammy-apps-security", "jammy-apps-updates"}, components: []string{"main"}, - pro: "apps", + pro: "esm-apps", archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, archs: []string{"amd64"}, pkg: "hello", @@ -739,7 +739,7 @@ var proArchiveTests = []realArchiveTest{{ version: "22.04", suites: []string{"jammy-infra-security", "jammy-infra-updates"}, components: []string{"main"}, - pro: "infra", + pro: "esm-infra", archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, archs: []string{"amd64"}, pkg: "hello", @@ -749,7 +749,7 @@ var proArchiveTests = []realArchiveTest{{ version: "24.04", suites: []string{"noble-apps-security", "noble-apps-updates"}, components: []string{"main"}, - pro: "apps", + pro: "esm-apps", archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, archs: []string{"amd64"}, pkg: "hello", @@ -759,7 +759,7 @@ var proArchiveTests = []realArchiveTest{{ version: "24.04", suites: []string{"noble-infra-security", "noble-infra-updates"}, components: []string{"main"}, - pro: "infra", + pro: "esm-infra", archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, archs: []string{"amd64"}, pkg: "hello", diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 3866da12..32e81428 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1765,18 +1765,18 @@ var setupTests = []setupTest{{ pro: fips-updates priority: 21 public-keys: [test-key] - apps: + esm-apps: version: 20.04 components: [main] suites: [focal-apps-security] - pro: apps + pro: esm-apps priority: 16 public-keys: [test-key] - infra: + esm-infra: version: 20.04 components: [main] suites: [focal-infra-security] - pro: infra + pro: esm-infra priority: 15 public-keys: [test-key] ignored: @@ -1823,21 +1823,21 @@ var setupTests = []setupTest{{ Priority: 21, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, - "apps": { - Name: "apps", + "esm-apps": { + Name: "esm-apps", Version: "20.04", Suites: []string{"focal-apps-security"}, Components: []string{"main"}, - Pro: "apps", + Pro: "esm-apps", Priority: 16, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, - "infra": { - Name: "infra", + "esm-infra": { + Name: "esm-infra", Version: "20.04", Suites: []string{"focal-infra-security"}, Components: []string{"main"}, - Pro: "infra", + Pro: "esm-infra", Priority: 15, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, diff --git a/tests/pro-archives/chisel-releases/chisel.yaml b/tests/pro-archives/chisel-releases/chisel.yaml index 7de9d8c5..5beca3a9 100644 --- a/tests/pro-archives/chisel-releases/chisel.yaml +++ b/tests/pro-archives/chisel-releases/chisel.yaml @@ -3,7 +3,7 @@ format: v1 archives: ubuntu: version: 24.04 - pro: infra + pro: esm-infra components: [main] suites: [noble-infra-security, noble-infra-updates] public-keys: [ubuntu-esm-key-v2] From 5ed336169aac102fa84069ea60837f11951a8fbe Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Thu, 14 Nov 2024 18:14:21 +0100 Subject: [PATCH 13/13] change pro enable on the README to infra --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eea794f1..14ce897f 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ services are enabled, the credentials will be automatically picked up from `/etc/apt/auth.conf.d/`. However, the default permissions of the credentials file need to be changed so that Chisel can read it. Example: ```shell -sudo pro enable fips +sudo pro enable esm-infra sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage # or, alternatively,