From 6418c54042cf05808a521b94950262e6d2b7dd8c Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 15 Nov 2024 17:00:05 +0100 Subject: [PATCH 1/7] feat: support Pro archives (#167) In chisel.yaml, archive definitions can now use the "pro" value to specify Ubuntu Pro archives. The `archives..pro` value currently accepts the following values: "fips", "fips-updates", "esm-apps" and "esm-infra". Any other values are ignored. By default, Chisel will look for credentials in the `/etc/apt/auth.conf.d/` directory, unless the environment variable `CHISEL_AUTH_DIR` is set. In which case, it will look for configuration files in that directory. The configuration files may only have the ".conf" extensions or no extensions, the format is described in https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html. Co-authored-by: Rafid Bin Mostofa --- .github/workflows/pro_tests.yaml | 81 ++++ .github/workflows/spread.yml | 2 +- .github/workflows/tests.yaml | 15 + README.md | 42 ++ cmd/chisel/cmd_cut.go | 5 + cmd/chisel/main.go | 1 + internal/archive/archive.go | 109 ++++- internal/archive/archive_test.go | 417 ++++++++++++++---- internal/archive/export_test.go | 2 + internal/setup/setup.go | 10 + internal/setup/setup_test.go | 111 +++++ internal/slicer/slicer_test.go | 24 + internal/testutil/pgpkeys.go | 117 +++++ snap/snapcraft.yaml | 10 + spread.yaml | 1 + .../pro-archives/chisel-releases/chisel.yaml | 45 ++ .../chisel-releases/slices/hello.yaml | 13 + tests/pro-archives/task.yaml | 24 + 18 files changed, 938 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/pro_tests.yaml create mode 100644 tests/pro-archives/chisel-releases/chisel.yaml create mode 100644 tests/pro-archives/chisel-releases/slices/hello.yaml create mode 100644 tests/pro-archives/task.yaml diff --git a/.github/workflows/pro_tests.yaml b/.github/workflows/pro_tests.yaml new file mode 100644 index 00000000..af52a20d --- /dev/null +++ b/.github/workflows/pro_tests.yaml @@ -0,0 +1,81 @@ +name: Pro Tests + +on: + workflow_dispatch: + push: + paths-ignore: + - '**.md' + schedule: + - cron: "0 0 */2 * *" + workflow_run: + workflows: ["CLA check"] + types: + - completed + +jobs: + real-archive-tests: + name: Real Archive Tests + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + # Do not change to newer releases as "fips" may not be available there. + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run real archive tests + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + set -ex + + detach() { + sudo pro detach --assume-yes || true + sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage + } + trap detach EXIT + + # Attach pro token and enable services + sudo pro attach ${PRO_TOKEN} --no-auto-enable + + # Cannot enable fips and fips-updates at the same time. + # Hack: enable fips, copy the credentials and then after enabling + # other services, add the credentials back. + sudo pro enable fips --assume-yes + sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + # This will disable the fips service. + sudo pro enable fips-updates esm-apps esm-infra --assume-yes + # Add the fips credentials back. + sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage' + sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + + # Make apt credentials accessible to USER. + sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage + + # Run tests on Pro real archives. + go test ./internal/archive/ --real-pro-archive + + spread-tests: + name: Spread tests + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 + with: + repository: snapcore/spread + path: _spread + + - uses: actions/setup-go@v3 + with: + go-version: '>=1.17.0' + + - name: Build and run spread + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + (cd _spread/cmd/spread && go build) + _spread/cmd/spread/spread -v tests/pro-archives diff --git a/.github/workflows/spread.yml b/.github/workflows/spread.yml index 190a9061..3ac906f6 100644 --- a/.github/workflows/spread.yml +++ b/.github/workflows/spread.yml @@ -29,4 +29,4 @@ jobs: - name: Build and run spread run: | (cd _spread/cmd/spread && go build) - _spread/cmd/spread/spread -v focal jammy mantic noble + _spread/cmd/spread/spread -v diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7579f22d..0f2e0425 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -43,3 +43,18 @@ jobs: with: name: chisel-test-coverage.html path: ./*.html + + real-archive-tests: + # Do not change to newer releases as "fips" may not be available there. + runs-on: ubuntu-20.04 + name: Real Archive Tests + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run real archive tests + run: | + go test ./internal/archive/ --real-archive diff --git a/README.md b/README.md index 4e448125..14ce897f 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,48 @@ provided packages and install only the desired slices into the *myrootfs* folder, according to the slice definitions available in the ["ubuntu-22.04" chisel-releases branch](). +## Support for Pro archives +> [!IMPORTANT] +> To chisel a Pro package you need to have a Pro-enabled host. + +To fetch and install slices from Ubuntu Pro packages, the Pro archive has to be +defined with the `archives..pro` field in `chisel.yaml`: + + +```yaml +# chisel.yaml +format: v1 +archives: + : + pro: + ... +... +``` + +The following Pro archives are currently supported: + +| `pro` value | Archive URL | +|--------------|--------------------------------------------| +| fips | https://esm.ubuntu.com/fips/ubuntu | +| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | +| esm-apps | https://esm.ubuntu.com/apps/ubuntu | +| esm-infra | https://esm.ubuntu.com/infra/ubuntu | + +If the system is using the [Pro client](https://ubuntu.com/pro/tutorial), and the +services are enabled, the credentials will be automatically picked up from +`/etc/apt/auth.conf.d/`. However, the default permissions of the credentials file +need to be changed so that Chisel can read it. Example: +```shell +sudo pro enable esm-infra + +sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage +# or, alternatively, +sudo chmod u+r /etc/apt/auth.conf.d/90ubuntu-advantage +``` + +The location of the credentials can be configured using the environment variable +`CHISEL_AUTH_DIR`. + ## Reference ### Chisel releases diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index e5ed42bc..5c3088b3 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -70,10 +70,15 @@ func (cmd *cmdCut) Execute(args []string) error { Arch: cmd.Arch, Suites: archiveInfo.Suites, Components: archiveInfo.Components, + Pro: archiveInfo.Pro, CacheDir: cache.DefaultDir("chisel"), PubKeys: archiveInfo.PubKeys, }) if err != nil { + if err == archive.ErrCredentialsNotFound { + logf("Archive %q ignored: credentials not found", archiveName) + continue + } return err } archives[archiveName] = openArchive diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index e8f925a2..50792839 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -327,6 +327,7 @@ func run() error { deb.SetLogger(log.Default()) setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) + SetLogger(log.Default()) parser := Parser() xtra, err := parser.Parse() diff --git a/internal/archive/archive.go b/internal/archive/archive.go index c09fbdfb..170e3bd5 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -36,6 +36,7 @@ type Options struct { Arch string Suites []string Components []string + Pro string CacheDir string PubKeys []*packet.PublicKey } @@ -77,6 +78,8 @@ type ubuntuArchive struct { indexes []*ubuntuIndex cache *cache.Cache pubKeys []*packet.PublicKey + baseURL string + creds *credentials } type ubuntuIndex struct { @@ -147,6 +150,54 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" +const ( + ProFIPS = "fips" + ProFIPSUpdates = "fips-updates" + ProApps = "esm-apps" + ProInfra = "esm-infra" +) + +var proArchiveInfo = map[string]struct { + BaseURL, Label string +}{ + ProFIPS: { + BaseURL: "https://esm.ubuntu.com/fips/ubuntu/", + Label: "UbuntuFIPS", + }, + ProFIPSUpdates: { + BaseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", + Label: "UbuntuFIPSUpdates", + }, + ProApps: { + BaseURL: "https://esm.ubuntu.com/apps/ubuntu/", + Label: "UbuntuESMApps", + }, + ProInfra: { + BaseURL: "https://esm.ubuntu.com/infra/ubuntu/", + Label: "UbuntuESM", + }, +} + +func archiveURL(pro, arch string) (string, *credentials, error) { + if pro != "" { + archiveInfo, ok := proArchiveInfo[pro] + if !ok { + return "", nil, fmt.Errorf("invalid pro value: %q", pro) + } + url := archiveInfo.BaseURL + creds, err := findCredentials(url) + if err != nil { + return "", nil, err + } + return url, creds, nil + } + + if arch == "amd64" || arch == "i386" { + return ubuntuURL, nil, nil + } + return ubuntuPortsURL, nil, nil +} + func openUbuntu(options *Options) (Archive, error) { if len(options.Components) == 0 { return nil, fmt.Errorf("archive options missing components") @@ -158,12 +209,19 @@ func openUbuntu(options *Options) (Archive, error) { return nil, fmt.Errorf("archive options missing version") } + baseURL, creds, err := archiveURL(options.Pro, options.Arch) + if err != nil { + return nil, err + } + archive := &ubuntuArchive{ options: *options, cache: &cache.Cache{ Dir: options.CacheDir, }, pubKeys: options.PubKeys, + baseURL: baseURL, + creds: creds, } for _, suite := range options.Suites { @@ -184,6 +242,11 @@ func openUbuntu(options *Options) (Archive, error) { return nil, err } release = index.release + if !index.supportsArch(options.Arch) { + // Release does not support the specified architecture, do + // not add any of its indexes. + break + } err = index.checkComponents(options.Components) if err != nil { return nil, err @@ -201,7 +264,7 @@ func openUbuntu(options *Options) (Archive, error) { } func (index *ubuntuIndex) fetchRelease() error { - logf("Fetching %s %s %s suite details...", index.label, index.version, index.suite) + logf("Fetching %s %s %s suite details...", index.displayName(), index.version, index.suite) reader, err := index.fetch("InRelease", "", fetchDefault) if err != nil { return err @@ -235,12 +298,14 @@ func (index *ubuntuIndex) fetchRelease() error { if err != nil { return fmt.Errorf("cannot parse InRelease file: %v", err) } - section := ctrl.Section("Ubuntu") + // Parse the appropriate section for the type of archive. + label := "Ubuntu" + if index.archive.options.Pro != "" { + label = proArchiveInfo[index.archive.options.Pro].Label + } + section := ctrl.Section(label) if section == nil { - section = ctrl.Section("UbuntuProFIPS") - if section == nil { - return fmt.Errorf("corrupted archive InRelease file: no Ubuntu section") - } + return fmt.Errorf("corrupted archive InRelease file: no %s section", label) } logf("Release date: %s", section.Get("Date")) @@ -256,7 +321,7 @@ func (index *ubuntuIndex) fetchIndex() error { return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component) } - logf("Fetching index for %s %s %s %s component...", index.label, index.version, index.suite, index.component) + logf("Fetching index for %s %s %s %s component...", index.displayName(), index.version, index.suite, index.component) reader, err := index.fetch(packagesPath+".gz", digest, fetchBulk) if err != nil { return err @@ -270,6 +335,17 @@ func (index *ubuntuIndex) fetchIndex() error { return nil } +// supportsArch returns true if the Architectures field in the index release +// contains "arch". Per the Debian wiki [1], index release files should list the +// supported architectures in the "Architectures" field. +// The "ubuntuURL" archive only supports the amd64 and i386 architectures +// whereas the "ubuntuPortsURL" one supports the rest. But each of them +// (faultly) specifies all those architectures in their InRelease files. +// Reference: [1] https://wiki.debian.org/DebianRepository/Format#Architectures +func (index *ubuntuIndex) supportsArch(arch string) bool { + return strings.Contains(index.release.Get("Architectures"), arch) +} + func (index *ubuntuIndex) checkComponents(components []string) error { releaseComponents := strings.Fields(index.release.Get("Components")) for _, c1 := range components { @@ -295,10 +371,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return nil, err } - baseURL := ubuntuURL - if index.arch != "amd64" && index.arch != "i386" { - baseURL = ubuntuPortsURL - } + baseURL, creds := index.archive.baseURL, index.archive.creds var url string if strings.HasPrefix(suffix, "pool/") { @@ -311,6 +384,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } + if creds != nil && !creds.Empty() { + req.SetBasicAuth(creds.Username, creds.Password) + } var resp *http.Response if flags&fetchBulk != 0 { resp, err = bulkDo(req) @@ -325,7 +401,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea switch resp.StatusCode { case 200: // ok - case 401, 404: + case 401: + return nil, fmt.Errorf("cannot fetch from %q: unauthorized", index.label) + case 404: return nil, fmt.Errorf("cannot find archive data") default: return nil, fmt.Errorf("error from archive: %v", resp.Status) @@ -363,3 +441,10 @@ func sectionPackageInfo(section control.Section) *PackageInfo { SHA256: section.Get("SHA256"), } } + +func (index *ubuntuIndex) displayName() string { + if index.archive.options.Pro == "" { + return index.label + } + return index.label + " " + index.archive.options.Pro + " (pro)" +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 703148fb..d8a243d2 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -40,9 +40,12 @@ type httpSuite struct { var _ = Suite(&httpSuite{}) var ( - key1 = testutil.PGPKeys["key1"] - key2 = testutil.PGPKeys["key2"] - keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + key1 = testutil.PGPKeys["key1"] + key2 = testutil.PGPKeys["key2"] + keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + keyUbuntuFIPSv1 = testutil.PGPKeys["key-ubuntu-fips-v1"] + keyUbuntuApps = testutil.PGPKeys["key-ubuntu-apps"] + keyUbuntuESMv2 = testutil.PGPKeys["key-ubuntu-esm-v2"] ) func (s *httpSuite) SetUpTest(c *C) { @@ -178,6 +181,16 @@ var optionErrorTests = []optionErrorTest{{ Components: []string{"main", "other"}, }, error: `invalid package architecture: foo`, +}, { + options: archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "other"}, + Pro: "invalid", + }, + error: `invalid pro value: "invalid"`, }} func (s *httpSuite) TestOptionErrors(c *C) { @@ -328,44 +341,112 @@ func (s *httpSuite) TestArchiveLabels(c *C) { } } - s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + tests := []struct { + summary string + label string + err string + }{{ + summary: "Ubuntu label", + label: "Ubuntu", + }, { + summary: "Unknown label", + label: "Unknown", + err: "corrupted archive InRelease file: no Ubuntu section", + }} + + for _, test := range tests { + c.Logf("Summary: %s", test.summary) - options := archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + var adjust func(*testarchive.Release) + if test.label != "" { + adjust = setLabel(test.label) + } + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, adjust) + + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err := archive.Open(&options) + if test.err != "" { + c.Assert(err, ErrorMatches, test.err) + } else { + c.Assert(err, IsNil) + } } +} - _, err := archive.Open(&options) +func (s *httpSuite) TestProArchives(c *C) { + setLabel := func(label string) func(*testarchive.Release) { + return func(r *testarchive.Release) { + r.Label = label + } + } + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "" + for _, info := range archive.ProArchiveInfo { + contents += fmt.Sprintf("machine %s login foo password bar\n", info.BaseURL) + } + err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("Ubuntu")) + do := func(req *http.Request) (*http.Response, error) { + auth, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + c.Assert(auth, DeepEquals, []string{"Basic Zm9vOmJhcg=="}) + return s.Do(req) + } + restoreDo := archive.FakeDo(do) + defer restoreDo() - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err = archive.Open(&options) + c.Assert(err, IsNil) } - _, err = archive.Open(&options) - c.Assert(err, IsNil) + // Test non-pro archives. + do = func(req *http.Request) (*http.Response, error) { + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, false, Commentf("Non-pro archives should not have any authorization header")) + return s.Do(req) + } + restoreDo = archive.FakeDo(do) + defer restoreDo() - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("UbuntuProFIPS")) + s.base = "http://archive.ubuntu.com/ubuntu/" + s.prepareArchive("focal", "20.04", "amd64", []string{"main"}) - options = archive.Options{ + options := archive.Options{ Label: "ubuntu", - Version: "22.04", + Version: "20.04", Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, + Suites: []string{"focal"}, + Components: []string{"main"}, CacheDir: c.MkDir(), PubKeys: []*packet.PublicKey{s.pubKey}, } @@ -373,20 +454,41 @@ func (s *httpSuite) TestArchiveLabels(c *C) { _, err = archive.Open(&options) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("ThirdParty")) - - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + // Test Pro archives with bad credentials. + do = func(req *http.Request) (*http.Response, error) { + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + if strings.Contains(req.URL.String(), "/pool/") { + s.status = 401 + } else { + s.status = 200 + } + return s.Do(req) } + restoreDo = archive.FakeDo(do) + defer restoreDo() - _, err = archive.Open(&options) - c.Assert(err, ErrorMatches, `.*\bno Ubuntu section`) + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) + + _, _, err = testArchive.Fetch("mypkg1") + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) + } } type verifyArchiveReleaseTest struct { @@ -491,45 +593,177 @@ func read(r io.Reader) string { } // ---------------------------------------------------------------------------------------- -// Real archive tests, only enabled via --real-archive. +// Real archive tests, only enabled via: +// 1. --real-archive for non-Pro archives (e.g. standard jammy archive), +// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). +// +// To run the tests for Ubuntu Pro archives, the host machine must be Pro +// enabled and relevant Pro services must be enabled. The following commands +// might help: +// sudo pro attach --no-auto-enable +// sudo pro enable fips-updates esm-apps esm-infra --assume-yes var realArchiveFlag = flag.Bool("real-archive", false, "Perform tests against real archive") +var proArchiveFlag = flag.Bool("real-pro-archive", false, "Perform tests against real Ubuntu Pro archive") func (s *S) TestRealArchive(c *C) { if !*realArchiveFlag { c.Skip("--real-archive not provided") } - for _, release := range ubuntuReleases { - for _, arch := range elfToDebArch { - s.testOpenArchiveArch(c, release, arch) + s.runRealArchiveTests(c, realArchiveTests) +} + +func (s *S) TestRealProArchives(c *C) { + if !*proArchiveFlag { + c.Skip("--real-pro-archive not provided") + } + s.runRealArchiveTests(c, proArchiveTests) + s.testRealProArchiveBadCreds(c) +} + +func (s *S) runRealArchiveTests(c *C, tests []realArchiveTest) { + allArch := make([]string, 0, len(elfToDebArch)) + for _, arch := range elfToDebArch { + allArch = append(allArch, arch) + } + for _, test := range tests { + if len(test.archs) == 0 { + test.archs = allArch + } + for _, arch := range test.archs { + s.testOpenArchiveArch(c, test, arch) } } } -type ubuntuRelease struct { +type realArchiveTest struct { name string version string + suites []string + components []string + pro string archivePubKeys []*packet.PublicKey + archs []string + pkg string + path string } -var ubuntuReleases = []ubuntuRelease{{ - name: "focal", - version: "20.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, +var realArchiveTests = []realArchiveTest{{ + name: "focal", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/bin/hostname", }, { - name: "jammy", - version: "22.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, + name: "jammy", + version: "22.04", + suites: []string{"jammy"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/bin/hostname", }, { - name: "noble", - version: "24.04", - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, + name: "noble", + version: "24.04", + suites: []string{"noble"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + path: "/usr/bin/hostname", +}} + +var proArchiveTests = []realArchiveTest{{ + name: "focal-fips", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main"}, + pro: "fips", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "focal-fips-updates", + version: "20.04", + suites: []string{"focal-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "focal-esm-apps", + version: "20.04", + suites: []string{"focal-apps-security", "focal-apps-updates"}, + components: []string{"main"}, + pro: "esm-apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "focal-esm-infra", + version: "20.04", + suites: []string{"focal-infra-security", "focal-infra-updates"}, + components: []string{"main"}, + pro: "esm-infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "jammy-fips-updates", + version: "22.04", + suites: []string{"jammy-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + archs: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { + name: "jammy-esm-apps", + version: "22.04", + suites: []string{"jammy-apps-security", "jammy-apps-updates"}, + components: []string{"main"}, + pro: "esm-apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "jammy-esm-infra", + version: "22.04", + suites: []string{"jammy-infra-security", "jammy-infra-updates"}, + components: []string{"main"}, + pro: "esm-infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "noble-esm-apps", + version: "24.04", + suites: []string{"noble-apps-security", "noble-apps-updates"}, + components: []string{"main"}, + pro: "esm-apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", +}, { + name: "noble-esm-infra", + version: "24.04", + suites: []string{"noble-infra-security", "noble-infra-updates"}, + components: []string{"main"}, + pro: "esm-infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + archs: []string{"amd64"}, + pkg: "hello", + path: "/usr/bin/hello", }} var elfToDebArch = map[elf.Machine]string{ @@ -551,17 +785,18 @@ func (s *S) checkArchitecture(c *C, arch string, binaryPath string) { c.Assert(binaryArch, Equals, arch) } -func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { - c.Logf("Checking ubuntu archive %s %s...", release.name, arch) +func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { + c.Logf("Checking ubuntu archive %s %s...", test.name, arch) options := archive.Options{ Label: "ubuntu", - Version: release.version, + Version: test.version, Arch: arch, - Suites: []string{release.name}, - Components: []string{"main", "universe"}, + Suites: test.suites, + Components: test.components, CacheDir: c.MkDir(), - PubKeys: release.archivePubKeys, + Pro: test.pro, + PubKeys: test.archivePubKeys, } testArchive, err := archive.Open(&options) @@ -569,30 +804,56 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { extractDir := c.MkDir() - pkg, info, err := testArchive.Fetch("hostname") + pkg, info, err := testArchive.Fetch(test.pkg) c.Assert(err, IsNil) - c.Assert(info.Name, DeepEquals, "hostname") + c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) err = deb.Extract(pkg, &deb.ExtractOptions{ - Package: "hostname", + Package: test.pkg, TargetDir: extractDir, Extract: map[string][]deb.ExtractInfo{ - "/usr/share/doc/hostname/copyright": { + fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, }, - "/bin/hostname": { - {Path: "/hostname"}, + test.path: { + {Path: "/binary"}, }, }, }) c.Assert(err, IsNil) - data, err := os.ReadFile(filepath.Join(extractDir, "copyright")) + s.checkArchitecture(c, arch, filepath.Join(extractDir, "binary")) +} + +func (s *S) testRealProArchiveBadCreds(c *C) { + c.Logf("Cannot fetch Pro packages with bad credentials") + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "machine esm.ubuntu.com/fips/ubuntu/ login bearer password invalid" + err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) - copyrightTop := "This package was written by Peter Tobias " - c.Assert(strings.Contains(string(data), copyrightTop), Equals, true) + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: "fips", + PubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + } + + // The archive can be "opened" without any credentials since the dists/ path + // containing InRelease files, does not require any credentials. + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) - s.checkArchitecture(c, arch, filepath.Join(extractDir, "hostname")) + _, _, err = testArchive.Fetch("openssh-client") + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) } diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index cd75d5f4..3569a699 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -19,3 +19,5 @@ type Credentials = credentials var FindCredentials = findCredentials var FindCredentialsInDir = findCredentialsInDir + +var ProArchiveInfo = proArchiveInfo diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 00337bb6..eb8114f5 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -13,6 +13,7 @@ import ( "golang.org/x/crypto/openpgp/packet" "gopkg.in/yaml.v3" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" "github.com/canonical/chisel/internal/strdist" @@ -33,6 +34,7 @@ type Archive struct { Suites []string Components []string Priority int + Pro string PubKeys []*packet.PublicKey } @@ -399,6 +401,7 @@ type yamlArchive struct { Suites []string `yaml:"suites"` Components []string `yaml:"components"` Priority *int `yaml:"priority"` + Pro string `yaml:"pro"` Default bool `yaml:"default"` PubKeys []string `yaml:"public-keys"` } @@ -553,6 +556,12 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if len(details.Components) == 0 { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } + switch details.Pro { + case "", archive.ProApps, archive.ProFIPS, archive.ProFIPSUpdates, archive.ProInfra: + default: + logf("Archive %q ignored: invalid pro value: %q", archiveName, details.Pro) + continue + } if details.Default && defaultArchive != "" { if archiveName < defaultArchive { archiveName, defaultArchive = defaultArchive, archiveName @@ -591,6 +600,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Version: details.Version, Suites: details.Suites, Components: details.Components, + Pro: details.Pro, Priority: priority, PubKeys: archiveKeys, } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index d8d2f564..32e81428 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1739,6 +1739,117 @@ var setupTests = []setupTest{{ }, }, }, +}, { + summary: "Pro values in archives", + input: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + ubuntu: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + public-keys: [test-key] + fips: + version: 20.04 + components: [main] + suites: [focal] + pro: fips + priority: 20 + public-keys: [test-key] + fips-updates: + version: 20.04 + components: [main] + suites: [focal-updates] + pro: fips-updates + priority: 21 + public-keys: [test-key] + esm-apps: + version: 20.04 + components: [main] + suites: [focal-apps-security] + pro: esm-apps + priority: 16 + public-keys: [test-key] + esm-infra: + version: 20.04 + components: [main] + suites: [focal-infra-security] + pro: esm-infra + priority: 15 + public-keys: [test-key] + ignored: + version: 20.04 + components: [main] + suites: [foo] + pro: unknown-value + priority: 10 + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Priority: 10, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips": { + Name: "fips", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Pro: "fips", + Priority: 20, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips-updates": { + Name: "fips-updates", + Version: "20.04", + Suites: []string{"focal-updates"}, + Components: []string{"main"}, + Pro: "fips-updates", + Priority: 21, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "esm-apps": { + Name: "esm-apps", + Version: "20.04", + Suites: []string{"focal-apps-security"}, + Components: []string{"main"}, + Pro: "esm-apps", + Priority: 16, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "esm-infra": { + Name: "esm-infra", + Version: "20.04", + Suites: []string{"focal-infra-security"}, + Components: []string{"main"}, + Pro: "esm-infra", + Priority: 15, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, }, { summary: "Default is ignored", input: map[string]string{ diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 6cdb8f01..4974d623 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1445,6 +1445,29 @@ var slicerTests = []slicerTest{{ contents: `, }, +}, { + summary: "No valid archives defined due to invalid pro value", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + invalid: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + public-keys: [test-key] + pro: unknown-value + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, }} var defaultChiselYaml = ` @@ -1537,6 +1560,7 @@ func (s *S) TestRun(c *C) { Version: setupArchive.Version, Suites: setupArchive.Suites, Components: setupArchive.Components, + Pro: setupArchive.Pro, Arch: test.arch, }, Packages: pkgs, diff --git a/internal/testutil/pgpkeys.go b/internal/testutil/pgpkeys.go index e999a9a3..98eb6b05 100644 --- a/internal/testutil/pgpkeys.go +++ b/internal/testutil/pgpkeys.go @@ -21,6 +21,18 @@ var PGPKeys = map[string]*PGPKeyData{ ID: "871920D1991BC93C", PubKeyArmor: pubKeyUbuntu2018Armor, }, + "key-ubuntu-fips-v1": { + ID: "C1997C40EDE22758", + PubKeyArmor: pubKeyUbuntuFIPSv1Armor, + }, + "key-ubuntu-apps": { + ID: "AB01A101DB53907B", + PubKeyArmor: pubKeyUbuntuAppsArmor, + }, + "key-ubuntu-esm-v2": { + ID: "4067E40313CB4B13", + PubKeyArmor: pubKeyUbuntuESMv2Armor, + }, "key1": { ID: "854BAF1AA9D76600", PubKeyArmor: pubKey1Armor, @@ -87,6 +99,111 @@ uOgcXny1UlwtCUzlrSaP -----END PGP PUBLIC KEY BLOCK----- ` +// Ubuntu Federal Information Processing Standards Automatic Signing Key V1 . +// ID: C1997C40EDE22758. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuFIPSv1Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFzZxGABEADSWmX0+K//0cosKPyr5m1ewmwWKjRo/KBPTyR8icHhbBWfFd8T +DtYggvQHPU0YnKRcWits0et8JqSgZttNa28s7SaSUTBzfgzFJZgULAi/4i8u8TUj ++KH2zSoUX55NKC9aozba1cR66jM6O/BHXK5YoZzTpmiY1AHlIWAJ9s6cCClhnYMR +zwxSZVbefcYFbVPX/dQw/FMvJVeZ2aQ18NDgMQciu786aYklMFowxWNs/eLLTqum +cDHaw9UpKyhgfL/mkaIXuhYy6YRByYq1oOnJ5XffAOtovvCti1MvsPc0NDhPiGLf +9Fd/GtnqHxzVDqZmtUXX50mGu4LnJoHgWRjml3mapDPStzFr7Xgbb0NnyflmxnfN +kQcu2lFyXFfndWwg/RAOFdBPxBQhRK52uZiCfydKD7zCXz9YGm9xEK541EG0FrwA +6Vk1xaFol/jI8MQdP1o3JySX0Pqva3IHF7FHWHmxrIPaJLIHi0IrFG6Fgmk4sQ2w +XSc8kbxR+wYYKqIhBUZP0eb1jkFfvRVS6YvAy18xtw5pFD+VURdA0Uu5cotESfyz +oHsQ5R7wzg76oV/mYukHGC0x8peqxiPwbyhGFAhG8eUR66iYZgGbzmNI+OJz2EUi +UZJJXt4rnI1RVJLbhK9RjeobkOjf58Cm8RExlqJU16gy9saCMSiAqHx8swARAQAB +tFxVYnVudHUgRmVkZXJhbCBJbmZvcm1hdGlvbiBQcm9jZXNzaW5nIFN0YW5kYXJk +cyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgVjEgPGVzbUBjYW5vbmljYWwuY29tPokC +OAQTAQIAIgUCXNnEYAIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQwZl8 +QO3iJ1j4Vw//SawfmZi1GW+EUnuPqSz+zcmIKdx6AWZTe9/vSj6jgq4SYt//LAiD +NQz3dn2m0m5AaCucza2BCixUBrNhMh66m+lXfTqymUtTIpWpu4L1WLUbPjQ+s3Ad +xuF7S5wJtQrYmPvmZduZgg1wcb8eaqVltRJREpOP6sxcuqtvcfv4v4QYZ+iYd7eJ +8fxPOiyJEOTQPTdPZahYTaUOIloN5pT6uVg03u59Kh4aHCYxlRorvuRBabdctCfA +EBgomk4Us20Tv31dqlvMAiGKJqf1wdjhzlUmk4g/fOiRSNETKSC/VeUGH0fSbizl +Gs7Mg60jChPKpwzB6Rb5Nv2/Aw/FlSkfFhMdCdfKjl8IWOMPmElTVJFyVx1mmURi +3LgsloDFmJfebXefSFA7S8KLyBGlZJ/APaym64Ls12PUOjfh1Glie3E8KO66AGLo +ID1dQnzRizuHxW80ET03dSjzTXHLSi+iFycmNAxo6gB3GyOQ8tlIHjo1FfDfNYDf +qKic3Q0B9TvF6hqVRIcyePK4lN5YtRpVRdVj/jv8AqbzaIaVCP4k4nNrbaVx5zQf +BWq2E9IH+vLZfPyiP+hwxswfrlU3mrXBpPStIxq41yXFwQiDnqgkhEVAcrYPjBnS +T6s3+b+4HbAW6mbp4jEHUd/F1+iXz90T2WArrNIkMbmpChMuSyRN8Hc= +=DWhM +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Apps Automatic Signing Key . +// ID: AB01A101DB53907B. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuAppsArmor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF3WVA4BEAC7MDr8HClfKptSd4VeB12Vy+Ao/4NpY2ITdkRed4vfh/4eBWWn +3+in6So2ekweifACSxScB/M9zVObsI1cab7QPMkIiATNUfIyOEP7iNWLX4+AytM1 +LP3bZo8OpghnLZNstCGbiRUO4CDNmCI04DOPCu9EVEO4WWNuWIMRwCLShDSf7Cid +J2fn2TT/7vsmA4eI3YnAne+u8g4X2zMHQFkHANhylB0lPyThXo5jaxHImzm4wf/2 +LF8f1Y1nRQObS2jcvYc3fm9B7iOGpyNAw3h6hrPKH5T9tY/ZoMtFHqn66J1CBSHb +hDkEvA46X50su4yAHeSiEG/hMYG7SoHzmAsjEXnvkTIE41WhmxlidQnRs2uWy34U +7VmOpaidWn3R99fNHYOtSOB6bpIvls8snWSQ63jcFXnt05nVZsp/Ixzl0Oqitynx +DFwoxEwt3ZuCHwxbx2vZ+FiZXVFN7I0IyBDOEL6XS27FNaMCZ7Q/6z/ckdWto55E +264OWf9lnw31bXFXHWSusRXWzD6FK8dqWgjtrWwRxlvF4jm688lqpjac6fFES3UK +BhjyHXFGL/+HHZ9CNxlLYF5QnXq1mGR0Ykw975u8KoOFSLBqsx+1a21m6dfzujY7 +2Gq6Sju+9Yo1aOF+CNvTMYdRBoDL4sFj6VAmUsszMA5aAb+82pOCaDvGJQARAQAB +tDVVYnVudHUgQXBwcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGVzbUBjYW5vbmlj +YWwuY29tPokCOAQTAQIAIgUCXdZUDgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC +F4AACgkQqwGhAdtTkHuTOw/8Czv42TSpwHz+eNtl3ZFyxta9rR/qWC3h+vMu0R/l +5KU3aQQOygWOoUcr1QTPSSg3v/H+v/8vqVq2UuUxSIfpMxBj2kIX2vqskv6Roez7 +xR8lVDa0a47z/NYMfKpxrEJxOLh/c7I6aAsa597bTqDHtucHL/22BvfUJJqw6jq1 +7SswP5lqKPBFz7x+E2hgfJE7Vn7h0ICm29FkWnOeTKfj8VwTAeKXKUI9Hw6+aqr9 +29Y2NdLsYZ57mpivRLNM9sBZoF3avP1pUC2k0IwP3dwh4AxUMXjRRPh173iXBfR2 +yAf1lWET/5+8dSBrfFIZSo+FF/EEBmqIVtJpHkq8+YxUbCLbkoikRi2kwrgyXLEn +FqxSU2Ab0xurFHiHcJoCGVD38xjznO5cQl7H4K9+B/rFpTTowOHbOcFpKAzpYqB5 +8rnR1yRSsB33zac8xesUIfzYWRtLc5/VIb5mOkWlb62d8emILx2XuRFVjKq6mKki +oGckhDUOuEFrjW1cQq+PWBBxyJoXcy6wGSoPJ/ELeaf9zg8SF0jwuN6BPHVBeJ/E +W53zR5iV0N9fRT+M2JN5tc5HenO92xLgPAh+GPWLYmPdTmHu+kFozqsHx/NUw2iP +PBL6Q1VZytt2Uf6qLPUx7GpYMKf42Vldb0feFo/YA/lzOgPlY29pDLKXbse6o+Sr +kmk= +=AEEr +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Extended Security Maintenance Automatic Signing Key v2 . +// ID: 4067E40313CB4B13. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuESMv2Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY +680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs +8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t +YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 +t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI +OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG +JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 +rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 +K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT +Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE +6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB +tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj +IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 +kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C +NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB +m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S +nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 +1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F +EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC +puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG +tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C +DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS +89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 +RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi +JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc +=KWLs +-----END PGP PUBLIC KEY BLOCK----- +` + // Test-purpose RSA 2048 bits signing key-pairs without a passphrase. // ID: 854BAF1AA9D76600. User: "foo-bar ". const pubKey1Armor = ` diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ee1c284c..958a5970 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -41,6 +41,8 @@ parts: override-build: | go generate ./cmd craftctl default + stage: + - -bin/chrorder chisel-release-data: plugin: nil @@ -56,9 +58,17 @@ parts: craftctl set grade="$grade" after: [chisel] +plugs: + pro-credentials: + interface: system-files + read: + - /etc/apt/auth.conf.d + - /etc/apt/auth.conf.d/90ubuntu-advantage + apps: chisel: command: bin/chisel plugs: - network - home + - pro-credentials diff --git a/spread.yaml b/spread.yaml index a8de3d36..274bbd4b 100644 --- a/spread.yaml +++ b/spread.yaml @@ -4,6 +4,7 @@ path: /chisel environment: OS: ubuntu + PRO_TOKEN: $(HOST:echo $PRO_TOKEN) backends: # Cannot use LXD backend due to https://github.com/snapcore/spread/issues/154 diff --git a/tests/pro-archives/chisel-releases/chisel.yaml b/tests/pro-archives/chisel-releases/chisel.yaml new file mode 100644 index 00000000..5beca3a9 --- /dev/null +++ b/tests/pro-archives/chisel-releases/chisel.yaml @@ -0,0 +1,45 @@ +format: v1 + +archives: + ubuntu: + version: 24.04 + pro: esm-infra + components: [main] + suites: [noble-infra-security, noble-infra-updates] + public-keys: [ubuntu-esm-key-v2] + +public-keys: + # Ubuntu Extended Security Maintenance Automatic Signing Key v2 + # rsa4096/56f7650a24c9e9ecf87c4d8d4067e40313cb4b13 2019-04-17T02:33:33Z + ubuntu-esm-key-v2: + id: "4067E40313CB4B13" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY + 680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs + 8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t + YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 + t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI + OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG + JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 + rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 + K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT + Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE + 6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB + tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj + IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 + kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C + NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB + m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S + nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 + 1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F + EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC + puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG + tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C + DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS + 89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 + RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi + JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc + =KWLs + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/pro-archives/chisel-releases/slices/hello.yaml b/tests/pro-archives/chisel-releases/slices/hello.yaml new file mode 100644 index 00000000..7bd1a393 --- /dev/null +++ b/tests/pro-archives/chisel-releases/slices/hello.yaml @@ -0,0 +1,13 @@ +package: hello + +essential: + - hello_copyright + +slices: + bins: + contents: + /usr/bin/hello: + + copyright: + contents: + /usr/share/doc/hello/copyright: diff --git a/tests/pro-archives/task.yaml b/tests/pro-archives/task.yaml new file mode 100644 index 00000000..21963548 --- /dev/null +++ b/tests/pro-archives/task.yaml @@ -0,0 +1,24 @@ +summary: Chisel can fetch packages from Ubuntu Pro archives + +manual: true + +variants: + - noble + +environment: + ROOTFS: rootfs + +prepare: | + apt update && apt install -y ubuntu-pro-client + pro attach ${PRO_TOKEN} --no-auto-enable + pro enable esm-infra --assume-yes + mkdir ${ROOTFS} + +restore: | + pro detach --assume-yes + rm -r ${ROOTFS} + +execute: | + chisel cut --release ./chisel-releases/ --root ${ROOTFS} hello_bins + test -f ${ROOTFS}/usr/bin/hello + test -f ${ROOTFS}/usr/share/doc/hello/copyright From ccfe87a0e76aecf9135fab20ebacb0e532a91ee2 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 15 Nov 2024 17:05:19 +0100 Subject: [PATCH 2/7] fix: explicit parents override implicit (#166) This commits introduces a new flag for fsutil.Create called OverrideMode which updates the mode of existing entries. It is used to ensure that the explicit folder overrides the permissions of the implicit ones. --- internal/deb/extract.go | 11 +++-- internal/deb/extract_test.go | 27 +++++++++-- internal/fsutil/create.go | 14 +++++- internal/fsutil/create_test.go | 87 ++++++++++++++++++++++++++++++++++ internal/slicer/slicer_test.go | 44 +++++++++++------ 5 files changed, 160 insertions(+), 23 deletions(-) diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 07f219bd..d9e84875 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -247,11 +247,12 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { } // Create the entry itself. createOptions := &fsutil.CreateOptions{ - Path: filepath.Join(options.TargetDir, targetPath), - Mode: tarHeader.FileInfo().Mode(), - Data: pathReader, - Link: tarHeader.Linkname, - MakeParents: true, + Path: filepath.Join(options.TargetDir, targetPath), + Mode: tarHeader.FileInfo().Mode(), + Data: pathReader, + Link: tarHeader.Linkname, + MakeParents: true, + OverrideMode: true, } err := options.Create(extractInfos, createOptions) if err != nil { diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go index 22a1fd18..db73df86 100644 --- a/internal/deb/extract_test.go +++ b/internal/deb/extract_test.go @@ -2,6 +2,8 @@ package deb_test import ( "bytes" + "os" + "path" "path/filepath" "sort" "strings" @@ -17,7 +19,7 @@ type extractTest struct { summary string pkgdata []byte options deb.ExtractOptions - hackopt func(o *deb.ExtractOptions) + hackopt func(c *C, o *deb.ExtractOptions) result map[string]string // paths which the extractor did not create explicitly. notCreated []string @@ -98,7 +100,7 @@ var extractTests = []extractTest{{ "/dir/several/levels/deep/file": "file 0644 6bc26dff", "/other-dir/": "dir 0755", }, - hackopt: func(o *deb.ExtractOptions) { + hackopt: func(c *C, o *deb.ExtractOptions) { o.Create = nil }, }, { @@ -352,6 +354,25 @@ var extractTests = []extractTest{{ }, }, error: `cannot extract from package "test-package": path /dir/ requested twice with diverging mode: 0777 != 0000`, +}, { + summary: "Explicit extraction overrides existing file", + pkgdata: testutil.PackageData["test-package"], + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/dir/": []deb.ExtractInfo{{ + Path: "/dir/", + Mode: 0777, + }}, + }, + }, + hackopt: func(c *C, o *deb.ExtractOptions) { + err := os.Mkdir(path.Join(o.TargetDir, "/dir"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/dir/": "dir 0777", + }, + notCreated: []string{}, }} func (s *S) TestExtract(c *C) { @@ -374,7 +395,7 @@ func (s *S) TestExtract(c *C) { } if test.hackopt != nil { - test.hackopt(&options) + test.hackopt(c, &options) } err := deb.Extract(bytes.NewBuffer(test.pkgdata), &options) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index f76271f1..76561b77 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -19,6 +19,9 @@ type CreateOptions struct { // If MakeParents is true, missing parent directories of Path are // created with permissions 0755. MakeParents bool + // If OverrideMode is true and entry already exists, update the mode. Does + // not affect symlinks. + OverrideMode bool } type Entry struct { @@ -65,9 +68,18 @@ func Create(options *CreateOptions) (*Entry, error) { if err != nil { return nil, err } + mode := s.Mode() + if o.OverrideMode && mode != o.Mode && o.Mode&fs.ModeSymlink == 0 { + err := os.Chmod(o.Path, o.Mode) + if err != nil { + return nil, err + } + mode = o.Mode + } + entry := &Entry{ Path: o.Path, - Mode: s.Mode(), + Mode: mode, SHA256: hash, Size: rp.size, Link: o.Link, diff --git a/internal/fsutil/create_test.go b/internal/fsutil/create_test.go index d41bbf5a..be086ea5 100644 --- a/internal/fsutil/create_test.go +++ b/internal/fsutil/create_test.go @@ -93,6 +93,93 @@ var createTests = []createTest{{ // mode is not updated. "/foo": "file 0666 d67e2e94", }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Mode: fs.ModeDir | 0775, + OverrideMode: true, + }, + hackdir: func(c *C, dir string) { + c.Assert(os.Mkdir(filepath.Join(dir, "foo/"), fs.ModeDir|0765), IsNil) + }, + result: map[string]string{ + // mode is updated. + "/foo/": "dir 0775", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Mode: 0775, + Data: bytes.NewBufferString("whatever"), + OverrideMode: true, + }, + hackdir: func(c *C, dir string) { + err := os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + // mode is updated. + "/foo": "file 0775 85738f8f", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Link: "./bar", + Mode: 0666 | fs.ModeSymlink, + }, + hackdir: func(c *C, dir string) { + err := os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/foo": "symlink ./bar", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Link: "./bar", + Mode: 0776 | fs.ModeSymlink, + OverrideMode: true, + }, + hackdir: func(c *C, dir string) { + err := os.WriteFile(filepath.Join(dir, "bar"), []byte("data"), 0666) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/foo": "symlink ./bar", + // mode is not updated. + "/bar": "file 0666 3a6eb079", + }, +}, { + options: fsutil.CreateOptions{ + Path: "bar", + // Existing link with different target. + Link: "other", + Mode: 0666 | fs.ModeSymlink, + }, + hackdir: func(c *C, dir string) { + err := os.Symlink("foo", filepath.Join(dir, "bar")) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/bar": "symlink other", + }, +}, { + options: fsutil.CreateOptions{ + Path: "bar", + // Existing link with same target. + Link: "foo", + Mode: 0666 | fs.ModeSymlink, + }, + hackdir: func(c *C, dir string) { + err := os.Symlink("foo", filepath.Join(dir, "bar")) + c.Assert(err, IsNil) + }, + result: map[string]string{ + "/bar": "symlink foo", + }, }} func (s *S) TestCreate(c *C) { diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 4974d623..9e232fae 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -321,43 +321,59 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, explicit path has preference over implicit parent", slices: []setup.SliceKey{ - {"implicit-parent", "myslice"}, - {"explicit-dir", "myslice"}}, + {"a-implicit-parent", "myslice"}, + {"b-explicit-dir", "myslice"}, + {"c-implicit-parent", "myslice"}}, pkgs: []*testutil.TestPackage{{ - Name: "implicit-parent", + Name: "a-implicit-parent", Data: testutil.MustMakeDeb([]testutil.TarEntry{ testutil.Dir(0755, "./dir/"), - testutil.Reg(0644, "./dir/file", "random"), + testutil.Reg(0644, "./dir/file-1", "random"), }), }, { - Name: "explicit-dir", + Name: "b-explicit-dir", Data: testutil.MustMakeDeb([]testutil.TarEntry{ testutil.Dir(01777, "./dir/"), }), + }, { + Name: "c-implicit-parent", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0755, "./dir/"), + testutil.Reg(0644, "./dir/file-2", "random"), + }), }}, release: map[string]string{ - "slices/mydir/implicit-parent.yaml": ` - package: implicit-parent + "slices/mydir/a-implicit-parent.yaml": ` + package: a-implicit-parent slices: myslice: contents: - /dir/file: + /dir/file-1: `, - "slices/mydir/explicit-dir.yaml": ` - package: explicit-dir + "slices/mydir/b-explicit-dir.yaml": ` + package: b-explicit-dir slices: myslice: contents: /dir/: `, + "slices/mydir/c-implicit-parent.yaml": ` + package: c-implicit-parent + slices: + myslice: + contents: + /dir/file-2: + `, }, filesystem: map[string]string{ - "/dir/": "dir 01777", - "/dir/file": "file 0644 a441b15f", + "/dir/": "dir 01777", + "/dir/file-1": "file 0644 a441b15f", + "/dir/file-2": "file 0644 a441b15f", }, manifestPaths: map[string]string{ - "/dir/": "dir 01777 {explicit-dir_myslice}", - "/dir/file": "file 0644 a441b15f {implicit-parent_myslice}", + "/dir/": "dir 01777 {b-explicit-dir_myslice}", + "/dir/file-1": "file 0644 a441b15f {a-implicit-parent_myslice}", + "/dir/file-2": "file 0644 a441b15f {c-implicit-parent_myslice}", }, }, { summary: "Valid same file in two slices in different packages", From 363f657c456468d41744a4834361ec1ef06b544e Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 22 Nov 2024 12:08:05 +0100 Subject: [PATCH 3/7] ci: remove mantic from spread tests (#175) --- spread.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/spread.yaml b/spread.yaml index 274bbd4b..4644770e 100644 --- a/spread.yaml +++ b/spread.yaml @@ -56,5 +56,4 @@ suites: environment: RELEASE/jammy: 22.04 RELEASE/focal: 20.04 - RELEASE/mantic: 23.10 RELEASE/noble: 24.04 From 078334bc5d909f5c4e422fbb7740f9a74cc7ad8f Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 22 Nov 2024 12:15:44 +0100 Subject: [PATCH 4/7] fix: remove implicit copyright installation (#170) --- internal/slicer/slicer.go | 11 ----------- internal/slicer/slicer_test.go | 8 +------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2991e253..2d4c45e4 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -100,8 +100,6 @@ func Run(options *RunOptions) error { extract[slice.Package] = extractPackage } arch := pkgArchive[slice.Package].Options().Arch - copyrightPath := "/usr/share/doc/" + slice.Package + "/copyright" - hasCopyright := false for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue @@ -119,9 +117,6 @@ func Run(options *RunOptions) error { Path: targetPath, Context: slice, }) - if sourcePath == copyrightPath && targetPath == copyrightPath { - hasCopyright = true - } } else { // When the content is not extracted from the package (i.e. path is // not glob or copy), we add a ExtractInfo for the parent directory @@ -136,12 +131,6 @@ func Run(options *RunOptions) error { }) } } - if !hasCopyright { - extractPackage[copyrightPath] = append(extractPackage[copyrightPath], deb.ExtractInfo{ - Path: copyrightPath, - Optional: true, - }) - } } // Fetch all packages, using the selection order. diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 9e232fae..a49fdd97 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -246,7 +246,7 @@ var slicerTests = []slicerTest{{ "/dir/text-file-3": "file 0644 5b41362b {test-package_myslice}", }, }, { - summary: "Copyright is installed", + summary: "Copyright is not installed implicitly", slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -265,12 +265,6 @@ var slicerTests = []slicerTest{{ filesystem: map[string]string{ "/dir/": "dir 0755", "/dir/file": "file 0644 cc55e2ec", - // Hardcoded copyright entries. - "/usr/": "dir 0755", - "/usr/share/": "dir 0755", - "/usr/share/doc/": "dir 0755", - "/usr/share/doc/test-package/": "dir 0755", - "/usr/share/doc/test-package/copyright": "file 0644 c2fca2aa", }, manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", From b45d61fcec60ad542e36d779f737cefc8429a1aa Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 22 Nov 2024 12:18:52 +0100 Subject: [PATCH 5/7] refactor(setup): move yaml logic to yaml.go (#169) Co-authored-by: Rafid Bin Mostofa --- internal/setup/setup.go | 507 --------------------------------------- internal/setup/yaml.go | 516 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 516 insertions(+), 507 deletions(-) create mode 100644 internal/setup/yaml.go diff --git a/internal/setup/setup.go b/internal/setup/setup.go index eb8114f5..db9fe8cc 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -1,21 +1,14 @@ package setup import ( - "bytes" "fmt" "os" - "path" "path/filepath" "regexp" - "slices" "strings" "golang.org/x/crypto/openpgp/packet" - "gopkg.in/yaml.v3" - "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/deb" - "github.com/canonical/chisel/internal/pgputil" "github.com/canonical/chisel/internal/strdist" ) @@ -46,12 +39,6 @@ type Package struct { Slices map[string]*Slice } -func (p *Package) MarshalYAML() (interface{}, error) { - return packageToYAML(p) -} - -var _ yaml.Marshaler = (*Package)(nil) - // Slice holds the details about a package slice. type Slice struct { Package string @@ -385,434 +372,6 @@ func readSlices(release *Release, baseDir, dirName string) error { return nil } -type yamlRelease struct { - Format string `yaml:"format"` - Archives map[string]yamlArchive `yaml:"archives"` - PubKeys map[string]yamlPubKey `yaml:"public-keys"` -} - -const ( - MaxArchivePriority = 1000 - MinArchivePriority = -1000 -) - -type yamlArchive struct { - Version string `yaml:"version"` - Suites []string `yaml:"suites"` - Components []string `yaml:"components"` - Priority *int `yaml:"priority"` - Pro string `yaml:"pro"` - Default bool `yaml:"default"` - PubKeys []string `yaml:"public-keys"` -} - -type yamlPackage struct { - Name string `yaml:"package"` - Archive string `yaml:"archive,omitempty"` - Essential []string `yaml:"essential,omitempty"` - Slices map[string]yamlSlice `yaml:"slices,omitempty"` -} - -type yamlPath struct { - Dir bool `yaml:"make,omitempty"` - Mode yamlMode `yaml:"mode,omitempty"` - Copy string `yaml:"copy,omitempty"` - Text *string `yaml:"text,omitempty"` - Symlink string `yaml:"symlink,omitempty"` - Mutable bool `yaml:"mutable,omitempty"` - Until PathUntil `yaml:"until,omitempty"` - Arch yamlArch `yaml:"arch,omitempty"` - Generate GenerateKind `yaml:"generate,omitempty"` -} - -func (yp *yamlPath) MarshalYAML() (interface{}, error) { - type flowPath *yamlPath - node := &yaml.Node{} - err := node.Encode(flowPath(yp)) - if err != nil { - return nil, err - } - node.Style |= yaml.FlowStyle - return node, nil -} - -var _ yaml.Marshaler = (*yamlPath)(nil) - -// SameContent returns whether the path has the same content properties as some -// other path. In other words, the resulting file/dir entry is the same. The -// Mutable flag must also match, as that's a common agreement that the actual -// content is not well defined upfront. -func (yp *yamlPath) SameContent(other *yamlPath) bool { - return (yp.Dir == other.Dir && - yp.Mode == other.Mode && - yp.Copy == other.Copy && - yp.Text == other.Text && - yp.Symlink == other.Symlink && - yp.Mutable == other.Mutable) -} - -type yamlArch struct { - List []string -} - -func (ya *yamlArch) UnmarshalYAML(value *yaml.Node) error { - var s string - var l []string - if value.Decode(&s) == nil { - ya.List = []string{s} - } else if value.Decode(&l) == nil { - ya.List = l - } else { - return fmt.Errorf("cannot decode arch") - } - // Validate arch correctness later for a better error message. - return nil -} - -func (ya yamlArch) MarshalYAML() (interface{}, error) { - if len(ya.List) == 1 { - return ya.List[0], nil - } - return ya.List, nil -} - -var _ yaml.Marshaler = yamlArch{} - -type yamlMode uint - -func (ym yamlMode) MarshalYAML() (interface{}, error) { - // Workaround for marshalling integers in octal format. - // Ref: https://github.com/go-yaml/yaml/issues/420. - node := &yaml.Node{} - err := node.Encode(uint(ym)) - if err != nil { - return nil, err - } - node.Value = fmt.Sprintf("0%o", ym) - return node, nil -} - -var _ yaml.Marshaler = yamlMode(0) - -type yamlSlice struct { - Essential []string `yaml:"essential,omitempty"` - Contents map[string]*yamlPath `yaml:"contents,omitempty"` - Mutate string `yaml:"mutate,omitempty"` -} - -type yamlPubKey struct { - ID string `yaml:"id"` - Armor string `yaml:"armor"` -} - -func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { - release := &Release{ - Path: baseDir, - Packages: make(map[string]*Package), - Archives: make(map[string]*Archive), - } - - fileName := stripBase(baseDir, filePath) - - yamlVar := yamlRelease{} - dec := yaml.NewDecoder(bytes.NewBuffer(data)) - dec.KnownFields(false) - err := dec.Decode(&yamlVar) - if err != nil { - return nil, fmt.Errorf("%s: cannot parse release definition: %v", fileName, err) - } - if yamlVar.Format != "v1" { - return nil, fmt.Errorf("%s: unknown format %q", fileName, yamlVar.Format) - } - if len(yamlVar.Archives) == 0 { - return nil, fmt.Errorf("%s: no archives defined", fileName) - } - - // Decode the public keys and match against provided IDs. - pubKeys := make(map[string]*packet.PublicKey, len(yamlVar.PubKeys)) - for keyName, yamlPubKey := range yamlVar.PubKeys { - key, err := pgputil.DecodePubKey([]byte(yamlPubKey.Armor)) - if err != nil { - return nil, fmt.Errorf("%s: cannot decode public key %q: %w", fileName, keyName, err) - } - if yamlPubKey.ID != key.KeyIdString() { - return nil, fmt.Errorf("%s: public key %q armor has incorrect ID: expected %q, got %q", fileName, keyName, yamlPubKey.ID, key.KeyIdString()) - } - pubKeys[keyName] = key - } - - // For compatibility if there is a default archive set and priorities are - // not being used, we will revert back to the default archive behaviour. - hasPriority := false - var defaultArchive string - var archiveNoPriority string - for archiveName, details := range yamlVar.Archives { - if details.Version == "" { - return nil, fmt.Errorf("%s: archive %q missing version field", fileName, archiveName) - } - if len(details.Suites) == 0 { - return nil, fmt.Errorf("%s: archive %q missing suites field", fileName, archiveName) - } - if len(details.Components) == 0 { - return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) - } - switch details.Pro { - case "", archive.ProApps, archive.ProFIPS, archive.ProFIPSUpdates, archive.ProInfra: - default: - logf("Archive %q ignored: invalid pro value: %q", archiveName, details.Pro) - continue - } - if details.Default && defaultArchive != "" { - if archiveName < defaultArchive { - archiveName, defaultArchive = defaultArchive, archiveName - } - return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, defaultArchive, archiveName) - } - if details.Default { - defaultArchive = archiveName - } - if len(details.PubKeys) == 0 { - return nil, fmt.Errorf("%s: archive %q missing public-keys field", fileName, archiveName) - } - var archiveKeys []*packet.PublicKey - for _, keyName := range details.PubKeys { - key, ok := pubKeys[keyName] - if !ok { - return nil, fmt.Errorf("%s: archive %q refers to undefined public key %q", fileName, archiveName, keyName) - } - archiveKeys = append(archiveKeys, key) - } - priority := 0 - if details.Priority != nil { - hasPriority = true - priority = *details.Priority - if priority > MaxArchivePriority || priority < MinArchivePriority || priority == 0 { - return nil, fmt.Errorf("%s: archive %q has invalid priority value of %d", fileName, archiveName, priority) - } - } else { - if archiveNoPriority == "" || archiveName < archiveNoPriority { - // Make it deterministic. - archiveNoPriority = archiveName - } - } - release.Archives[archiveName] = &Archive{ - Name: archiveName, - Version: details.Version, - Suites: details.Suites, - Components: details.Components, - Pro: details.Pro, - Priority: priority, - PubKeys: archiveKeys, - } - } - if (hasPriority && archiveNoPriority != "") || - (!hasPriority && defaultArchive == "" && len(yamlVar.Archives) > 1) { - return nil, fmt.Errorf("%s: archive %q is missing the priority setting", fileName, archiveNoPriority) - } - if defaultArchive != "" && !hasPriority { - // For compatibility with the default archive behaviour we will set - // negative priorities to all but the default one, which means all - // others will be ignored unless pinned. - var archiveNames []string - for archiveName := range yamlVar.Archives { - archiveNames = append(archiveNames, archiveName) - } - // Make it deterministic. - slices.Sort(archiveNames) - for i, archiveName := range archiveNames { - release.Archives[archiveName].Priority = -i - 1 - } - release.Archives[defaultArchive].Priority = 1 - } - - return release, err -} - -func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, error) { - pkg := Package{ - Name: pkgName, - Path: pkgPath, - Slices: make(map[string]*Slice), - } - - yamlPkg := yamlPackage{} - dec := yaml.NewDecoder(bytes.NewBuffer(data)) - dec.KnownFields(false) - err := dec.Decode(&yamlPkg) - if err != nil { - return nil, fmt.Errorf("cannot parse package %q slice definitions: %v", pkgName, err) - } - if yamlPkg.Name != pkg.Name { - return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) - } - pkg.Archive = yamlPkg.Archive - - zeroPath := yamlPath{} - for sliceName, yamlSlice := range yamlPkg.Slices { - match := snameExp.FindStringSubmatch(sliceName) - if match == nil { - return nil, fmt.Errorf("invalid slice name %q in %s", sliceName, pkgPath) - } - - slice := &Slice{ - Package: pkgName, - Name: sliceName, - Scripts: SliceScripts{ - Mutate: yamlSlice.Mutate, - }, - } - for _, refName := range yamlPkg.Essential { - sliceKey, err := ParseSliceKey(refName) - if err != nil { - return nil, fmt.Errorf("package %q has invalid essential slice reference: %q", pkgName, refName) - } - if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { - // Do not add the slice to its own essentials list. - continue - } - if slices.Contains(slice.Essential, sliceKey) { - return nil, fmt.Errorf("package %s defined with redundant essential slice: %s", pkgName, refName) - } - slice.Essential = append(slice.Essential, sliceKey) - } - for _, refName := range yamlSlice.Essential { - sliceKey, err := ParseSliceKey(refName) - if err != nil { - return nil, fmt.Errorf("package %q has invalid essential slice reference: %q", pkgName, refName) - } - if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { - return nil, fmt.Errorf("cannot add slice to itself as essential %q in %s", refName, pkgPath) - } - if slices.Contains(slice.Essential, sliceKey) { - return nil, fmt.Errorf("slice %s defined with redundant essential slice: %s", slice, refName) - } - slice.Essential = append(slice.Essential, sliceKey) - } - - if len(yamlSlice.Contents) > 0 { - slice.Contents = make(map[string]PathInfo, len(yamlSlice.Contents)) - } - for contPath, yamlPath := range yamlSlice.Contents { - isDir := strings.HasSuffix(contPath, "/") - comparePath := contPath - if isDir { - comparePath = comparePath[:len(comparePath)-1] - } - if !path.IsAbs(contPath) || path.Clean(contPath) != comparePath { - return nil, fmt.Errorf("slice %s_%s has invalid content path: %s", pkgName, sliceName, contPath) - } - var kinds = make([]PathKind, 0, 3) - var info string - var mode uint - var mutable bool - var until PathUntil - var arch []string - var generate GenerateKind - if yamlPath != nil && yamlPath.Generate != "" { - zeroPathGenerate := zeroPath - zeroPathGenerate.Generate = yamlPath.Generate - if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Until != UntilNone { - return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options", - pkgName, sliceName, contPath) - } - if _, err := validateGeneratePath(contPath); err != nil { - return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err) - } - kinds = append(kinds, GeneratePath) - } else if strings.ContainsAny(contPath, "*?") { - if yamlPath != nil { - if !yamlPath.SameContent(&zeroPath) { - return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options", - pkgName, sliceName, contPath) - } - } - kinds = append(kinds, GlobPath) - } - if yamlPath != nil { - mode = uint(yamlPath.Mode) - mutable = yamlPath.Mutable - generate = yamlPath.Generate - if yamlPath.Dir { - if !strings.HasSuffix(contPath, "/") { - return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid", - pkgName, sliceName, contPath) - } - kinds = append(kinds, DirPath) - } - if yamlPath.Text != nil { - kinds = append(kinds, TextPath) - info = *yamlPath.Text - } - if len(yamlPath.Symlink) > 0 { - kinds = append(kinds, SymlinkPath) - info = yamlPath.Symlink - } - if len(yamlPath.Copy) > 0 { - kinds = append(kinds, CopyPath) - info = yamlPath.Copy - if info == contPath { - info = "" - } - } - until = yamlPath.Until - switch until { - case UntilNone, UntilMutate: - default: - return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkgName, sliceName, contPath, until) - } - arch = yamlPath.Arch.List - for _, s := range arch { - if deb.ValidateArch(s) != nil { - return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkgName, sliceName, contPath, s) - } - } - } - if len(kinds) == 0 { - kinds = append(kinds, CopyPath) - } - if len(kinds) != 1 { - list := make([]string, len(kinds)) - for i, s := range kinds { - list[i] = string(s) - } - return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkgName, sliceName, contPath, strings.Join(list, ", ")) - } - if mutable && kinds[0] != TextPath && (kinds[0] != CopyPath || isDir) { - return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath) - } - slice.Contents[contPath] = PathInfo{ - Kind: kinds[0], - Info: info, - Mode: mode, - Mutable: mutable, - Until: until, - Arch: arch, - Generate: generate, - } - } - - pkg.Slices[sliceName] = slice - } - - return &pkg, err -} - -// validateGeneratePath validates that the path follows the following format: -// - /slashed/path/to/dir/** -// -// Wildcard characters can only appear at the end as **, and the path before -// those wildcards must be a directory. -func validateGeneratePath(path string) (string, error) { - if !strings.HasSuffix(path, "/**") { - return "", fmt.Errorf("%s does not end with /**", path) - } - dirPath := strings.TrimSuffix(path, "**") - if strings.ContainsAny(dirPath, "*?") { - return "", fmt.Errorf("%s contains wildcard characters in addition to trailing **", path) - } - return dirPath, nil -} - func stripBase(baseDir, path string) string { // Paths must be clean for this to work correctly. return strings.TrimPrefix(path, baseDir+string(filepath.Separator)) @@ -861,69 +420,3 @@ func Select(release *Release, slices []SliceKey) (*Selection, error) { return selection, nil } - -// pathInfoToYAML converts a PathInfo object to a yamlPath object. -// The returned object takes pointers to the given PathInfo object. -func pathInfoToYAML(pi *PathInfo) (*yamlPath, error) { - path := &yamlPath{ - Mode: yamlMode(pi.Mode), - Mutable: pi.Mutable, - Until: pi.Until, - Arch: yamlArch{List: pi.Arch}, - Generate: pi.Generate, - } - switch pi.Kind { - case DirPath: - path.Dir = true - case CopyPath: - path.Copy = pi.Info - case TextPath: - path.Text = &pi.Info - case SymlinkPath: - path.Symlink = pi.Info - case GlobPath, GeneratePath: - // Nothing more needs to be done for these types. - default: - return nil, fmt.Errorf("internal error: unrecognised PathInfo type: %s", pi.Kind) - } - return path, nil -} - -// sliceToYAML converts a Slice object to a yamlSlice object. -func sliceToYAML(s *Slice) (*yamlSlice, error) { - slice := &yamlSlice{ - Essential: make([]string, 0, len(s.Essential)), - Contents: make(map[string]*yamlPath, len(s.Contents)), - Mutate: s.Scripts.Mutate, - } - for _, key := range s.Essential { - slice.Essential = append(slice.Essential, key.String()) - } - for path, info := range s.Contents { - // TODO remove the following line after upgrading to Go 1.22 or higher. - info := info - yamlPath, err := pathInfoToYAML(&info) - if err != nil { - return nil, err - } - slice.Contents[path] = yamlPath - } - return slice, nil -} - -// packageToYAML converts a Package object to a yamlPackage object. -func packageToYAML(p *Package) (*yamlPackage, error) { - pkg := &yamlPackage{ - Name: p.Name, - Archive: p.Archive, - Slices: make(map[string]yamlSlice, len(p.Slices)), - } - for name, slice := range p.Slices { - yamlSlice, err := sliceToYAML(slice) - if err != nil { - return nil, err - } - pkg.Slices[name] = *yamlSlice - } - return pkg, nil -} diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go new file mode 100644 index 00000000..3f46f713 --- /dev/null +++ b/internal/setup/yaml.go @@ -0,0 +1,516 @@ +package setup + +import ( + "bytes" + "fmt" + "path" + "slices" + "strings" + + "golang.org/x/crypto/openpgp/packet" + "gopkg.in/yaml.v3" + + "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/pgputil" +) + +func (p *Package) MarshalYAML() (interface{}, error) { + return packageToYAML(p) +} + +var _ yaml.Marshaler = (*Package)(nil) + +type yamlRelease struct { + Format string `yaml:"format"` + Archives map[string]yamlArchive `yaml:"archives"` + PubKeys map[string]yamlPubKey `yaml:"public-keys"` +} + +const ( + MaxArchivePriority = 1000 + MinArchivePriority = -1000 +) + +type yamlArchive struct { + Version string `yaml:"version"` + Suites []string `yaml:"suites"` + Components []string `yaml:"components"` + Priority *int `yaml:"priority"` + Pro string `yaml:"pro"` + Default bool `yaml:"default"` + PubKeys []string `yaml:"public-keys"` +} + +type yamlPackage struct { + Name string `yaml:"package"` + Archive string `yaml:"archive,omitempty"` + Essential []string `yaml:"essential,omitempty"` + Slices map[string]yamlSlice `yaml:"slices,omitempty"` +} + +type yamlPath struct { + Dir bool `yaml:"make,omitempty"` + Mode yamlMode `yaml:"mode,omitempty"` + Copy string `yaml:"copy,omitempty"` + Text *string `yaml:"text,omitempty"` + Symlink string `yaml:"symlink,omitempty"` + Mutable bool `yaml:"mutable,omitempty"` + Until PathUntil `yaml:"until,omitempty"` + Arch yamlArch `yaml:"arch,omitempty"` + Generate GenerateKind `yaml:"generate,omitempty"` +} + +func (yp *yamlPath) MarshalYAML() (interface{}, error) { + type flowPath *yamlPath + node := &yaml.Node{} + err := node.Encode(flowPath(yp)) + if err != nil { + return nil, err + } + node.Style |= yaml.FlowStyle + return node, nil +} + +var _ yaml.Marshaler = (*yamlPath)(nil) + +// SameContent returns whether the path has the same content properties as some +// other path. In other words, the resulting file/dir entry is the same. The +// Mutable flag must also match, as that's a common agreement that the actual +// content is not well defined upfront. +func (yp *yamlPath) SameContent(other *yamlPath) bool { + return (yp.Dir == other.Dir && + yp.Mode == other.Mode && + yp.Copy == other.Copy && + yp.Text == other.Text && + yp.Symlink == other.Symlink && + yp.Mutable == other.Mutable) +} + +type yamlArch struct { + List []string +} + +func (ya *yamlArch) UnmarshalYAML(value *yaml.Node) error { + var s string + var l []string + if value.Decode(&s) == nil { + ya.List = []string{s} + } else if value.Decode(&l) == nil { + ya.List = l + } else { + return fmt.Errorf("cannot decode arch") + } + // Validate arch correctness later for a better error message. + return nil +} + +func (ya yamlArch) MarshalYAML() (interface{}, error) { + if len(ya.List) == 1 { + return ya.List[0], nil + } + return ya.List, nil +} + +var _ yaml.Marshaler = yamlArch{} + +type yamlMode uint + +func (ym yamlMode) MarshalYAML() (interface{}, error) { + // Workaround for marshalling integers in octal format. + // Ref: https://github.com/go-yaml/yaml/issues/420. + node := &yaml.Node{} + err := node.Encode(uint(ym)) + if err != nil { + return nil, err + } + node.Value = fmt.Sprintf("0%o", ym) + return node, nil +} + +var _ yaml.Marshaler = yamlMode(0) + +type yamlSlice struct { + Essential []string `yaml:"essential,omitempty"` + Contents map[string]*yamlPath `yaml:"contents,omitempty"` + Mutate string `yaml:"mutate,omitempty"` +} + +type yamlPubKey struct { + ID string `yaml:"id"` + Armor string `yaml:"armor"` +} + +func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { + release := &Release{ + Path: baseDir, + Packages: make(map[string]*Package), + Archives: make(map[string]*Archive), + } + + fileName := stripBase(baseDir, filePath) + + yamlVar := yamlRelease{} + dec := yaml.NewDecoder(bytes.NewBuffer(data)) + dec.KnownFields(false) + err := dec.Decode(&yamlVar) + if err != nil { + return nil, fmt.Errorf("%s: cannot parse release definition: %v", fileName, err) + } + if yamlVar.Format != "v1" { + return nil, fmt.Errorf("%s: unknown format %q", fileName, yamlVar.Format) + } + if len(yamlVar.Archives) == 0 { + return nil, fmt.Errorf("%s: no archives defined", fileName) + } + + // Decode the public keys and match against provided IDs. + pubKeys := make(map[string]*packet.PublicKey, len(yamlVar.PubKeys)) + for keyName, yamlPubKey := range yamlVar.PubKeys { + key, err := pgputil.DecodePubKey([]byte(yamlPubKey.Armor)) + if err != nil { + return nil, fmt.Errorf("%s: cannot decode public key %q: %w", fileName, keyName, err) + } + if yamlPubKey.ID != key.KeyIdString() { + return nil, fmt.Errorf("%s: public key %q armor has incorrect ID: expected %q, got %q", fileName, keyName, yamlPubKey.ID, key.KeyIdString()) + } + pubKeys[keyName] = key + } + + // For compatibility if there is a default archive set and priorities are + // not being used, we will revert back to the default archive behaviour. + hasPriority := false + var defaultArchive string + var archiveNoPriority string + for archiveName, details := range yamlVar.Archives { + if details.Version == "" { + return nil, fmt.Errorf("%s: archive %q missing version field", fileName, archiveName) + } + if len(details.Suites) == 0 { + return nil, fmt.Errorf("%s: archive %q missing suites field", fileName, archiveName) + } + if len(details.Components) == 0 { + return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) + } + switch details.Pro { + case "", archive.ProApps, archive.ProFIPS, archive.ProFIPSUpdates, archive.ProInfra: + default: + logf("Archive %q ignored: invalid pro value: %q", archiveName, details.Pro) + continue + } + if details.Default && defaultArchive != "" { + if archiveName < defaultArchive { + archiveName, defaultArchive = defaultArchive, archiveName + } + return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, defaultArchive, archiveName) + } + if details.Default { + defaultArchive = archiveName + } + if len(details.PubKeys) == 0 { + return nil, fmt.Errorf("%s: archive %q missing public-keys field", fileName, archiveName) + } + var archiveKeys []*packet.PublicKey + for _, keyName := range details.PubKeys { + key, ok := pubKeys[keyName] + if !ok { + return nil, fmt.Errorf("%s: archive %q refers to undefined public key %q", fileName, archiveName, keyName) + } + archiveKeys = append(archiveKeys, key) + } + priority := 0 + if details.Priority != nil { + hasPriority = true + priority = *details.Priority + if priority > MaxArchivePriority || priority < MinArchivePriority || priority == 0 { + return nil, fmt.Errorf("%s: archive %q has invalid priority value of %d", fileName, archiveName, priority) + } + } else { + if archiveNoPriority == "" || archiveName < archiveNoPriority { + // Make it deterministic. + archiveNoPriority = archiveName + } + } + release.Archives[archiveName] = &Archive{ + Name: archiveName, + Version: details.Version, + Suites: details.Suites, + Components: details.Components, + Pro: details.Pro, + Priority: priority, + PubKeys: archiveKeys, + } + } + if (hasPriority && archiveNoPriority != "") || + (!hasPriority && defaultArchive == "" && len(yamlVar.Archives) > 1) { + return nil, fmt.Errorf("%s: archive %q is missing the priority setting", fileName, archiveNoPriority) + } + if defaultArchive != "" && !hasPriority { + // For compatibility with the default archive behaviour we will set + // negative priorities to all but the default one, which means all + // others will be ignored unless pinned. + var archiveNames []string + for archiveName := range yamlVar.Archives { + archiveNames = append(archiveNames, archiveName) + } + // Make it deterministic. + slices.Sort(archiveNames) + for i, archiveName := range archiveNames { + release.Archives[archiveName].Priority = -i - 1 + } + release.Archives[defaultArchive].Priority = 1 + } + + return release, err +} + +func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, error) { + pkg := Package{ + Name: pkgName, + Path: pkgPath, + Slices: make(map[string]*Slice), + } + + yamlPkg := yamlPackage{} + dec := yaml.NewDecoder(bytes.NewBuffer(data)) + dec.KnownFields(false) + err := dec.Decode(&yamlPkg) + if err != nil { + return nil, fmt.Errorf("cannot parse package %q slice definitions: %v", pkgName, err) + } + if yamlPkg.Name != pkg.Name { + return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) + } + pkg.Archive = yamlPkg.Archive + + zeroPath := yamlPath{} + for sliceName, yamlSlice := range yamlPkg.Slices { + match := snameExp.FindStringSubmatch(sliceName) + if match == nil { + return nil, fmt.Errorf("invalid slice name %q in %s", sliceName, pkgPath) + } + + slice := &Slice{ + Package: pkgName, + Name: sliceName, + Scripts: SliceScripts{ + Mutate: yamlSlice.Mutate, + }, + } + for _, refName := range yamlPkg.Essential { + sliceKey, err := ParseSliceKey(refName) + if err != nil { + return nil, fmt.Errorf("package %q has invalid essential slice reference: %q", pkgName, refName) + } + if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { + // Do not add the slice to its own essentials list. + continue + } + if slices.Contains(slice.Essential, sliceKey) { + return nil, fmt.Errorf("package %s defined with redundant essential slice: %s", pkgName, refName) + } + slice.Essential = append(slice.Essential, sliceKey) + } + for _, refName := range yamlSlice.Essential { + sliceKey, err := ParseSliceKey(refName) + if err != nil { + return nil, fmt.Errorf("package %q has invalid essential slice reference: %q", pkgName, refName) + } + if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { + return nil, fmt.Errorf("cannot add slice to itself as essential %q in %s", refName, pkgPath) + } + if slices.Contains(slice.Essential, sliceKey) { + return nil, fmt.Errorf("slice %s defined with redundant essential slice: %s", slice, refName) + } + slice.Essential = append(slice.Essential, sliceKey) + } + + if len(yamlSlice.Contents) > 0 { + slice.Contents = make(map[string]PathInfo, len(yamlSlice.Contents)) + } + for contPath, yamlPath := range yamlSlice.Contents { + isDir := strings.HasSuffix(contPath, "/") + comparePath := contPath + if isDir { + comparePath = comparePath[:len(comparePath)-1] + } + if !path.IsAbs(contPath) || path.Clean(contPath) != comparePath { + return nil, fmt.Errorf("slice %s_%s has invalid content path: %s", pkgName, sliceName, contPath) + } + var kinds = make([]PathKind, 0, 3) + var info string + var mode uint + var mutable bool + var until PathUntil + var arch []string + var generate GenerateKind + if yamlPath != nil && yamlPath.Generate != "" { + zeroPathGenerate := zeroPath + zeroPathGenerate.Generate = yamlPath.Generate + if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Until != UntilNone { + return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options", + pkgName, sliceName, contPath) + } + if _, err := validateGeneratePath(contPath); err != nil { + return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err) + } + kinds = append(kinds, GeneratePath) + } else if strings.ContainsAny(contPath, "*?") { + if yamlPath != nil { + if !yamlPath.SameContent(&zeroPath) { + return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options", + pkgName, sliceName, contPath) + } + } + kinds = append(kinds, GlobPath) + } + if yamlPath != nil { + mode = uint(yamlPath.Mode) + mutable = yamlPath.Mutable + generate = yamlPath.Generate + if yamlPath.Dir { + if !strings.HasSuffix(contPath, "/") { + return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid", + pkgName, sliceName, contPath) + } + kinds = append(kinds, DirPath) + } + if yamlPath.Text != nil { + kinds = append(kinds, TextPath) + info = *yamlPath.Text + } + if len(yamlPath.Symlink) > 0 { + kinds = append(kinds, SymlinkPath) + info = yamlPath.Symlink + } + if len(yamlPath.Copy) > 0 { + kinds = append(kinds, CopyPath) + info = yamlPath.Copy + if info == contPath { + info = "" + } + } + until = yamlPath.Until + switch until { + case UntilNone, UntilMutate: + default: + return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkgName, sliceName, contPath, until) + } + arch = yamlPath.Arch.List + for _, s := range arch { + if deb.ValidateArch(s) != nil { + return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkgName, sliceName, contPath, s) + } + } + } + if len(kinds) == 0 { + kinds = append(kinds, CopyPath) + } + if len(kinds) != 1 { + list := make([]string, len(kinds)) + for i, s := range kinds { + list[i] = string(s) + } + return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkgName, sliceName, contPath, strings.Join(list, ", ")) + } + if mutable && kinds[0] != TextPath && (kinds[0] != CopyPath || isDir) { + return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath) + } + slice.Contents[contPath] = PathInfo{ + Kind: kinds[0], + Info: info, + Mode: mode, + Mutable: mutable, + Until: until, + Arch: arch, + Generate: generate, + } + } + + pkg.Slices[sliceName] = slice + } + + return &pkg, err +} + +// validateGeneratePath validates that the path follows the following format: +// - /slashed/path/to/dir/** +// +// Wildcard characters can only appear at the end as **, and the path before +// those wildcards must be a directory. +func validateGeneratePath(path string) (string, error) { + if !strings.HasSuffix(path, "/**") { + return "", fmt.Errorf("%s does not end with /**", path) + } + dirPath := strings.TrimSuffix(path, "**") + if strings.ContainsAny(dirPath, "*?") { + return "", fmt.Errorf("%s contains wildcard characters in addition to trailing **", path) + } + return dirPath, nil +} + +// pathInfoToYAML converts a PathInfo object to a yamlPath object. +// The returned object takes pointers to the given PathInfo object. +func pathInfoToYAML(pi *PathInfo) (*yamlPath, error) { + path := &yamlPath{ + Mode: yamlMode(pi.Mode), + Mutable: pi.Mutable, + Until: pi.Until, + Arch: yamlArch{List: pi.Arch}, + Generate: pi.Generate, + } + switch pi.Kind { + case DirPath: + path.Dir = true + case CopyPath: + path.Copy = pi.Info + case TextPath: + path.Text = &pi.Info + case SymlinkPath: + path.Symlink = pi.Info + case GlobPath, GeneratePath: + // Nothing more needs to be done for these types. + default: + return nil, fmt.Errorf("internal error: unrecognised PathInfo type: %s", pi.Kind) + } + return path, nil +} + +// sliceToYAML converts a Slice object to a yamlSlice object. +func sliceToYAML(s *Slice) (*yamlSlice, error) { + slice := &yamlSlice{ + Essential: make([]string, 0, len(s.Essential)), + Contents: make(map[string]*yamlPath, len(s.Contents)), + Mutate: s.Scripts.Mutate, + } + for _, key := range s.Essential { + slice.Essential = append(slice.Essential, key.String()) + } + for path, info := range s.Contents { + // TODO remove the following line after upgrading to Go 1.22 or higher. + info := info + yamlPath, err := pathInfoToYAML(&info) + if err != nil { + return nil, err + } + slice.Contents[path] = yamlPath + } + return slice, nil +} + +// packageToYAML converts a Package object to a yamlPackage object. +func packageToYAML(p *Package) (*yamlPackage, error) { + pkg := &yamlPackage{ + Name: p.Name, + Archive: p.Archive, + Slices: make(map[string]yamlSlice, len(p.Slices)), + } + for name, slice := range p.Slices { + yamlSlice, err := sliceToYAML(slice) + if err != nil { + return nil, err + } + pkg.Slices[name] = *yamlSlice + } + return pkg, nil +} From a09dd150b86999e6fa61bb5db640b48bd76ac1af Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Fri, 22 Nov 2024 12:20:38 +0100 Subject: [PATCH 6/7] snap: update description (#165) Removes the usage from snap description. Resolves #164. --- snap/snapcraft.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 958a5970..4342338e 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -8,16 +8,6 @@ description: | filesystem which can be packaged into an OCI-compliant container image or similar. - Usage: chisel [...] - - Commands can be classified as follows: - - Basic: help, version - Action: cut - - For more information about a command, run 'chisel help '. - For a short summary of all commands, run 'chisel help --all'. - This snap can only install the slices in a location inside the user $HOME directory i.e. the --root option in "cut" command should have a location inside the user $HOME directory. From e2ee603c7396b33038e47352c0722b5b1202fbfe Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Mon, 25 Nov 2024 17:42:26 +0100 Subject: [PATCH 7/7] chore: add missing Generate equivalency (#173) Also consider Generate in yamlPath.SameContent. Implementation does not touch this right now. Co-authored-by: Alberto Carretero --- internal/setup/export_test.go | 3 +++ internal/setup/setup_test.go | 29 +++++++++++++++++++++++++++++ internal/setup/yaml.go | 3 ++- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 internal/setup/export_test.go diff --git a/internal/setup/export_test.go b/internal/setup/export_test.go new file mode 100644 index 00000000..35231e50 --- /dev/null +++ b/internal/setup/export_test.go @@ -0,0 +1,3 @@ +package setup + +type YAMLPath = yamlPath diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 32e81428..a5d16613 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -2240,3 +2240,32 @@ func (s *S) TestParseSliceKey(c *C) { c.Assert(key, DeepEquals, test.expected) } } + +// This is an awkward test because right now the fact Generate is considered +// by SameContent is irrelevant to the implementation, because the code path +// happens to not touch it. More important than this test, there's an entry +// in setupTests that verifies that two packages with slices having +// {generate: manifest} in the same path are considered equal. +var yamlPathGenerateTests = []struct { + summary string + path1, path2 *setup.YAMLPath + result bool +}{{ + summary: `Same "generate" value`, + path1: &setup.YAMLPath{Generate: setup.GenerateManifest}, + path2: &setup.YAMLPath{Generate: setup.GenerateManifest}, + result: true, +}, { + summary: `Different "generate" value`, + path1: &setup.YAMLPath{Generate: setup.GenerateManifest}, + path2: &setup.YAMLPath{Generate: setup.GenerateNone}, + result: false, +}} + +func (s *S) TestYAMLPathGenerate(c *C) { + for _, test := range yamlPathGenerateTests { + c.Logf("Summary: %s", test.summary) + result := test.path1.SameContent(test.path2) + c.Assert(result, Equals, test.result) + } +} diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 3f46f713..f2dbe127 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -84,7 +84,8 @@ func (yp *yamlPath) SameContent(other *yamlPath) bool { yp.Copy == other.Copy && yp.Text == other.Text && yp.Symlink == other.Symlink && - yp.Mutable == other.Mutable) + yp.Mutable == other.Mutable && + yp.Generate == other.Generate) } type yamlArch struct {