diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 7a20c8d9..a8ebccf2 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -92,11 +92,19 @@ func (cmd *cmdCut) Execute(args []string) error { Components: archiveInfo.Components, CacheDir: cache.DefaultDir("chisel"), Priority: archiveInfo.Priority, + Pro: archiveInfo.Pro, }) if err != nil { - return err + if err != archive.ErrCredentialsNotFound { + return err + } + } else { + archives[archiveName] = openArchive } - archives[archiveName] = openArchive + } + + if len(archives) == 0 { + return fmt.Errorf("no valid archives (%d skipped)", len(release.Archives)) } return slicer.Run(&slicer.RunOptions{ diff --git a/internal/archive/archive.go b/internal/archive/archive.go index bd5f23c6..f8c99870 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "sort" "strings" "time" @@ -35,6 +36,7 @@ type Options struct { Components []string CacheDir string Priority int32 + Pro string } func Open(options *Options) (Archive, error) { @@ -73,6 +75,8 @@ type ubuntuArchive struct { options Options indexes []*ubuntuIndex cache *cache.Cache + baseURL string + auth string } type ubuntuIndex struct { @@ -83,7 +87,7 @@ type ubuntuIndex struct { component string release control.Section packages control.File - cache *cache.Cache + archive *ubuntuArchive } func (a *ubuntuArchive) Options() *Options { @@ -148,6 +152,52 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadCloser, error) { const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" +const ubuntuProURL = "https://esm.ubuntu.com/" + +// keep it sorted +var validPro = []string{ + "fips", + "fips-updates", +} + +func initProArchive(pro string, archive *ubuntuArchive) error { + if i := sort.SearchStrings(validPro, pro); !(i < len(validPro) && validPro[i] == pro) { + strvals := strings.Join(validPro, ", ") + return fmt.Errorf("invalid pro type, supported types: %s", strvals) + } + + baseURL := ubuntuProURL + pro + "/ubuntu/" + creds, err := findCredentials(baseURL) + if err != nil { + return err + } + + // Check that credentials are valid. + // It appears that only pool/ URLs are protected. + req, err := http.NewRequest("HEAD", baseURL+"pool/", nil) + if err != nil { + return fmt.Errorf("cannot create HTTP request: %w", err) + } + req.SetBasicAuth(creds.Username, creds.Password) + + resp, err := httpDo(req) + if err != nil { + return fmt.Errorf("cannot talk to the archive: %w", err) + } + resp.Body.Close() + switch resp.StatusCode { + case 200: // ok + case 401: + return fmt.Errorf("cannot authenticate to the archive") + default: + return fmt.Errorf("error from the archive: %v", resp.Status) + } + + archive.baseURL = baseURL + archive.auth = req.Header.Get("Authorization") + + return nil +} func openUbuntu(options *Options) (Archive, error) { if len(options.Components) == 0 { @@ -167,6 +217,18 @@ func openUbuntu(options *Options) (Archive, error) { }, } + if options.Pro != "" { + if err := initProArchive(options.Pro, archive); err != nil { + return nil, err + } + } else { + if options.Arch == "amd64" || options.Arch == "i386" { + archive.baseURL = ubuntuURL + } else { + archive.baseURL = ubuntuPortsURL + } + } + for _, suite := range options.Suites { var release control.Section for _, component := range options.Components { @@ -177,7 +239,7 @@ func openUbuntu(options *Options) (Archive, error) { suite: suite, component: component, release: release, - cache: archive.cache, + archive: archive, } if release == nil { err := index.fetchRelease() @@ -265,29 +327,27 @@ func (index *ubuntuIndex) checkComponents(components []string) error { } func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.ReadCloser, error) { - reader, err := index.cache.Open(digest) + reader, err := index.archive.cache.Open(digest) if err == nil { return reader, nil } else if err != cache.MissErr { return nil, err } - baseURL := ubuntuURL - if index.arch != "amd64" && index.arch != "i386" { - baseURL = ubuntuPortsURL - } - var url string if strings.HasPrefix(suffix, "pool/") { - url = baseURL + suffix + url = index.archive.baseURL + suffix } else { - url = baseURL + "dists/" + index.suite + "/" + suffix + url = index.archive.baseURL + "dists/" + index.suite + "/" + suffix } req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } + if index.archive.auth != "" { + req.Header.Set("Authorization", index.archive.auth) + } var resp *http.Response if flags&fetchBulk != 0 { resp, err = bulkDo(req) @@ -302,7 +362,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea switch resp.StatusCode { case 200: // ok - case 401, 404: + case 404: return nil, fmt.Errorf("cannot find archive data") default: return nil, fmt.Errorf("error from archive: %v", resp.Status) @@ -318,7 +378,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea body = reader } - writer := index.cache.Create(digest) + writer := index.archive.cache.Create(digest) defer writer.Close() _, err = io.Copy(writer, body) @@ -329,5 +389,5 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return nil, fmt.Errorf("cannot fetch from archive: %v", err) } - return index.cache.Open(writer.Digest()) + return index.archive.cache.Open(writer.Digest()) } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index b18fc60c..c941a352 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -365,6 +365,59 @@ func (s *httpSuite) TestPackageInfo(c *C) { c.Assert(info99, IsNil) } +func (s *httpSuite) TestFetchProPackage(c *C) { + var err error + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + s.base = "https://esm.ubuntu.com/fips/ubuntu/" + s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + + invalidOptions := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + Pro: "invalid", + } + + _, err = archive.Open(&invalidOptions) + c.Assert(err, ErrorMatches, "invalid pro type, supported types: fips, fips-updates") + + fipsOptions := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + Pro: "fips", + } + + _, err = archive.Open(&fipsOptions) + c.Assert(err, Equals, archive.ErrCredentialsNotFound) + + credsFile := filepath.Join(credsDir, "90ubuntu-advantage") + credsData := "machine https://esm.ubuntu.com/fips/ubuntu/ login user password pw\n" + err = os.WriteFile(credsFile, []byte(credsData), 0600) + c.Assert(err, IsNil) + + archive, err := archive.Open(&fipsOptions) + c.Assert(err, IsNil) + + pkg, err := archive.Fetch("mypkg1") + c.Assert(err, IsNil) + c.Assert(read(pkg), Equals, "mypkg1 1.1 data") + + pkg, err = archive.Fetch("mypkg4") + c.Assert(err, IsNil) + c.Assert(read(pkg), Equals, "mypkg4 1.4 data") +} + func read(r io.Reader) string { data, err := io.ReadAll(r) if err != nil { diff --git a/internal/setup/setup.go b/internal/setup/setup.go index e7c616e9..6fa7fadb 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -30,6 +30,7 @@ type Archive struct { Suites []string Components []string Priority int32 + Pro string } // Package holds a collection of slices that represent parts of themselves. @@ -323,6 +324,7 @@ type yamlArchive struct { Suites []string `yaml:"suites"` Components []string `yaml:"components"` Priority int32 `yaml:"priority"` + Pro string `yaml:"pro"` } type yamlPackage struct { @@ -430,6 +432,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Suites: details.Suites, Components: details.Components, Priority: details.Priority, + Pro: details.Pro, } } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 0a954db2..eb409d2d 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -790,6 +790,50 @@ var setupTests = []setupTest{{ `, }, relerror: "(?s).*\\bcannot unmarshal !!int `2147483648` into int32\\b.*", +}, { + summary: "Pro property", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + ubuntu: + version: 22.04 + components: [main, universe] + suites: [jammy, jammy-updates, jammy-security] + ubuntu-fips: + pro: fips + version: 22.04 + components: [main] + suites: [jammy] + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy", "jammy-updates", "jammy-security"}, + Components: []string{"main", "universe"}, + }, + "ubuntu-fips": { + Name: "ubuntu-fips", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main"}, + Pro: "fips", + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, }} const defaultChiselYaml = ` diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index a7122901..0049bac3 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -948,6 +948,7 @@ func (s *S) TestRun(c *C) { Suites: setupArchive.Suites, Components: setupArchive.Components, Priority: setupArchive.Priority, + Pro: setupArchive.Pro, Arch: test.arch, }, pkgs: archivePkgs,