Merge branch 'main' into parallel-conflict-search

.github/workflows/pro_tests.yaml

@@ -0,0 +1,81 @@
+name: Pro Tests
+
+on:
+  workflow_dispatch:
+  push:
+    paths-ignore:
+      - '**.md'
+  schedule:
+    - cron: "0 0 */2 * *"
+  workflow_run:
+    workflows: ["CLA check"]
+    types:
+      - completed
+
+jobs:
+  real-archive-tests:
+    name: Real Archive Tests
+    if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
+    # Do not change to newer releases as "fips" may not be available there.
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run real archive tests
+        env:
+          PRO_TOKEN: ${{ secrets.PRO_TOKEN }}
+        run: |
+          set -ex
+
+          detach() {
+            sudo pro detach --assume-yes || true
+            sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage
+          }
+          trap detach EXIT
+
+          # Attach pro token and enable services
+          sudo pro attach ${PRO_TOKEN} --no-auto-enable
+
+          # Cannot enable fips and fips-updates at the same time.
+          # Hack: enable fips, copy the credentials and then after enabling
+          # other services, add the credentials back.
+          sudo pro enable fips --assume-yes
+          sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds
+          # This will disable the fips service.
+          sudo pro enable fips-updates esm-apps esm-infra --assume-yes
+          # Add the fips credentials back.
+          sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage'
+          sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds
+
+          # Make apt credentials accessible to USER.
+          sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage
+
+          # Run tests on Pro real archives.
+          go test ./internal/archive/ --real-pro-archive
+
+  spread-tests:
+    name: Spread tests
+    if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/checkout@v3
+        with:
+          repository: snapcore/spread
+          path: _spread
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.17.0'
+
+      - name: Build and run spread
+        env:
+          PRO_TOKEN: ${{ secrets.PRO_TOKEN }}
+        run: |
+          (cd _spread/cmd/spread && go build)
+          _spread/cmd/spread/spread -v tests/pro-archives

.github/workflows/spread.yml

@@ -29,4 +29,4 @@ jobs:
       - name: Build and run spread
         run: |
           (cd _spread/cmd/spread && go build)
-          _spread/cmd/spread/spread -v focal jammy mantic noble
+          _spread/cmd/spread/spread -v

.github/workflows/tests.yaml

@@ -43,3 +43,18 @@ jobs:
         with:
           name: chisel-test-coverage.html
           path: ./*.html
+
+  real-archive-tests:
+    # Do not change to newer releases as "fips" may not be available there.
+    runs-on: ubuntu-20.04
+    name: Real Archive Tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run real archive tests
+        run: |
+          go test ./internal/archive/ --real-archive

README.md

@@ -67,6 +67,48 @@ provided packages and install only the desired slices into the *myrootfs*
 folder, according to the slice definitions available in the
 ["ubuntu-22.04" chisel-releases branch](<https://github.com/canonical/chisel-releases/tree/ubuntu-22.04>).
 
+## Support for Pro archives
+> [!IMPORTANT]
+> To chisel a Pro package you need to have a Pro-enabled host.
+
+To fetch and install slices from Ubuntu Pro packages, the Pro archive has to be
+defined with the `archives.<name>.pro` field in `chisel.yaml`:
+
+
+```yaml
+# chisel.yaml
+format: v1
+archives:
+  <name>:
+    pro: <value>
+    ...
+...
+```
+
+The following Pro archives are currently supported:
+
+| `pro` value  | Archive URL                                |
+|--------------|--------------------------------------------|
+| fips         | https://esm.ubuntu.com/fips/ubuntu         |
+| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu |
+| esm-apps     | https://esm.ubuntu.com/apps/ubuntu         |
+| esm-infra    | https://esm.ubuntu.com/infra/ubuntu        |
+
+If the system is using the [Pro client](https://ubuntu.com/pro/tutorial), and the
+services are enabled, the credentials will be automatically picked up from
+`/etc/apt/auth.conf.d/`. However, the default permissions of the credentials file
+need to be changed so that Chisel can read it. Example:
+```shell
+sudo pro enable esm-infra
+
+sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage
+# or, alternatively,
+sudo chmod u+r /etc/apt/auth.conf.d/90ubuntu-advantage
+```
+
+The location of the credentials can be configured using the environment variable
+`CHISEL_AUTH_DIR`.
+
 ## Reference
 
 ### Chisel releases

cmd/chisel/cmd_cut.go

@@ -70,10 +70,15 @@ func (cmd *cmdCut) Execute(args []string) error {
 			Arch:       cmd.Arch,
 			Suites:     archiveInfo.Suites,
 			Components: archiveInfo.Components,
+			Pro:        archiveInfo.Pro,
 			CacheDir:   cache.DefaultDir("chisel"),
 			PubKeys:    archiveInfo.PubKeys,
 		})
 		if err != nil {
+			if err == archive.ErrCredentialsNotFound {
+				logf("Archive %q ignored: credentials not found", archiveName)
+				continue
+			}
 			return err
 		}
 		archives[archiveName] = openArchive

cmd/chisel/main.go

@@ -327,6 +327,7 @@ func run() error {
 	deb.SetLogger(log.Default())
 	setup.SetLogger(log.Default())
 	slicer.SetLogger(log.Default())
+	SetLogger(log.Default())
 
 	parser := Parser()
 	xtra, err := parser.Parse()

internal/archive/archive.go

@@ -36,6 +36,7 @@ type Options struct {
 	Arch       string
 	Suites     []string
 	Components []string
+	Pro        string
 	CacheDir   string
 	PubKeys    []*packet.PublicKey
 }
@@ -77,6 +78,8 @@ type ubuntuArchive struct {
 	indexes []*ubuntuIndex
 	cache   *cache.Cache
 	pubKeys []*packet.PublicKey
+	baseURL string
+	creds   *credentials
 }
 
 type ubuntuIndex struct {
@@ -147,6 +150,54 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) {
 const ubuntuURL = "http://archive.ubuntu.com/ubuntu/"
 const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/"
 
+const (
+	ProFIPS        = "fips"
+	ProFIPSUpdates = "fips-updates"
+	ProApps        = "esm-apps"
+	ProInfra       = "esm-infra"
+)
+
+var proArchiveInfo = map[string]struct {
+	BaseURL, Label string
+}{
+	ProFIPS: {
+		BaseURL: "https://esm.ubuntu.com/fips/ubuntu/",
+		Label:   "UbuntuFIPS",
+	},
+	ProFIPSUpdates: {
+		BaseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/",
+		Label:   "UbuntuFIPSUpdates",
+	},
+	ProApps: {
+		BaseURL: "https://esm.ubuntu.com/apps/ubuntu/",
+		Label:   "UbuntuESMApps",
+	},
+	ProInfra: {
+		BaseURL: "https://esm.ubuntu.com/infra/ubuntu/",
+		Label:   "UbuntuESM",
+	},
+}
+
+func archiveURL(pro, arch string) (string, *credentials, error) {
+	if pro != "" {
+		archiveInfo, ok := proArchiveInfo[pro]
+		if !ok {
+			return "", nil, fmt.Errorf("invalid pro value: %q", pro)
+		}
+		url := archiveInfo.BaseURL
+		creds, err := findCredentials(url)
+		if err != nil {
+			return "", nil, err
+		}
+		return url, creds, nil
+	}
+
+	if arch == "amd64" || arch == "i386" {
+		return ubuntuURL, nil, nil
+	}
+	return ubuntuPortsURL, nil, nil
+}
+
 func openUbuntu(options *Options) (Archive, error) {
 	if len(options.Components) == 0 {
 		return nil, fmt.Errorf("archive options missing components")
@@ -158,12 +209,19 @@ func openUbuntu(options *Options) (Archive, error) {
 		return nil, fmt.Errorf("archive options missing version")
 	}
 
+	baseURL, creds, err := archiveURL(options.Pro, options.Arch)
+	if err != nil {
+		return nil, err
+	}
+
 	archive := &ubuntuArchive{
 		options: *options,
 		cache: &cache.Cache{
 			Dir: options.CacheDir,
 		},
 		pubKeys: options.PubKeys,
+		baseURL: baseURL,
+		creds:   creds,
 	}
 
 	for _, suite := range options.Suites {
@@ -184,6 +242,11 @@ func openUbuntu(options *Options) (Archive, error) {
 					return nil, err
 				}
 				release = index.release
+				if !index.supportsArch(options.Arch) {
+					// Release does not support the specified architecture, do
+					// not add any of its indexes.
+					break
+				}
 				err = index.checkComponents(options.Components)
 				if err != nil {
 					return nil, err
@@ -201,7 +264,7 @@ func openUbuntu(options *Options) (Archive, error) {
 }
 
 func (index *ubuntuIndex) fetchRelease() error {
-	logf("Fetching %s %s %s suite details...", index.label, index.version, index.suite)
+	logf("Fetching %s %s %s suite details...", index.displayName(), index.version, index.suite)
 	reader, err := index.fetch("InRelease", "", fetchDefault)
 	if err != nil {
 		return err
@@ -235,12 +298,14 @@ func (index *ubuntuIndex) fetchRelease() error {
 	if err != nil {
 		return fmt.Errorf("cannot parse InRelease file: %v", err)
 	}
-	section := ctrl.Section("Ubuntu")
+	// Parse the appropriate section for the type of archive.
+	label := "Ubuntu"
+	if index.archive.options.Pro != "" {
+		label = proArchiveInfo[index.archive.options.Pro].Label
+	}
+	section := ctrl.Section(label)
 	if section == nil {
-		section = ctrl.Section("UbuntuProFIPS")
-		if section == nil {
-			return fmt.Errorf("corrupted archive InRelease file: no Ubuntu section")
-		}
+		return fmt.Errorf("corrupted archive InRelease file: no %s section", label)
 	}
 	logf("Release date: %s", section.Get("Date"))
 
@@ -256,7 +321,7 @@ func (index *ubuntuIndex) fetchIndex() error {
 		return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component)
 	}
 
-	logf("Fetching index for %s %s %s %s component...", index.label, index.version, index.suite, index.component)
+	logf("Fetching index for %s %s %s %s component...", index.displayName(), index.version, index.suite, index.component)
 	reader, err := index.fetch(packagesPath+".gz", digest, fetchBulk)
 	if err != nil {
 		return err
@@ -270,6 +335,17 @@ func (index *ubuntuIndex) fetchIndex() error {
 	return nil
 }
 
+// supportsArch returns true if the Architectures field in the index release
+// contains "arch". Per the Debian wiki [1], index release files should list the
+// supported architectures in the "Architectures" field.
+// The "ubuntuURL" archive only supports the amd64 and i386 architectures
+// whereas the "ubuntuPortsURL" one supports the rest. But each of them
+// (faultly) specifies all those architectures in their InRelease files.
+// Reference: [1] https://wiki.debian.org/DebianRepository/Format#Architectures
+func (index *ubuntuIndex) supportsArch(arch string) bool {
+	return strings.Contains(index.release.Get("Architectures"), arch)
+}
+
 func (index *ubuntuIndex) checkComponents(components []string) error {
 	releaseComponents := strings.Fields(index.release.Get("Components"))
 	for _, c1 := range components {
@@ -295,10 +371,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 		return nil, err
 	}
 
-	baseURL := ubuntuURL
-	if index.arch != "amd64" && index.arch != "i386" {
-		baseURL = ubuntuPortsURL
-	}
+	baseURL, creds := index.archive.baseURL, index.archive.creds
 
 	var url string
 	if strings.HasPrefix(suffix, "pool/") {
@@ -311,6 +384,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP request: %v", err)
 	}
+	if creds != nil && !creds.Empty() {
+		req.SetBasicAuth(creds.Username, creds.Password)
+	}
 	var resp *http.Response
 	if flags&fetchBulk != 0 {
 		resp, err = bulkDo(req)
@@ -325,7 +401,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea
 	switch resp.StatusCode {
 	case 200:
 		// ok
-	case 401, 404:
+	case 401:
+		return nil, fmt.Errorf("cannot fetch from %q: unauthorized", index.label)
+	case 404:
 		return nil, fmt.Errorf("cannot find archive data")
 	default:
 		return nil, fmt.Errorf("error from archive: %v", resp.Status)
@@ -363,3 +441,10 @@ func sectionPackageInfo(section control.Section) *PackageInfo {
 		SHA256:  section.Get("SHA256"),
 	}
 }
+
+func (index *ubuntuIndex) displayName() string {
+	if index.archive.options.Pro == "" {
+		return index.label
+	}
+	return index.label + " " + index.archive.options.Pro + " (pro)"
+}