From f5739440c244b3cc2e6f5126a909dbefb98dffc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Tue, 25 Apr 2023 16:52:13 +0200 Subject: [PATCH 01/23] Add apt credentials parser Add apt.auth.conf(5) machine credentials parser. See the parser code for documentation. This commit doesn't change the current behavior. It's only a preparation for future commits that will add support for different types of archives. --- internal/archive/credentials.go | 304 +++++++++++++++++++++++++++ internal/archive/credentials_test.go | 262 +++++++++++++++++++++++ internal/archive/export_test.go | 9 + 3 files changed, 575 insertions(+) create mode 100644 internal/archive/credentials.go create mode 100644 internal/archive/credentials_test.go diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go new file mode 100644 index 00000000..5bd7b54b --- /dev/null +++ b/internal/archive/credentials.go @@ -0,0 +1,304 @@ +package archive + +import ( + "bufio" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strings" +) + +// This file defines findCredentials() function for searching repository +// credentials in Apt configuration, see +// https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html. + +// credentials contains matched non-empty Username and Password. +// Username is left empty if the search is unsuccessful. +type credentials struct { + Username string + Password string +} + +// Empty checks whether c represents unsuccessful search. +func (c credentials) Empty() bool { + return c.Username == "" +} + +// credentialsQuery contains parsed input URL data used for search. +type credentialsQuery struct { + scheme string + host string + port string + path string + needScheme bool +} + +// queryFor parses repoUrl into credentialsQuery and fills provided credentials with +// username and password if they are specified in repoUrl. +func queryFor(repoUrl string, creds *credentials) (*credentialsQuery, error) { + u, err := url.Parse(repoUrl) + if err != nil { + return nil, err + } + host := u.Host + port := u.Port() + if port != "" { + // u.Hostname() would remove brackets from IPv6 address but we + // need it verbatim for string search in netrc file. This is + // also faster because both u.Port() and u.Hostname() parse + // u.Host into port and hostname. + host = u.Host[0 : len(u.Host)-len(port)-1] + } + + query := credentialsQuery{ + scheme: u.Scheme, + host: host, + port: port, + path: u.Path, + // If the input URL specifies unencrypted scheme, the scheme in + // machine declarations in netrc file is not optional and must + // also match. + needScheme: u.Scheme != "https" && u.Scheme != "tor+https", + } + + if creds != nil { + creds.Username = u.User.Username() + creds.Password, _ = u.User.Password() + } + + return &query, nil +} + +var testCredentialsDir string + +// findCredentials searches credentials for repoUrl in configuration files in +// directory specified by CHISEL_AUTH_DIR environment variable if it's +// non-empty or /etc/apt/auth.conf.d. +func findCredentials(repoUrl string) (credentials, error) { + credsDir := "/etc/apt/auth.conf.d" + if testCredentialsDir == "" { + if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { + credsDir = v + } + } else { + credsDir = testCredentialsDir + } + return findCredsInDir(repoUrl, credsDir) +} + +// findCredsInDir searches for credentials for repoUrl in configuration +// files in credsDir directory. If the directory does not exist, empty +// credentials structure with nil err is returned. +// Only files that do not begin with dot and have either no or ".conf" +// extension are searched. The files are searched in ascending lexicographic +// order. The first file that contains machine declaration matching repoUrl +// ends the search. If no file contain matching machine declaration, empty +// credentials structure with nil err is returned. +func findCredsInDir(repoUrl string, credsDir string) (creds credentials, err error) { + contents, err := os.ReadDir(credsDir) + if err != nil { + if os.IsNotExist(err) { + err = nil + } else { + err = fmt.Errorf("cannot open credentials directory: %w", err) + } + return + } + + query, err := queryFor(repoUrl, &creds) + if err != nil || !creds.Empty() { + return + } + + errs := make([]error, 0) + + confFiles := make([]string, 0, len(contents)) + for _, entry := range contents { + name := entry.Name() + if strings.HasPrefix(name, ".") { + continue + } + info, err := entry.Info() + if err != nil { + errs = append(errs, fmt.Errorf("cannot stat credentials file: %w", err)) + continue + } + if !info.Mode().IsRegular() { + continue + } + ext := filepath.Ext(name) + if ext == "" || ext == ".conf" { + confFiles = append(confFiles, name) + } + } + if len(confFiles) == 0 { + err = errors.Join(errs...) + return + } + sort.Strings(confFiles) + + for _, file := range confFiles { + f, err := os.Open(filepath.Join(credsDir, file)) + if err != nil { + errs = append(errs, fmt.Errorf("cannot read credentials file: %w", err)) + continue + } + + if err = findCredsInFile(query, f, &creds); err != nil { + errs = append(errs, fmt.Errorf("cannot parse credentials file: %w", err)) + } else if !creds.Empty() { + break + } + } + + err = errors.Join(errs...) + return +} + +type netrcParser struct { + query *credentialsQuery + scanner *bufio.Scanner + creds *credentials +} + +// findCredsInFile searches for credentials in netrc file matching query +// and fills creds with matched credentials if there's a match. The first match +// ends the search. +// +// The format of the netrc file is described in [1]. The parser is adapted from +// the Apt parser (see [2]). When the parser is looking for a matching machine +// declaration it disregards the current context and only considers the input +// token. For example when given the following netrc file +// +// machine http://acme.com/foo login u1 password machine +// machine http://acme.com/bar login u2 password p2 +// +// and http://acme.com/bar input URL, the second line won't match, because the +// second "machine" will be treated as start of machine declaration. This also +// means unknown tokens are ignored, so comments are not treated specially. +// +// When a matching machine declaration is found the search stops on next +// machine token or on end of file. This means that arbitrary number of login +// and password declarations (or in fact, any tokens) can follow a machine +// declaration. The last username and password declaration overrides the +// previous ones. For example when given the following netrc file +// +// machine http://acme.com login a foo login b password c bar login d password e +// +// and the input URL is http://acme.com, the matched username and password will +// be "d" and "e" respectively. Tokens foo and bar will be ignored. +// +// This parser diverges from the Apt parser in the following ways: +// 1. The port specification in machine declaration is optional whether or +// not a path is specified. While the Apt documentation[1] implies the +// same behavior, the code adheres to it only when the machine declaration +// does not specify a path, see line 96 in [2]. +// 2. When the input URL has unencrypted scheme and the machine declaration +// does not specify a scheme, it is skipped silently. The Apt parser warns +// the user about it, see line 113 in [2]. +// +// References: +// +// [1] https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html +// [2] https://salsa.debian.org/apt-team/apt/-/blob/d9039b24/apt-pkg/contrib/netrc.cc +// [3] https://salsa.debian.org/apt-team/apt/-/blob/4e04cbaf/methods/aptmethod.h#L560 +// [4] https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html +// [5] https://daniel.haxx.se/blog/2022/05/31/netrc-pains/ +func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credentials) error { + s := bufio.NewScanner(netrc) + s.Split(bufio.ScanWords) + p := netrcParser{ + query: query, + scanner: s, + creds: creds, + } + var err error + for state := netrcInvalid; state != nil; { + state, err = state(&p) + } + if err := p.scanner.Err(); err != nil { + return err + } + return err +} + +type netrcState func(*netrcParser) (netrcState, error) + +func netrcInvalid(p *netrcParser) (netrcState, error) { + for p.scanner.Scan() { + if p.scanner.Text() == "machine" { + return netrcMachine, nil + } + } + return nil, nil +} + +func netrcMachine(p *netrcParser) (netrcState, error) { + if !p.scanner.Scan() { + return nil, errors.New("syntax error: reached end of file while expecting machine text") + } + token := p.scanner.Text() + if i := strings.Index(token, "://"); i != -1 { + if token[0:i] != p.query.scheme { + return netrcInvalid, nil + } + token = token[i+3:] + } else if p.query.needScheme { + return netrcInvalid, nil + } + if !strings.HasPrefix(token, p.query.host) { + return netrcInvalid, nil + } + token = token[len(p.query.host):] + if len(token) > 0 { + if token[0] == ':' { + if p.query.port == "" { + return netrcInvalid, nil + } + token = token[1:] + if !strings.HasPrefix(token, p.query.port) { + return netrcInvalid, nil + } + token = token[len(p.query.port):] + } + if !strings.HasPrefix(p.query.path, token) { + return netrcInvalid, nil + } + } + return netrcGoodMachine, nil +} + +func netrcGoodMachine(p *netrcParser) (netrcState, error) { +loop: + for p.scanner.Scan() { + switch p.scanner.Text() { + case "login": + return netrcUsername, nil + case "password": + return netrcPassword, nil + case "machine": + break loop + } + } + return nil, nil +} + +func netrcUsername(p *netrcParser) (netrcState, error) { + if !p.scanner.Scan() { + return nil, errors.New("syntax error: reached end of file while expecting username text") + } + p.creds.Username = p.scanner.Text() + return netrcGoodMachine, nil +} + +func netrcPassword(p *netrcParser) (netrcState, error) { + if !p.scanner.Scan() { + return nil, errors.New("syntax error: reached end of file while expecting password text") + } + p.creds.Password = p.scanner.Text() + return netrcGoodMachine, nil +} diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go new file mode 100644 index 00000000..af0e4c64 --- /dev/null +++ b/internal/archive/credentials_test.go @@ -0,0 +1,262 @@ +package archive_test + +import ( + "io/ioutil" + "os" + "path/filepath" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/archive" +) + +type matchTest struct { + url string + err string + username string + password string +} + +type credentialsTest struct { + summary string + credsFiles map[string]string + matchTests []matchTest +} + +var credentialsTests = []credentialsTest{{ + summary: "Parsing", + credsFiles: map[string]string{ + "50test-logins": ` +machine example.netter login bar password foo +machine example.net login foo password bar + +machine example.org:90 login apt password apt +machine example.org:8080 +login +example password foobar + +machine example.org +login anonymous +password pass + +machine example.com/foo login user1 unknown token password pass1 +machine example.com/bar password pass2 login user2 + unknown token +machine example.com/user login user +machine example.netter login unused password firstentry +machine socks5h://example.last/debian login debian password rules +`, + }, + matchTests: []matchTest{ + {"https://example.net/foo", "", "foo", "bar"}, + {"https://user:pass@example.net/foo", "", "user", "pass"}, + {"https://example.org:90/foo", "", "apt", "apt"}, + {"https://example.org:8080/foo", "", "example", "foobar"}, + {"https://example.net:42/foo", "", "foo", "bar"}, + {"https://example.org/foo", "", "anonymous", "pass"}, + {"https://example.com/apt", "", "", ""}, + {"https://example.com/foo", "", "user1", "pass1"}, + {"https://example.com/fooo", "", "user1", "pass1"}, + {"https://example.com/fo", "", "", ""}, + {"https://example.com/bar", "", "user2", "pass2"}, + {"https://example.com/user", "", "user", ""}, + {"socks5h://example.last/debian", "", "debian", "rules"}, + {"socks5h://example.debian/", "", "", ""}, + {"socks5h://user:pass@example.debian/", "", "user", "pass"}, + }, +}, { + summary: "Bad file: No machine", + credsFiles: map[string]string{ + "50test-logins.conf": ` +foo example.org login foo1 password bar +machin example.org login foo2 password bar +machine2 example.org login foo3 password bar +`, + }, + matchTests: []matchTest{ + {"https://example.org/foo", "", "", ""}, + }, +}, { + summary: "Bad file: Ends machine", + credsFiles: map[string]string{ + "50test-logins.conf": ` +machine example.org login foo1 password bar +machine`, + }, + matchTests: []matchTest{ + {"https://example.org/foo", "", "foo1", "bar"}, + {"https://example.net/foo", ".*\\breached end of file while expecting machine text\\b.*", "", ""}, + {"https://foo:bar@example.net/foo", "", "foo", "bar"}, + }, +}, { + summary: "Bad file: Ends login", + credsFiles: map[string]string{ + "50test-logins.conf": ` +machine example.org login foo1 password bar +machine example.net login +`, + }, + matchTests: []matchTest{ + {"https://example.org/foo", "", "foo1", "bar"}, + {"https://example.net/foo", ".*\\breached end of file while expecting username text\\b.*", "", ""}, + {"https://foo:bar@example.net/foo", "", "foo", "bar"}, + }, +}, { + summary: "Matches only HTTPS", + credsFiles: map[string]string{ + "50test-logins.conf": ` +machine https.example login foo1 password bar +machine http://http.example login foo1 password bar +`, + }, + matchTests: []matchTest{ + {"https://https.example/foo", "", "foo1", "bar"}, + {"http://https.example/foo", "", "", ""}, + {"http://http.example/foo", "", "foo1", "bar"}, + {"https://http.example/foo", "", "", ""}, + }, +}, { + summary: "Password is machine", + credsFiles: map[string]string{ + "50test-logins.conf": ` +machine http://site1.com login u1 password machine +machine http://site2.com login u2 password p2 +`, + }, + matchTests: []matchTest{ + {"http://site1.com/foo", "", "u1", "machine"}, + {"http://site2.com/bar", "", "", ""}, + }, +}, { + summary: "Multiple login and password tokens", + credsFiles: map[string]string{ + "50test-logins.conf": ` +machine http://site1.com login a login b password c login d password e +machine http://site2.com login f password g +`, + }, + matchTests: []matchTest{ + {"http://site1.com/foo", "", "d", "e"}, + {"http://site2.com/bar", "", "f", "g"}, + }, +}, { + summary: "Empty auth dir", + credsFiles: map[string]string{}, + matchTests: []matchTest{ + {"https://example.com/foo", "", "", ""}, + {"http://zombo.com", "", "", ""}, + }, +}, { + summary: "Invalid input URL", + credsFiles: map[string]string{ + "logins": ` +machine login foo password bar login baz +`, + }, + matchTests: []matchTest{ + {":http:foo", "parse \":http:foo\": missing protocol scheme", "", ""}, + {"", "", "", ""}, // this is fine URL apparently, but won't ever match + {"https://login", "", "baz", "bar"}, + }, +}, { + summary: "First entry wins", + credsFiles: map[string]string{ + "logins": ` +machine http://example.com/foo login a password b +machine http://example.com/foo login c password d + +machine example.com/bar login e password f +machine http://example.com/bar login g password h + +machine http://example.com/baz login i password j +machine http://example.com/baz/qux login k password l +`, + }, + matchTests: []matchTest{ + {"http://example.com/foo", "", "a", "b"}, + {"http://example.com/bar", "", "g", "h"}, + {"http://example.com/baz/qux", "", "i", "j"}, + }, +}, { + summary: "First file wins", + credsFiles: map[string]string{ + "10first": ` +machine http://example.com/foo login a password b +machine example.com/bar login e password f +machine http://example.com/baz login i password j +`, + "50second": ` +machine http://example.com/foo login b password c +machine http://example.com/bar login g password h +machine http://example.com/baz/qux login k password l +`, + }, + matchTests: []matchTest{ + {"http://example.com/foo", "", "a", "b"}, + {"http://example.com/bar", "", "g", "h"}, + {"http://example.com/baz/qux", "", "i", "j"}, + }, +}} + +func (s *S) TestFindCredentials(c *C) { + for _, t := range credentialsTests { + s.runFindCredentialsTest(c, &t) + } +} + +func (s *S) runFindCredentialsTest(c *C, t *credentialsTest) { + credsDir := c.MkDir() + restore := archive.FakeCredentialsDir(credsDir) + defer restore() + + for filename, data := range t.credsFiles { + fpath := filepath.Join(credsDir, filename) + err := os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + err = ioutil.WriteFile(fpath, []byte(data), 0644) + c.Assert(err, IsNil) + } + + for _, matchTest := range t.matchTests { + c.Logf("Summary: %s for URL %s", t.summary, matchTest.url) + creds, err := archive.FindCredentials(matchTest.url) + if matchTest.err != "" { + c.Assert(err, ErrorMatches, matchTest.err) + } else { + c.Assert(err, IsNil) + } + c.Assert(creds.Username, Equals, matchTest.username) + c.Assert(creds.Password, Equals, matchTest.password) + } +} + +func (s *S) TestFindCredentialsMissingDir(c *C) { + var creds, emptyCreds archive.Credentials + var err error + + workDir := c.MkDir() + credsDir := filepath.Join(workDir, "auth.conf.d") + restore := archive.FakeCredentialsDir(credsDir) + defer restore() + + creds, err = archive.FindCredentials("https://example.com/foo/bar") + c.Assert(err, IsNil) + c.Assert(creds, Equals, emptyCreds) + + err = os.Mkdir(credsDir, 0755) + c.Assert(err, IsNil) + + creds, err = archive.FindCredentials("https://example.com/foo/bar") + c.Assert(err, IsNil) + c.Assert(creds, Equals, emptyCreds) + + confFile := filepath.Join(credsDir, "example") + err = os.WriteFile(confFile, []byte("machine example.com login admin password swordfish"), 0600) + c.Assert(err, IsNil) + + creds, err = archive.FindCredentials("https://example.com/foo/bar") + c.Assert(err, IsNil) + c.Assert(creds, Not(Equals), emptyCreds) + c.Assert(creds.Username, Equals, "admin") + c.Assert(creds.Password, Equals, "swordfish") +} diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index c2bffea3..662fa937 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -14,3 +14,12 @@ func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) bulkDo = _bulkDo } } + +type Credentials = credentials +var FindCredentials = findCredentials +func FakeCredentialsDir(credentialsDir string) (restore func()) { + testCredentialsDir = credentialsDir + return func() { + testCredentialsDir = "" + } +} From 88c1fb0c51d91bd5dd771a7571bf7eac33edac98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 13:53:46 +0200 Subject: [PATCH 02/23] fixup! Add apt credentials parser Remove global comment. --- internal/archive/credentials.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 5bd7b54b..7ad9f0a9 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -12,10 +12,6 @@ import ( "strings" ) -// This file defines findCredentials() function for searching repository -// credentials in Apt configuration, see -// https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html. - // credentials contains matched non-empty Username and Password. // Username is left empty if the search is unsuccessful. type credentials struct { From cfcc048b0f6660fb8940bfb437566eacb498baa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 13:55:53 +0200 Subject: [PATCH 03/23] fixup! Add apt credentials parser Test findCredsInDir() instead of findCredentials(). The latter calls the former with the directory set to a CHISEL_AUTH_DIR environment variable value if it's non-empty or "/etc/apt/auth.conf.d". --- internal/archive/credentials.go | 10 ++-------- internal/archive/credentials_test.go | 12 ++++-------- internal/archive/export_test.go | 8 +------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 7ad9f0a9..12d36b06 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -69,19 +69,13 @@ func queryFor(repoUrl string, creds *credentials) (*credentialsQuery, error) { return &query, nil } -var testCredentialsDir string - // findCredentials searches credentials for repoUrl in configuration files in // directory specified by CHISEL_AUTH_DIR environment variable if it's // non-empty or /etc/apt/auth.conf.d. func findCredentials(repoUrl string) (credentials, error) { credsDir := "/etc/apt/auth.conf.d" - if testCredentialsDir == "" { - if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { - credsDir = v - } - } else { - credsDir = testCredentialsDir + if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { + credsDir = v } return findCredsInDir(repoUrl, credsDir) } diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go index af0e4c64..375f3b3a 100644 --- a/internal/archive/credentials_test.go +++ b/internal/archive/credentials_test.go @@ -206,8 +206,6 @@ func (s *S) TestFindCredentials(c *C) { func (s *S) runFindCredentialsTest(c *C, t *credentialsTest) { credsDir := c.MkDir() - restore := archive.FakeCredentialsDir(credsDir) - defer restore() for filename, data := range t.credsFiles { fpath := filepath.Join(credsDir, filename) @@ -219,7 +217,7 @@ func (s *S) runFindCredentialsTest(c *C, t *credentialsTest) { for _, matchTest := range t.matchTests { c.Logf("Summary: %s for URL %s", t.summary, matchTest.url) - creds, err := archive.FindCredentials(matchTest.url) + creds, err := archive.FindCredsInDir(matchTest.url, credsDir) if matchTest.err != "" { c.Assert(err, ErrorMatches, matchTest.err) } else { @@ -236,17 +234,15 @@ func (s *S) TestFindCredentialsMissingDir(c *C) { workDir := c.MkDir() credsDir := filepath.Join(workDir, "auth.conf.d") - restore := archive.FakeCredentialsDir(credsDir) - defer restore() - creds, err = archive.FindCredentials("https://example.com/foo/bar") + creds, err = archive.FindCredsInDir("https://example.com/foo/bar", credsDir) c.Assert(err, IsNil) c.Assert(creds, Equals, emptyCreds) err = os.Mkdir(credsDir, 0755) c.Assert(err, IsNil) - creds, err = archive.FindCredentials("https://example.com/foo/bar") + creds, err = archive.FindCredsInDir("https://example.com/foo/bar", credsDir) c.Assert(err, IsNil) c.Assert(creds, Equals, emptyCreds) @@ -254,7 +250,7 @@ func (s *S) TestFindCredentialsMissingDir(c *C) { err = os.WriteFile(confFile, []byte("machine example.com login admin password swordfish"), 0600) c.Assert(err, IsNil) - creds, err = archive.FindCredentials("https://example.com/foo/bar") + creds, err = archive.FindCredsInDir("https://example.com/foo/bar", credsDir) c.Assert(err, IsNil) c.Assert(creds, Not(Equals), emptyCreds) c.Assert(creds.Username, Equals, "admin") diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index 662fa937..1f7bccd5 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -16,10 +16,4 @@ func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) } type Credentials = credentials -var FindCredentials = findCredentials -func FakeCredentialsDir(credentialsDir string) (restore func()) { - testCredentialsDir = credentialsDir - return func() { - testCredentialsDir = "" - } -} +var FindCredsInDir = findCredsInDir From ff67466c0e5c18e9535bcfc4d029975a8806b912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 13:59:06 +0200 Subject: [PATCH 04/23] fixup! Add apt credentials parser Rename findCredsInDir() to findCredentialsInDir(). --- internal/archive/credentials.go | 6 +++--- internal/archive/credentials_test.go | 8 ++++---- internal/archive/export_test.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 12d36b06..cb88757b 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -77,10 +77,10 @@ func findCredentials(repoUrl string) (credentials, error) { if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { credsDir = v } - return findCredsInDir(repoUrl, credsDir) + return findCredentialsInDir(repoUrl, credsDir) } -// findCredsInDir searches for credentials for repoUrl in configuration +// findCredentialsInDir searches for credentials for repoUrl in configuration // files in credsDir directory. If the directory does not exist, empty // credentials structure with nil err is returned. // Only files that do not begin with dot and have either no or ".conf" @@ -88,7 +88,7 @@ func findCredentials(repoUrl string) (credentials, error) { // order. The first file that contains machine declaration matching repoUrl // ends the search. If no file contain matching machine declaration, empty // credentials structure with nil err is returned. -func findCredsInDir(repoUrl string, credsDir string) (creds credentials, err error) { +func findCredentialsInDir(repoUrl string, credsDir string) (creds credentials, err error) { contents, err := os.ReadDir(credsDir) if err != nil { if os.IsNotExist(err) { diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go index 375f3b3a..0abbcdfa 100644 --- a/internal/archive/credentials_test.go +++ b/internal/archive/credentials_test.go @@ -217,7 +217,7 @@ func (s *S) runFindCredentialsTest(c *C, t *credentialsTest) { for _, matchTest := range t.matchTests { c.Logf("Summary: %s for URL %s", t.summary, matchTest.url) - creds, err := archive.FindCredsInDir(matchTest.url, credsDir) + creds, err := archive.FindCredentialsInDir(matchTest.url, credsDir) if matchTest.err != "" { c.Assert(err, ErrorMatches, matchTest.err) } else { @@ -235,14 +235,14 @@ func (s *S) TestFindCredentialsMissingDir(c *C) { workDir := c.MkDir() credsDir := filepath.Join(workDir, "auth.conf.d") - creds, err = archive.FindCredsInDir("https://example.com/foo/bar", credsDir) + creds, err = archive.FindCredentialsInDir("https://example.com/foo/bar", credsDir) c.Assert(err, IsNil) c.Assert(creds, Equals, emptyCreds) err = os.Mkdir(credsDir, 0755) c.Assert(err, IsNil) - creds, err = archive.FindCredsInDir("https://example.com/foo/bar", credsDir) + creds, err = archive.FindCredentialsInDir("https://example.com/foo/bar", credsDir) c.Assert(err, IsNil) c.Assert(creds, Equals, emptyCreds) @@ -250,7 +250,7 @@ func (s *S) TestFindCredentialsMissingDir(c *C) { err = os.WriteFile(confFile, []byte("machine example.com login admin password swordfish"), 0600) c.Assert(err, IsNil) - creds, err = archive.FindCredsInDir("https://example.com/foo/bar", credsDir) + creds, err = archive.FindCredentialsInDir("https://example.com/foo/bar", credsDir) c.Assert(err, IsNil) c.Assert(creds, Not(Equals), emptyCreds) c.Assert(creds.Username, Equals, "admin") diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index 1f7bccd5..b1c27ec4 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -16,4 +16,4 @@ func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) } type Credentials = credentials -var FindCredsInDir = findCredsInDir +var FindCredentialsInDir = findCredentialsInDir From 023a9a1f80bb859da65ed65b19334594fd0c35d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 14:04:11 +0200 Subject: [PATCH 05/23] fixup! Add apt credentials parser Rename queryFor() to parseRepoURL(). --- internal/archive/credentials.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index cb88757b..b3a80be6 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -33,9 +33,9 @@ type credentialsQuery struct { needScheme bool } -// queryFor parses repoUrl into credentialsQuery and fills provided credentials with -// username and password if they are specified in repoUrl. -func queryFor(repoUrl string, creds *credentials) (*credentialsQuery, error) { +// parseRepoURL parses repoUrl into credentialsQuery and fills provided +// credentials with username and password if they are specified in repoUrl. +func parseRepoURL(repoUrl string, creds *credentials) (*credentialsQuery, error) { u, err := url.Parse(repoUrl) if err != nil { return nil, err @@ -99,7 +99,7 @@ func findCredentialsInDir(repoUrl string, credsDir string) (creds credentials, e return } - query, err := queryFor(repoUrl, &creds) + query, err := parseRepoURL(repoUrl, &creds) if err != nil || !creds.Empty() { return } From aea488ebaf6f2d67bc851181cab5e0bc7dbd66ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 14:07:33 +0200 Subject: [PATCH 06/23] fixup! Add apt credentials parser Return credentials parsed from input URL early. --- internal/archive/credentials.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index b3a80be6..8b461db7 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -40,6 +40,16 @@ func parseRepoURL(repoUrl string, creds *credentials) (*credentialsQuery, error) if err != nil { return nil, err } + + if creds != nil { + creds.Username = u.User.Username() + creds.Password, _ = u.User.Password() + + if !creds.Empty() { + return nil, nil + } + } + host := u.Host port := u.Port() if port != "" { @@ -61,11 +71,6 @@ func parseRepoURL(repoUrl string, creds *credentials) (*credentialsQuery, error) needScheme: u.Scheme != "https" && u.Scheme != "tor+https", } - if creds != nil { - creds.Username = u.User.Username() - creds.Password, _ = u.User.Password() - } - return &query, nil } From a803cf32834bfe6fc9f2824352feef4ac9f26778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 14:12:57 +0200 Subject: [PATCH 07/23] fixup! Add apt credentials parser Change parseRepoURL() to return credentials instead of using return pointer argument. --- internal/archive/credentials.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 8b461db7..6a7e9b37 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -35,19 +35,17 @@ type credentialsQuery struct { // parseRepoURL parses repoUrl into credentialsQuery and fills provided // credentials with username and password if they are specified in repoUrl. -func parseRepoURL(repoUrl string, creds *credentials) (*credentialsQuery, error) { +func parseRepoURL(repoUrl string) (creds credentials, query *credentialsQuery, err error) { u, err := url.Parse(repoUrl) if err != nil { - return nil, err + return } - if creds != nil { - creds.Username = u.User.Username() - creds.Password, _ = u.User.Password() + creds.Username = u.User.Username() + creds.Password, _ = u.User.Password() - if !creds.Empty() { - return nil, nil - } + if !creds.Empty() { + return } host := u.Host @@ -60,7 +58,7 @@ func parseRepoURL(repoUrl string, creds *credentials) (*credentialsQuery, error) host = u.Host[0 : len(u.Host)-len(port)-1] } - query := credentialsQuery{ + query = &credentialsQuery{ scheme: u.Scheme, host: host, port: port, @@ -71,7 +69,7 @@ func parseRepoURL(repoUrl string, creds *credentials) (*credentialsQuery, error) needScheme: u.Scheme != "https" && u.Scheme != "tor+https", } - return &query, nil + return } // findCredentials searches credentials for repoUrl in configuration files in @@ -104,7 +102,7 @@ func findCredentialsInDir(repoUrl string, credsDir string) (creds credentials, e return } - query, err := parseRepoURL(repoUrl, &creds) + creds, query, err := parseRepoURL(repoUrl) if err != nil || !creds.Empty() { return } From 15afa405a7d47bf515a30bbba1f2e44f748f6084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 14:14:19 +0200 Subject: [PATCH 08/23] fixup! Add apt credentials parser Capitalize URL. --- internal/archive/credentials.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 6a7e9b37..bd4fe12b 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -33,10 +33,10 @@ type credentialsQuery struct { needScheme bool } -// parseRepoURL parses repoUrl into credentialsQuery and fills provided -// credentials with username and password if they are specified in repoUrl. -func parseRepoURL(repoUrl string) (creds credentials, query *credentialsQuery, err error) { - u, err := url.Parse(repoUrl) +// parseRepoURL parses repoURL into credentialsQuery and fills provided +// credentials with username and password if they are specified in repoURL. +func parseRepoURL(repoURL string) (creds credentials, query *credentialsQuery, err error) { + u, err := url.Parse(repoURL) if err != nil { return } @@ -72,26 +72,26 @@ func parseRepoURL(repoUrl string) (creds credentials, query *credentialsQuery, e return } -// findCredentials searches credentials for repoUrl in configuration files in +// findCredentials searches credentials for repoURL in configuration files in // directory specified by CHISEL_AUTH_DIR environment variable if it's // non-empty or /etc/apt/auth.conf.d. -func findCredentials(repoUrl string) (credentials, error) { +func findCredentials(repoURL string) (credentials, error) { credsDir := "/etc/apt/auth.conf.d" if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { credsDir = v } - return findCredentialsInDir(repoUrl, credsDir) + return findCredentialsInDir(repoURL, credsDir) } -// findCredentialsInDir searches for credentials for repoUrl in configuration +// findCredentialsInDir searches for credentials for repoURL in configuration // files in credsDir directory. If the directory does not exist, empty // credentials structure with nil err is returned. // Only files that do not begin with dot and have either no or ".conf" // extension are searched. The files are searched in ascending lexicographic -// order. The first file that contains machine declaration matching repoUrl +// order. The first file that contains machine declaration matching repoURL // ends the search. If no file contain matching machine declaration, empty // credentials structure with nil err is returned. -func findCredentialsInDir(repoUrl string, credsDir string) (creds credentials, err error) { +func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, err error) { contents, err := os.ReadDir(credsDir) if err != nil { if os.IsNotExist(err) { @@ -102,7 +102,7 @@ func findCredentialsInDir(repoUrl string, credsDir string) (creds credentials, e return } - creds, query, err := parseRepoURL(repoUrl) + creds, query, err := parseRepoURL(repoURL) if err != nil || !creds.Empty() { return } From 8be5a54bfc5f2f1e5ca8f12d1d766bf0242ef48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 14:18:16 +0200 Subject: [PATCH 09/23] fixup! Add apt credentials parser Adorn error return value. --- internal/archive/credentials.go | 6 +++++- internal/archive/credentials_test.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index bd4fe12b..62988d0b 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -103,7 +103,11 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e } creds, query, err := parseRepoURL(repoURL) - if err != nil || !creds.Empty() { + if err != nil { + err = fmt.Errorf("cannot parse archive URL: %w", err) + return + } + if query == nil { // creds.Empty() == false return } diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go index 0abbcdfa..6075fcbb 100644 --- a/internal/archive/credentials_test.go +++ b/internal/archive/credentials_test.go @@ -154,7 +154,7 @@ machine login foo password bar login baz `, }, matchTests: []matchTest{ - {":http:foo", "parse \":http:foo\": missing protocol scheme", "", ""}, + {":http:foo", "cannot parse archive URL: parse \":http:foo\": missing protocol scheme", "", ""}, {"", "", "", ""}, // this is fine URL apparently, but won't ever match {"https://login", "", "baz", "bar"}, }, From 518af57ba0e5215842843f20b23828311196f592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 14:30:35 +0200 Subject: [PATCH 10/23] fixup! Add apt credentials parser Report filename on parser error. --- internal/archive/credentials.go | 5 +++-- internal/archive/credentials_test.go | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 62988d0b..8f0b7e4a 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -139,14 +139,15 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e sort.Strings(confFiles) for _, file := range confFiles { - f, err := os.Open(filepath.Join(credsDir, file)) + fpath := filepath.Join(credsDir, file) + f, err := os.Open(fpath) if err != nil { errs = append(errs, fmt.Errorf("cannot read credentials file: %w", err)) continue } if err = findCredsInFile(query, f, &creds); err != nil { - errs = append(errs, fmt.Errorf("cannot parse credentials file: %w", err)) + errs = append(errs, fmt.Errorf("cannot parse credentials file %s: %w", fpath, err)) } else if !creds.Empty() { break } diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go index 6075fcbb..34dfa4d5 100644 --- a/internal/archive/credentials_test.go +++ b/internal/archive/credentials_test.go @@ -196,6 +196,26 @@ machine http://example.com/baz/qux login k password l {"http://example.com/bar", "", "g", "h"}, {"http://example.com/baz/qux", "", "i", "j"}, }, +}, { + summary: "EOF while epxecting username", + credsFiles: map[string]string{ + "nouser": ` +machine http://example.com/foo login +`, + }, + matchTests: []matchTest{ + {"http://example.com/foo", "cannot parse credentials file .*/nouser: syntax error: reached end of file while expecting username text", "", ""}, + }, +}, { + summary: "EOF while epxecting password", + credsFiles: map[string]string{ + "nopw": ` +machine http://example.com/foo login a password +`, + }, + matchTests: []matchTest{ + {"http://example.com/foo", "cannot parse credentials file .*/nopw: syntax error: reached end of file while expecting password text", "a", ""}, + }, }} func (s *S) TestFindCredentials(c *C) { From 8c109e3209526c552c157289481b5522efdcb995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 28 Jun 2023 14:43:07 +0200 Subject: [PATCH 11/23] fixup! Add apt credentials parser Report file path on os.Open() error too. --- internal/archive/credentials.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 8f0b7e4a..b04b93c4 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -142,7 +142,7 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e fpath := filepath.Join(credsDir, file) f, err := os.Open(fpath) if err != nil { - errs = append(errs, fmt.Errorf("cannot read credentials file: %w", err)) + errs = append(errs, fmt.Errorf("cannot read credentials file %s: %w", fpath, err)) continue } From 959c8c255fb79faaddaafbba407dc1a8048f3bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Thu, 29 Jun 2023 10:49:27 +0200 Subject: [PATCH 12/23] fixup! Add apt credentials parser Skip files with invalid extension before stat()-ing them. --- internal/archive/credentials.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index b04b93c4..28b019e6 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -119,6 +119,9 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e if strings.HasPrefix(name, ".") { continue } + if ext := filepath.Ext(name); ext != "" && ext != ".conf" { + continue + } info, err := entry.Info() if err != nil { errs = append(errs, fmt.Errorf("cannot stat credentials file: %w", err)) @@ -127,10 +130,7 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e if !info.Mode().IsRegular() { continue } - ext := filepath.Ext(name) - if ext == "" || ext == ".conf" { - confFiles = append(confFiles, name) - } + confFiles = append(confFiles, name) } if len(confFiles) == 0 { err = errors.Join(errs...) From c6db9e71aac0600579232ffa13188c675816f67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Thu, 13 Jul 2023 14:47:13 +0200 Subject: [PATCH 13/23] fixup! fixup! Add apt credentials parser --- internal/archive/credentials.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 28b019e6..66cf99a5 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -215,7 +215,7 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential creds: creds, } var err error - for state := netrcInvalid; state != nil; { + for state := netrcStart; state != nil; { state, err = state(&p) } if err := p.scanner.Err(); err != nil { @@ -226,6 +226,8 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential type netrcState func(*netrcParser) (netrcState, error) +var netrcStart = netrcInvalid + func netrcInvalid(p *netrcParser) (netrcState, error) { for p.scanner.Scan() { if p.scanner.Text() == "machine" { From 552b9965881c334178a44f8a2d98b79626d1023b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Mon, 17 Jul 2023 13:29:01 +0200 Subject: [PATCH 14/23] Revert "fixup! fixup! Add apt credentials parser" This reverts commit c6db9e71aac0600579232ffa13188c675816f67c. --- internal/archive/credentials.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 66cf99a5..28b019e6 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -215,7 +215,7 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential creds: creds, } var err error - for state := netrcStart; state != nil; { + for state := netrcInvalid; state != nil; { state, err = state(&p) } if err := p.scanner.Err(); err != nil { @@ -226,8 +226,6 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential type netrcState func(*netrcParser) (netrcState, error) -var netrcStart = netrcInvalid - func netrcInvalid(p *netrcParser) (netrcState, error) { for p.scanner.Scan() { if p.scanner.Text() == "machine" { From dc4f7d9f6cce8615c32bd527068023fd006aec36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Mon, 17 Jul 2023 13:29:39 +0200 Subject: [PATCH 15/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 28b019e6..e7e5a671 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -215,7 +215,7 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential creds: creds, } var err error - for state := netrcInvalid; state != nil; { + for state := netrcStart; state != nil; { state, err = state(&p) } if err := p.scanner.Err(); err != nil { @@ -226,7 +226,7 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential type netrcState func(*netrcParser) (netrcState, error) -func netrcInvalid(p *netrcParser) (netrcState, error) { +func netrcStart(p *netrcParser) (netrcState, error) { for p.scanner.Scan() { if p.scanner.Text() == "machine" { return netrcMachine, nil @@ -242,29 +242,29 @@ func netrcMachine(p *netrcParser) (netrcState, error) { token := p.scanner.Text() if i := strings.Index(token, "://"); i != -1 { if token[0:i] != p.query.scheme { - return netrcInvalid, nil + return netrcStart, nil } token = token[i+3:] } else if p.query.needScheme { - return netrcInvalid, nil + return netrcStart, nil } if !strings.HasPrefix(token, p.query.host) { - return netrcInvalid, nil + return netrcStart, nil } token = token[len(p.query.host):] if len(token) > 0 { if token[0] == ':' { if p.query.port == "" { - return netrcInvalid, nil + return netrcStart, nil } token = token[1:] if !strings.HasPrefix(token, p.query.port) { - return netrcInvalid, nil + return netrcStart, nil } token = token[len(p.query.port):] } if !strings.HasPrefix(p.query.path, token) { - return netrcInvalid, nil + return netrcStart, nil } } return netrcGoodMachine, nil From 715a51cd360a680a853bcc12a44c56a303060276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Tue, 18 Jul 2023 09:31:16 +0200 Subject: [PATCH 16/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index e7e5a671..25e5779b 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -216,6 +216,9 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential } var err error for state := netrcStart; state != nil; { + if err != nil { + panic("internal error: state != nil && err != nil") + } state, err = state(&p) } if err := p.scanner.Err(); err != nil { From 5f4627f5daeb9706514263371c6c63c7aae616ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Mon, 24 Jul 2023 19:10:03 +0200 Subject: [PATCH 17/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 25e5779b..55bf6130 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -21,7 +21,7 @@ type credentials struct { // Empty checks whether c represents unsuccessful search. func (c credentials) Empty() bool { - return c.Username == "" + return c.Username == "" && c.Password == "" } // credentialsQuery contains parsed input URL data used for search. From b50901ee150cb9629686ba9cc911890688d55021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Mon, 24 Jul 2023 19:10:10 +0200 Subject: [PATCH 18/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 55bf6130..253b1b98 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -74,7 +74,7 @@ func parseRepoURL(repoURL string) (creds credentials, query *credentialsQuery, e // findCredentials searches credentials for repoURL in configuration files in // directory specified by CHISEL_AUTH_DIR environment variable if it's -// non-empty or /etc/apt/auth.conf.d. +// non-empty, otherwise /etc/apt/auth.conf.d. func findCredentials(repoURL string) (credentials, error) { credsDir := "/etc/apt/auth.conf.d" if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { From 010e9697dbeddfdebba21c2eefe301b6a61af96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Mon, 24 Jul 2023 19:10:18 +0200 Subject: [PATCH 19/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 253b1b98..1ca59f93 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -255,20 +255,18 @@ func netrcMachine(p *netrcParser) (netrcState, error) { return netrcStart, nil } token = token[len(p.query.host):] - if len(token) > 0 { - if token[0] == ':' { - if p.query.port == "" { - return netrcStart, nil - } - token = token[1:] - if !strings.HasPrefix(token, p.query.port) { - return netrcStart, nil - } - token = token[len(p.query.port):] + if len(token) > 0 && token[0] == ':' { + if p.query.port == "" { + return netrcStart, nil } - if !strings.HasPrefix(p.query.path, token) { + token = token[1:] + if !strings.HasPrefix(token, p.query.port) { return netrcStart, nil } + token = token[len(p.query.port):] + } + if !strings.HasPrefix(p.query.path, token) { + return netrcStart, nil } return netrcGoodMachine, nil } From 5e335b1f3e0ba6d23d04908f800958be35fd2d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Mon, 24 Jul 2023 19:10:43 +0200 Subject: [PATCH 20/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 1ca59f93..4e006d31 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -215,14 +215,9 @@ func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credential creds: creds, } var err error - for state := netrcStart; state != nil; { - if err != nil { - panic("internal error: state != nil && err != nil") - } + for state := netrcStart; err == nil && state != nil; { state, err = state(&p) - } - if err := p.scanner.Err(); err != nil { - return err + err = errors.Join(err, p.scanner.Err()) } return err } From 482ff15f9947ce7ad655b01abc2dbacaa6b278d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Tue, 25 Jul 2023 17:03:56 +0200 Subject: [PATCH 21/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 4e006d31..a0373be3 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -96,6 +96,7 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e if err != nil { if os.IsNotExist(err) { err = nil + debugf("credentials directory %#v does not exist", credsDir) } else { err = fmt.Errorf("cannot open credentials directory: %w", err) } From db609e137322af123e9e9dc09920bee7648a579b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Tue, 8 Aug 2023 14:13:22 +0200 Subject: [PATCH 22/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 65 +++++++++++++++------------- internal/archive/credentials_test.go | 45 +++++++++---------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index a0373be3..8a91476a 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -35,12 +35,13 @@ type credentialsQuery struct { // parseRepoURL parses repoURL into credentialsQuery and fills provided // credentials with username and password if they are specified in repoURL. -func parseRepoURL(repoURL string) (creds credentials, query *credentialsQuery, err error) { +func parseRepoURL(repoURL string) (creds *credentials, query *credentialsQuery, err error) { u, err := url.Parse(repoURL) if err != nil { return } + creds = &credentials{} creds.Username = u.User.Username() creds.Password, _ = u.User.Password() @@ -72,10 +73,12 @@ func parseRepoURL(repoURL string) (creds credentials, query *credentialsQuery, e return } +var ErrCredentialsNotFound = errors.New("credentials not found") + // findCredentials searches credentials for repoURL in configuration files in // directory specified by CHISEL_AUTH_DIR environment variable if it's // non-empty, otherwise /etc/apt/auth.conf.d. -func findCredentials(repoURL string) (credentials, error) { +func findCredentials(repoURL string) (*credentials, error) { credsDir := "/etc/apt/auth.conf.d" if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { credsDir = v @@ -83,6 +86,7 @@ func findCredentials(repoURL string) (credentials, error) { return findCredentialsInDir(repoURL, credsDir) } +var FindCredentials = findCredentials // findCredentialsInDir searches for credentials for repoURL in configuration // files in credsDir directory. If the directory does not exist, empty // credentials structure with nil err is returned. @@ -91,29 +95,21 @@ func findCredentials(repoURL string) (credentials, error) { // order. The first file that contains machine declaration matching repoURL // ends the search. If no file contain matching machine declaration, empty // credentials structure with nil err is returned. -func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, err error) { +func findCredentialsInDir(repoURL string, credsDir string) (*credentials, error) { contents, err := os.ReadDir(credsDir) if err != nil { - if os.IsNotExist(err) { - err = nil - debugf("credentials directory %#v does not exist", credsDir) - } else { - err = fmt.Errorf("cannot open credentials directory: %w", err) - } - return + logf("Cannot open credentials directory %q: %v", credsDir, err) + return nil, ErrCredentialsNotFound } creds, query, err := parseRepoURL(repoURL) if err != nil { - err = fmt.Errorf("cannot parse archive URL: %w", err) - return + return nil, fmt.Errorf("cannot parse archive URL: %v", err) } - if query == nil { // creds.Empty() == false - return + if !creds.Empty() { + return creds, nil } - errs := make([]error, 0) - confFiles := make([]string, 0, len(contents)) for _, entry := range contents { name := entry.Name() @@ -125,7 +121,7 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e } info, err := entry.Info() if err != nil { - errs = append(errs, fmt.Errorf("cannot stat credentials file: %w", err)) + logf("Cannot stat credentials file %q: %v", filepath.Join(credsDir, name), err) continue } if !info.Mode().IsRegular() { @@ -134,8 +130,7 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e confFiles = append(confFiles, name) } if len(confFiles) == 0 { - err = errors.Join(errs...) - return + return nil, ErrCredentialsNotFound } sort.Strings(confFiles) @@ -143,19 +138,21 @@ func findCredentialsInDir(repoURL string, credsDir string) (creds credentials, e fpath := filepath.Join(credsDir, file) f, err := os.Open(fpath) if err != nil { - errs = append(errs, fmt.Errorf("cannot read credentials file %s: %w", fpath, err)) + logf("Cannot open credentials file %q: %v", fpath, err) continue } - - if err = findCredsInFile(query, f, &creds); err != nil { - errs = append(errs, fmt.Errorf("cannot parse credentials file %s: %w", fpath, err)) - } else if !creds.Empty() { - break + creds, err = findCredentialsInternal(query, f) + if closeErr := f.Close(); closeErr != nil { + logf("Cannot close credentials file %q: %v", fpath, err) + } + if err == nil { + return creds, nil + } else if err != ErrCredentialsNotFound { + logf("Cannot parse credentials file %q: %v", fpath, err) } } - err = errors.Join(errs...) - return + return nil, ErrCredentialsNotFound } type netrcParser struct { @@ -164,7 +161,7 @@ type netrcParser struct { creds *credentials } -// findCredsInFile searches for credentials in netrc file matching query +// findCredentialsInternal searches for credentials in netrc file matching query // and fills creds with matched credentials if there's a match. The first match // ends the search. // @@ -207,20 +204,26 @@ type netrcParser struct { // [3] https://salsa.debian.org/apt-team/apt/-/blob/4e04cbaf/methods/aptmethod.h#L560 // [4] https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html // [5] https://daniel.haxx.se/blog/2022/05/31/netrc-pains/ -func findCredsInFile(query *credentialsQuery, netrc io.Reader, creds *credentials) error { +func findCredentialsInternal(query *credentialsQuery, netrc io.Reader) (*credentials, error) { s := bufio.NewScanner(netrc) s.Split(bufio.ScanWords) p := netrcParser{ query: query, scanner: s, - creds: creds, + creds: &credentials{}, } var err error for state := netrcStart; err == nil && state != nil; { state, err = state(&p) err = errors.Join(err, p.scanner.Err()) } - return err + if err != nil { + return nil, err + } + if p.creds.Empty() { + return nil, ErrCredentialsNotFound + } + return p.creds, nil } type netrcState func(*netrcParser) (netrcState, error) diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go index 34dfa4d5..6f2802f3 100644 --- a/internal/archive/credentials_test.go +++ b/internal/archive/credentials_test.go @@ -54,14 +54,14 @@ machine socks5h://example.last/debian login debian password rules {"https://example.org:8080/foo", "", "example", "foobar"}, {"https://example.net:42/foo", "", "foo", "bar"}, {"https://example.org/foo", "", "anonymous", "pass"}, - {"https://example.com/apt", "", "", ""}, + {"https://example.com/apt", "^credentials not found$", "", ""}, {"https://example.com/foo", "", "user1", "pass1"}, {"https://example.com/fooo", "", "user1", "pass1"}, - {"https://example.com/fo", "", "", ""}, + {"https://example.com/fo", "^credentials not found$", "", ""}, {"https://example.com/bar", "", "user2", "pass2"}, {"https://example.com/user", "", "user", ""}, {"socks5h://example.last/debian", "", "debian", "rules"}, - {"socks5h://example.debian/", "", "", ""}, + {"socks5h://example.debian/", "^credentials not found$", "", ""}, {"socks5h://user:pass@example.debian/", "", "user", "pass"}, }, }, { @@ -74,7 +74,7 @@ machine2 example.org login foo3 password bar `, }, matchTests: []matchTest{ - {"https://example.org/foo", "", "", ""}, + {"https://example.org/foo", "^credentials not found$", "", ""}, }, }, { summary: "Bad file: Ends machine", @@ -85,7 +85,7 @@ machine`, }, matchTests: []matchTest{ {"https://example.org/foo", "", "foo1", "bar"}, - {"https://example.net/foo", ".*\\breached end of file while expecting machine text\\b.*", "", ""}, + {"https://example.net/foo", "^credentials not found$", "", ""}, {"https://foo:bar@example.net/foo", "", "foo", "bar"}, }, }, { @@ -98,7 +98,7 @@ machine example.net login }, matchTests: []matchTest{ {"https://example.org/foo", "", "foo1", "bar"}, - {"https://example.net/foo", ".*\\breached end of file while expecting username text\\b.*", "", ""}, + {"https://example.net/foo", "^credentials not found$", "", ""}, {"https://foo:bar@example.net/foo", "", "foo", "bar"}, }, }, { @@ -111,9 +111,9 @@ machine http://http.example login foo1 password bar }, matchTests: []matchTest{ {"https://https.example/foo", "", "foo1", "bar"}, - {"http://https.example/foo", "", "", ""}, + {"http://https.example/foo", "^credentials not found$", "", ""}, {"http://http.example/foo", "", "foo1", "bar"}, - {"https://http.example/foo", "", "", ""}, + {"https://http.example/foo", "^credentials not found$", "", ""}, }, }, { summary: "Password is machine", @@ -125,7 +125,7 @@ machine http://site2.com login u2 password p2 }, matchTests: []matchTest{ {"http://site1.com/foo", "", "u1", "machine"}, - {"http://site2.com/bar", "", "", ""}, + {"http://site2.com/bar", "^credentials not found$", "", ""}, }, }, { summary: "Multiple login and password tokens", @@ -143,8 +143,8 @@ machine http://site2.com login f password g summary: "Empty auth dir", credsFiles: map[string]string{}, matchTests: []matchTest{ - {"https://example.com/foo", "", "", ""}, - {"http://zombo.com", "", "", ""}, + {"https://example.com/foo", "^credentials not found$", "", ""}, + {"http://zombo.com", "^credentials not found$", "", ""}, }, }, { summary: "Invalid input URL", @@ -155,7 +155,7 @@ machine login foo password bar login baz }, matchTests: []matchTest{ {":http:foo", "cannot parse archive URL: parse \":http:foo\": missing protocol scheme", "", ""}, - {"", "", "", ""}, // this is fine URL apparently, but won't ever match + {"", "^credentials not found$", "", ""}, // this is fine URL apparently, but won't ever match {"https://login", "", "baz", "bar"}, }, }, { @@ -204,7 +204,7 @@ machine http://example.com/foo login `, }, matchTests: []matchTest{ - {"http://example.com/foo", "cannot parse credentials file .*/nouser: syntax error: reached end of file while expecting username text", "", ""}, + {"http://example.com/foo", "^credentials not found$", "", ""}, }, }, { summary: "EOF while epxecting password", @@ -214,7 +214,7 @@ machine http://example.com/foo login a password `, }, matchTests: []matchTest{ - {"http://example.com/foo", "cannot parse credentials file .*/nopw: syntax error: reached end of file while expecting password text", "a", ""}, + {"http://example.com/foo", "^credentials not found$", "a", ""}, }, }} @@ -242,29 +242,30 @@ func (s *S) runFindCredentialsTest(c *C, t *credentialsTest) { c.Assert(err, ErrorMatches, matchTest.err) } else { c.Assert(err, IsNil) + c.Assert(creds, NotNil) + c.Assert(creds.Username, Equals, matchTest.username) + c.Assert(creds.Password, Equals, matchTest.password) } - c.Assert(creds.Username, Equals, matchTest.username) - c.Assert(creds.Password, Equals, matchTest.password) } } func (s *S) TestFindCredentialsMissingDir(c *C) { - var creds, emptyCreds archive.Credentials + var creds *archive.Credentials var err error workDir := c.MkDir() credsDir := filepath.Join(workDir, "auth.conf.d") creds, err = archive.FindCredentialsInDir("https://example.com/foo/bar", credsDir) - c.Assert(err, IsNil) - c.Assert(creds, Equals, emptyCreds) + c.Assert(err, ErrorMatches, "^credentials not found$") + c.Assert(creds, IsNil) err = os.Mkdir(credsDir, 0755) c.Assert(err, IsNil) creds, err = archive.FindCredentialsInDir("https://example.com/foo/bar", credsDir) - c.Assert(err, IsNil) - c.Assert(creds, Equals, emptyCreds) + c.Assert(err, ErrorMatches, "^credentials not found$") + c.Assert(creds, IsNil) confFile := filepath.Join(credsDir, "example") err = os.WriteFile(confFile, []byte("machine example.com login admin password swordfish"), 0600) @@ -272,7 +273,7 @@ func (s *S) TestFindCredentialsMissingDir(c *C) { creds, err = archive.FindCredentialsInDir("https://example.com/foo/bar", credsDir) c.Assert(err, IsNil) - c.Assert(creds, Not(Equals), emptyCreds) + c.Assert(creds, NotNil) c.Assert(creds.Username, Equals, "admin") c.Assert(creds.Password, Equals, "swordfish") } From ddefc450c140f73b03c832f2f184df3608156081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Wed, 9 Aug 2023 21:34:47 +0200 Subject: [PATCH 23/23] fixup! Add apt credentials parser --- internal/archive/credentials.go | 5 +-- internal/archive/credentials_test.go | 52 +++++++++++++++++++++++++--- internal/archive/export_test.go | 1 + 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/internal/archive/credentials.go b/internal/archive/credentials.go index 8a91476a..1f8c3289 100644 --- a/internal/archive/credentials.go +++ b/internal/archive/credentials.go @@ -73,20 +73,21 @@ func parseRepoURL(repoURL string) (creds *credentials, query *credentialsQuery, return } +const defaultCredsDir = "/etc/apt/auth.conf.d" + var ErrCredentialsNotFound = errors.New("credentials not found") // findCredentials searches credentials for repoURL in configuration files in // directory specified by CHISEL_AUTH_DIR environment variable if it's // non-empty, otherwise /etc/apt/auth.conf.d. func findCredentials(repoURL string) (*credentials, error) { - credsDir := "/etc/apt/auth.conf.d" + credsDir := defaultCredsDir if v := os.Getenv("CHISEL_AUTH_DIR"); v != "" { credsDir = v } return findCredentialsInDir(repoURL, credsDir) } -var FindCredentials = findCredentials // findCredentialsInDir searches for credentials for repoURL in configuration // files in credsDir directory. If the directory does not exist, empty // credentials structure with nil err is returned. diff --git a/internal/archive/credentials_test.go b/internal/archive/credentials_test.go index 6f2802f3..9fe3e75b 100644 --- a/internal/archive/credentials_test.go +++ b/internal/archive/credentials_test.go @@ -218,13 +218,13 @@ machine http://example.com/foo login a password }, }} -func (s *S) TestFindCredentials(c *C) { +func (s *S) TestFindCredentialsInDir(c *C) { for _, t := range credentialsTests { - s.runFindCredentialsTest(c, &t) + s.runFindCredentialsInDirTest(c, &t) } } -func (s *S) runFindCredentialsTest(c *C, t *credentialsTest) { +func (s *S) runFindCredentialsInDirTest(c *C, t *credentialsTest) { credsDir := c.MkDir() for filename, data := range t.credsFiles { @@ -249,7 +249,7 @@ func (s *S) runFindCredentialsTest(c *C, t *credentialsTest) { } } -func (s *S) TestFindCredentialsMissingDir(c *C) { +func (s *S) TestFindCredentialsInDirMissingDir(c *C) { var creds *archive.Credentials var err error @@ -277,3 +277,47 @@ func (s *S) TestFindCredentialsMissingDir(c *C) { c.Assert(creds.Username, Equals, "admin") c.Assert(creds.Password, Equals, "swordfish") } + +func fakeEnv(name, value string) (restore func()) { + origValue, origSet := os.LookupEnv(name) + os.Setenv(name, value) + return func() { + if origSet { + os.Setenv(name, origValue) + } else { + os.Unsetenv(name) + } + } +} + +func (s *S) TestFindCredentials(c *C) { + var creds *archive.Credentials + var err error + + workDir := c.MkDir() + credsDir := filepath.Join(workDir, "auth.conf.d") + + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + creds, err = archive.FindCredentials("http://example.com/my/site") + c.Assert(err, ErrorMatches, "^credentials not found$") + c.Assert(creds, IsNil) + + err = os.Mkdir(credsDir, 0755) + c.Assert(err, IsNil) + + creds, err = archive.FindCredentials("http://example.com/my/site") + c.Assert(err, ErrorMatches, "^credentials not found$") + c.Assert(creds, IsNil) + + confFile := filepath.Join(credsDir, "mysite") + err = os.WriteFile(confFile, []byte("machine http://example.com/my login johndoe password 12345"), 0600) + c.Assert(err, IsNil) + + creds, err = archive.FindCredentials("http://example.com/my/site") + c.Assert(err, IsNil) + c.Assert(creds, NotNil) + c.Assert(creds.Username, Equals, "johndoe") + c.Assert(creds.Password, Equals, "12345") +} diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index b1c27ec4..fe7c62fb 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -16,4 +16,5 @@ func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) } type Credentials = credentials +var FindCredentials = findCredentials var FindCredentialsInDir = findCredentialsInDir