From 787a56f7ea2694fc601e2956edf938e9da1205e4 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Fri, 4 Aug 2023 09:04:21 +0100 Subject: [PATCH 1/6] actions: debootstrap: Add new property parent-suite Allow downstream distros to indicate which suite the bootstrapping should be done for. For now, only use this property to gate the debootstrap workaround for excluding usr-is-merged package to the parent suite which should be beneficial to downstreams. Fixes: go-debos/debos!361 Signed-off-by: Christopher Obbard --- actions/debootstrap_action.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index ac42aa7a..2d548ed4 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -46,6 +46,9 @@ Example: - certificate -- client certificate stored in file to be used for downloading packages from the server. - private-key -- provide the client's private key in a file separate from the certificate. + +- parent-suite -- release code name which this suite is based on. Useful for downstreams which do + not use debian codenames for their suite names (e.g. "stable"). */ package actions @@ -64,6 +67,7 @@ import ( type DebootstrapAction struct { debos.BaseAction `yaml:",inline"` + ParentSuite string `yaml:"parent-suite"` Suite string Mirror string Variant string @@ -115,6 +119,10 @@ func (d *DebootstrapAction) Verify(context *debos.DebosContext) error { return fmt.Errorf("suite property not specified") } + if len(d.ParentSuite) == 0 { + d.ParentSuite = d.Suite + } + files := d.listOptionFiles(context) // Check if all needed files exists @@ -163,9 +171,9 @@ func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error { return err } -// Guess if suite is something before usr-is-merged was introduced -func (d *DebootstrapAction) isLikelyOldSuite() bool { - switch strings.ToLower(d.Suite) { +// Check if suite is something before usr-is-merged was introduced +func shouldExcludeUsrIsMerged(suite string) bool { + switch strings.ToLower(suite) { case "sid", "unstable": return false case "testing": @@ -226,9 +234,8 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error { cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant)) } - // workaround for https://github.com/go-debos/debos/issues/361 - if d.isLikelyOldSuite() { - log.Println("excluding usr-is-merged as package is not in suite") + if shouldExcludeUsrIsMerged(d.ParentSuite) { + log.Printf("excluding usr-is-merged as package is not in parent suite %s\n", d.ParentSuite) cmdline = append(cmdline, "--exclude=usr-is-merged") } From cb59e3c8a9fdf9d837753f5bd3748e2ee91bde7b Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Thu, 10 Aug 2023 10:43:30 +0100 Subject: [PATCH 2/6] actions: debootstrap: Allow Script property to be set Allow the debootstrap script to be set by the user to a full path. If unset, use the existing behaviour of using the unstable debootstrap script. Signed-off-by: Christopher Obbard --- actions/debootstrap_action.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index 2d548ed4..775393f5 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -49,6 +49,9 @@ Example: - parent-suite -- release code name which this suite is based on. Useful for downstreams which do not use debian codenames for their suite names (e.g. "stable"). + +- script -- the full path of the script to use to build the target rootfs. (e.g. `/usr/share/debootstrap/scripts/kali`) + If unspecified, the property will be set to use the `unstable` script. */ package actions @@ -78,6 +81,7 @@ type DebootstrapAction struct { Components []string MergedUsr bool `yaml:"merged-usr"` CheckGpg bool `yaml:"check-gpg"` + Script string } func NewDebootstrapAction() *DebootstrapAction { @@ -242,7 +246,16 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error { cmdline = append(cmdline, d.Suite) cmdline = append(cmdline, context.Rootdir) cmdline = append(cmdline, d.Mirror) - cmdline = append(cmdline, "/usr/share/debootstrap/scripts/unstable") + + if len(d.Script) > 0 { + if _, err := os.Stat(d.Script); err != nil { + return fmt.Errorf("cannot find debootstrap script %s", d.Script) + } + } else { + d.Script = "/usr/share/debootstrap/scripts/unstable" + } + + cmdline = append(cmdline, d.Script) /* Make sure /etc/apt/apt.conf.d exists inside the fakemachine otherwise debootstrap prints a warning about the path not existing. */ From 351da8c0eb51b89283c5caadd47511a9a3e67757 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Thu, 10 Aug 2023 10:47:11 +0100 Subject: [PATCH 3/6] actions: debootstrap: Determine default script based on suite/parent-suite If the debootstrap script property is unspecified, set the script to be the suite property, falling back to the parent suite if the script doesn't exist and finally falling back to unstable if the parent suite doesn't have a custom script. Signed-off-by: Christopher Obbard --- actions/debootstrap_action.go | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/actions/debootstrap_action.go b/actions/debootstrap_action.go index 775393f5..4b9d3b72 100644 --- a/actions/debootstrap_action.go +++ b/actions/debootstrap_action.go @@ -51,11 +51,14 @@ Example: not use debian codenames for their suite names (e.g. "stable"). - script -- the full path of the script to use to build the target rootfs. (e.g. `/usr/share/debootstrap/scripts/kali`) - If unspecified, the property will be set to use the `unstable` script. + If unspecified, the property will be automatically determined in the following order, + with the path "/usr/share/debootstrap/scripts/" prepended: + `suite` property, `parent-suite` property then `unstable`. */ package actions import ( + "errors" "fmt" "io" "log" @@ -193,6 +196,10 @@ func shouldExcludeUsrIsMerged(suite string) bool { } } +func getDebootstrapScriptPath(script string) string { + return path.Join("/usr/share/debootstrap/scripts/", script) +} + func (d *DebootstrapAction) Run(context *debos.DebosContext) error { cmdline := []string{"debootstrap"} @@ -252,7 +259,24 @@ func (d *DebootstrapAction) Run(context *debos.DebosContext) error { return fmt.Errorf("cannot find debootstrap script %s", d.Script) } } else { - d.Script = "/usr/share/debootstrap/scripts/unstable" + /* Auto determine debootstrap script to use from d.Suite, falling back to + d.ParentSuite if it doesn't exist. Finally, fallback to unstable if a + script for the parent suite does not exist. */ + for _, s := range []string{d.Suite, d.ParentSuite, "unstable"} { + d.Script = getDebootstrapScriptPath(s) + if _, err := os.Stat(d.Script); err == nil { + break + } else { + log.Printf("cannot find debootstrap script %s\n", d.Script) + + /* Unstable should always be available so error out if not */ + if s == "unstable" { + return errors.New("cannot find debootstrap script for unstable") + } + } + } + + log.Printf("using debootstrap script %s\n", d.Script) } cmdline = append(cmdline, d.Script) From 0c58396d1690dd28cb232651393ec719d65fcda1 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Thu, 10 Aug 2023 10:52:31 +0100 Subject: [PATCH 4/6] ci: Add Kali rolling Debootstrap test Kali rolling is based on Debian unstable. Add a test to ensure that we can build downstream distros with a different suite name to upstream Debian. Signed-off-by: Christopher Obbard --- .github/workflows/ci.yaml | 2 ++ tests/kali/kali-archive-keyring.gpg | Bin 0 -> 2271 bytes tests/kali/test.yaml | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 tests/kali/kali-archive-keyring.gpg create mode 100644 tests/kali/test.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d0d9cdee..3a20b965 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -136,6 +136,8 @@ jobs: test: { name: "arch", case: "arch" } - backend: kvm test: { name: "apertis", case: "apertis" } + - backend: kvm + test: { name: "kali", case: "kali" } name: ${{matrix.test.name}} on ${{matrix.backend}} runs-on: ${{ matrix.backend == 'kvm' && 'kvm' || 'ubuntu-latest' }} steps: diff --git a/tests/kali/kali-archive-keyring.gpg b/tests/kali/kali-archive-keyring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..244b2b5027842590a8525f914b94a98baaca3893 GIT binary patch literal 2271 zcmV<52q5>F0u2ODRMaQ|5CEZie|~cCQt~PXpd~uM7R&ZFh?A51W<%+WJ1D%%;+m#f zqcpsHKp8V_;3iRPYS%uk8D&@4PiA4NhSd4d62r_b@)YaEoSJ|>`c(|=qNM-S%oi$-fXCm1s$ZzA zs0Fn6uXmPZP5DkT6wKcc9{5ni4Opl*_F#d~uho4IJ&GPPykA<(Lt7U&#n9~`)7l3Y zhEr)lUL(RtnxFCXP`)nrl*pPb_TTZpLDf;+JY$34JC6d1bCas7ru;L-&<4jOYE!z2 z&akYv{njCqv2U^Y^w={Uv*b#$=>ZcwsZ4?h%)muc2&(*Mnl%-$>RHanPB1cRa7C?k zK~Te5Ln*lG^|bYSGvN#Z7Fg}fhYn;@I8yf?K(S(SCyLLc)YKx9t4qY%^I6<%T}`B7$D($3Gv%D!fbqf0?KB+OkSDED}q6qu&tyr63A?NnmzXV}@K|tlvf(E9`N%2-7 zw@axbxlN02TBi-a-0eSM2fer9KXR=R$!{nVy#V|KkU$Oua_{BHoGNtHUMB#SQoC&` zqLCoe&60Iuy%zux0RRECCQD&#X&_8#ZgqGdQe|*&b7^#Ma(N&;WMy_`Y(Q&aY-uiU za%Vn?0#yVP0SW*=0viJc3ke7Z0|EvW2m%QT3j`Jd0|5da0Rk6*79j*g#!)(sPqWhq zb@y29L{IR2jSKb#0%7m*P6Y`VKDVX-3JDPHL{IR2jSKdN)DQo2iO5WO4t`$|=cIJf znL|!S1K72B1k%Y702oyl`Mesvw)oDEE3g= zLEkYD>2M{9&QBG6XI}r2@6i1hOCQ+zRC_$aRXa7L7rHc%&Kn^-cQcS!V^1uV>KQQB zX!gXB&}VaPdI(-ifxyAOpA9OF`!XZM9I`HUQcoC#(4>K&8}jxt#LNz;qIyVz1T{Xg zb0%=&=Y?f&~J~8iOg)DEJm!KxW!*&OYQ=$WS2$}(;^SFGk-t?G*n`i~GL5_}!uyMG|F!#lbw@8)~c;pKhnb0 zaF{mO3X5aj!3#U?U_f->Js+F3bJ|l|fYWIPX}F z<-6p2Kgit;0Cx*rKYEttM(*#b)J;&w`>W#wMU(%0nK^^(JYtzj z^9Q68@Ao8jw9c{yFmnZ6|G@y796wm6dl?VZaY<`HzD?fE7|^1BHCu$VC#O_oy{4iL z;9D@`_Wm|-g?#r%vrA6OtDf;H@0C?m*9?*arEvz!JJuzc zG(p1V0RMsDyz6mVI~B?mmDa>RjCjxCtw*1lKD&JmD~~kbC{%5V`I0L*AOmq;IpqR_ zNE&o+Y#4vwe-!8gtyBLTL|9C&z?hENB~tz%V?uY_j57cc0RRDs0z3p50SW*n0vik# zAp}ImQ96!Kv(pH5_gL*jPw;(>3-$#9Vej%{1qm2FxViue2@vf>Pw;(>3-)q05CD=a zIJpPXIdt~rNMp_uIg0M%d%uX;UZq%cVjFB~3WAhrrQO0t+CC0OPeeT0|L5P{9z8_}R-0a-B9i z$$(#2-P%|zz5MX*fn!TSJYT;JJ1BUPl{#nX(WM& zFvS1qi)6POkJ?hX3HZ^Kggs#kQ$1M9GB|A|m_zt0W-;oxZ+C^Vnv%sRbe6JXE;H$TuYQA@p)=$LLdi`rx?jWD4NH}m{Jb?fJ literal 0 HcmV?d00001 diff --git a/tests/kali/test.yaml b/tests/kali/test.yaml new file mode 100644 index 00000000..5a40306f --- /dev/null +++ b/tests/kali/test.yaml @@ -0,0 +1,23 @@ +--- +# Test building a non-debian distribution based on unstable such as kali-rolling +# to ensure bootstrapping suites that debootstrap won't internally know about +# works +{{- $architecture := or .architecture "amd64" }} + +architecture: {{ $architecture }} + +actions: + - action: debootstrap + suite: kali-rolling + parent-suite: testing + components: + - main + mirror: https://http.kali.org/kali/ + variant: minbase + keyring-package: kali-archive-keyring + keyring-file: kali-archive-keyring.gpg + + - action: apt + description: Install some base packages + packages: + - procps From 5a838e55c81f58468016d9106807f8c5e67bc2a6 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Thu, 10 Aug 2023 10:56:55 +0100 Subject: [PATCH 5/6] ci: Add Apertis v2023 debootstrap test Apertis v2023 is based on Debian bookworm. Add a test to ensure the parent-suite logic works to build downstream distros with a different suite name to upstream Debian. Signed-off-by: Christopher Obbard --- .github/workflows/ci.yaml | 4 ++++ tests/apertis/test.yaml | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3a20b965..12244d92 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -138,6 +138,10 @@ jobs: test: { name: "apertis", case: "apertis" } - backend: kvm test: { name: "kali", case: "kali" } + - backend: kvm + test: { name: "apertis v2022", case: "apertis", variables: "-t suite:v2022" } + - backend: kvm + test: { name: "apertis v2024dev3", case: "apertis", variables: "-t suite:v2024dev3 -t parent_suite:bookworm" } name: ${{matrix.test.name}} on ${{matrix.backend}} runs-on: ${{ matrix.backend == 'kvm' && 'kvm' || 'ubuntu-latest' }} steps: diff --git a/tests/apertis/test.yaml b/tests/apertis/test.yaml index 4cde8606..62040447 100644 --- a/tests/apertis/test.yaml +++ b/tests/apertis/test.yaml @@ -1,12 +1,17 @@ --- # Test building a non-debian distribution such as apertis to ensure # bootstrapping suites that debootstrap won't internally know about works -{{- $architecture := or .architecture "amd64"}} -architecture: {{$architecture}} + +{{- $architecture := or .architecture "amd64" }} +{{- $suite := or .suite "v2022" }} +{{- $parent_suite := or .parent_suite "" }} + +architecture: {{ $architecture }} actions: - action: debootstrap - suite: v2022 + suite: {{ $suite }} + parent-suite: {{ $parent_suite }} components: - target mirror: https://repositories.apertis.org/apertis/ From 7b6ce520cd8f556b0906831b40c2c1daa7851764 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Thu, 10 Aug 2023 11:04:53 +0100 Subject: [PATCH 6/6] ci: Add Debian bookworm/trixie debootstrap test Ensure that we can build current debian stable & testing. Signed-off-by: Christopher Obbard --- .github/workflows/ci.yaml | 9 ++++++--- tests/debian/test.yaml | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 12244d92..9fbe02e9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -125,9 +125,12 @@ jobs: test: - { name: "recipes", case: "recipes" } - { name: "templating", case: "templating", variables: " -t escaped:\\$ba\\'d\\$gers\\ snakes" } - - { name: "debian (amd64)", case: "debian", variables: "-t architecture:amd64" } - - { name: "debian (arm64)", case: "debian", variables: "-t architecture:arm64" } - - { name: "debian (armhf)", case: "debian", variables: "-t architecture:armhf" } + - { name: "debian (bookworm amd64)", case: "debian", variables: "-t architecture:amd64 -t suite:bookworm" } + - { name: "debian (bookworm arm64)", case: "debian", variables: "-t architecture:arm64 -t suite:bookworm" } + - { name: "debian (bookworm armhf)", case: "debian", variables: "-t architecture:armhf -t suite:bookworm" } + - { name: "debian (trixie amd64)", case: "debian", variables: "-t architecture:amd64 -t suite:trixie" } + - { name: "debian (trixie arm64)", case: "debian", variables: "-t architecture:arm64 -t suite:trixie" } + - { name: "debian (trixie armhf)", case: "debian", variables: "-t architecture:armhf -t suite:trixie" } exclude: - backend: nofakemachine test: { name: "partitioning", case: "partitioning" } diff --git a/tests/debian/test.yaml b/tests/debian/test.yaml index bf4adb2a..d53316a2 100644 --- a/tests/debian/test.yaml +++ b/tests/debian/test.yaml @@ -1,10 +1,12 @@ --- -{{- $architecture := or .architecture "amd64"}} -architecture: {{$architecture}} +{{- $architecture := or .architecture "amd64" }} +{{- $suite := or .suite "bookworm" }} + +architecture: {{ $architecture }} actions: - action: debootstrap - suite: bullseye + suite: {{ $suite }} variant: minbase merged-usr: true