Dependency improvements, change archive strategy, move stdlib to separate repo, etc

Makefile

@@ -18,7 +18,7 @@ install:
 
 gotestsum: $(gopath)/bin/gotestsum
 $(gopath)/bin/gotestsum:
-	go get gotest.tools/gotestsum
+	go install gotest.tools/gotestsum@latest
 	go mod tidy
 
 build: install

bramble.lock

@@ -1,14 +1,14 @@
 [URLHashes]
-  "basic_fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf"
-  "basic_fetch_url https://brmbl.s3.amazonaws.com/url_fetcher.tar.gz" = "p2vbvabkdqckjlm43rf7bfccdseizych"
-  "fetch_git https://github.com/maxmcd/bramble.git@v0" = "ebzl4qxuufyxuejtyujh7mdul4spjcgk"
-  "fetch_url http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz" = "omtz2dd5irsgvbg7eeksovceutw5abhb"
-  "fetch_url http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" = "cspu7ndkuuq2f7t5vvgsmvdt3xnoypqb"
-  "fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf"
-  "fetch_url https://brmbl.s3.amazonaws.com/ca-certificates.crt" = "qucryjlyakz2x2asktkm6dtzpx4qz5rj"
-  "fetch_url https://brmbl.s3.amazonaws.com/file-links.tar.gz" = "rdk4ljksk75wjx7fflcv5vyc7a4dgbbt"
-  "fetch_url https://github.com/NixOS/patchelf/releases/download/0.13/patchelf-0.13.tar.bz2" = "qffzn3vcrs5tu4lb5vtm6eaplwfjgtth"
-  "fetch_url https://github.com/denoland/deno/releases/download/v1.14.0/deno-x86_64-unknown-linux-gnu.zip" = "k5fftqwt46izovwwttzh5ebmjiccxuhz"
-  "fetch_url https://golang.org/dl/go1.17.2.linux-amd64.tar.gz" = "fm3ujt23vqou74mr5fz5dyygzisllhnq"
-  "fetch_url https://maxmcd.com/" = "y6yjtwbeq54vudyqh573msqquk67lbd6"
-  "fetch_url https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.946+6237dc0ab.tar.xz" = "3yhnkjln5ctjemvbvyf7uiff4523lway"
+"basic_fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf"
+"basic_fetch_url https://brmbl.s3.amazonaws.com/url_fetcher.tar.gz" = "p2vbvabkdqckjlm43rf7bfccdseizych"
+"fetch_git https://github.com/maxmcd/bramble.git@v0" = "ebzl4qxuufyxuejtyujh7mdul4spjcgk"
+"fetch_url http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz" = "omtz2dd5irsgvbg7eeksovceutw5abhb"
+"fetch_url http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" = "cspu7ndkuuq2f7t5vvgsmvdt3xnoypqb"
+"fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf"
+"fetch_url https://brmbl.s3.amazonaws.com/ca-certificates.crt" = "qucryjlyakz2x2asktkm6dtzpx4qz5rj"
+"fetch_url https://brmbl.s3.amazonaws.com/file-links.tar.gz" = "rdk4ljksk75wjx7fflcv5vyc7a4dgbbt"
+"fetch_url https://github.com/NixOS/patchelf/releases/download/0.13/patchelf-0.13.tar.bz2" = "qffzn3vcrs5tu4lb5vtm6eaplwfjgtth"
+"fetch_url https://github.com/denoland/deno/releases/download/v1.14.0/deno-x86_64-unknown-linux-gnu.zip" = "k5fftqwt46izovwwttzh5ebmjiccxuhz"
+"fetch_url https://golang.org/dl/go1.17.2.linux-amd64.tar.gz" = "fm3ujt23vqou74mr5fz5dyygzisllhnq"
+"fetch_url https://maxmcd.com/" = "y6yjtwbeq54vudyqh573msqquk67lbd6"
+"fetch_url https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.946+6237dc0ab.tar.xz" = "3yhnkjln5ctjemvbvyf7uiff4523lway"

bramble.toml

@@ -3,4 +3,5 @@ name = "github.com/maxmcd/bramble"
 version = "0.0.3"
 
 [dependencies]
+"github.com/brmbl/std" = "0.0.1"
 "github.com/maxmcd/busybox" = "0.0.2"

cmd/replacer/main.go

@@ -6,8 +6,8 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/maxmcd/bramble/pkg/reptar"
 	"github.com/maxmcd/bramble/pkg/textreplace"
+	"github.com/maxmcd/reptar"
 	"github.com/mholt/archiver/v3"
 )
 
@@ -28,7 +28,7 @@ func run() (err error) {
 	if err != nil {
 		return err
 	}
-	if err := reptar.Reptar(location, f); err != nil {
+	if err := reptar.Archive(location, f); err != nil {
 		return err
 	}
 	f.Seek(0, 0)

go.mod

@@ -10,14 +10,16 @@ require (
 	github.com/charmbracelet/lipgloss v0.4.0
 	github.com/containerd/console v1.0.3
 	github.com/julienschmidt/httprouter v1.3.0
+	github.com/klauspost/pgzip v1.2.5
 	github.com/maxmcd/dag v0.0.0-20210909010249-5757e2034a95
+	github.com/maxmcd/reptar v0.0.0-20220507012129-b7cb8d03dbe9
 	github.com/mholt/archiver/v3 v3.5.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/opencontainers/runc v1.0.3
 	github.com/pkg/errors v0.9.1
-	github.com/rhnvrm/simples3 v0.7.0
+	github.com/rlmcpherson/s3gof3r v0.5.1-0.20170210004045-864ae0bf7cf2
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.25.0
@@ -28,6 +30,7 @@ require (
 	go.starlark.net v0.0.0-20210901212718-87f333178d59
 	go.uber.org/zap v1.19.1
 	golang.org/x/mod v0.4.2
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac
 	gotest.tools/v3 v3.0.3 // indirect
 )

go.sum

@@ -89,8 +89,9 @@ github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
 github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
@@ -111,6 +112,8 @@ github.com/maxmcd/archiver/v3 v3.3.2-0.20210923004632-06ef4f8f175b h1:Z+y8sx28U7
 github.com/maxmcd/archiver/v3 v3.3.2-0.20210923004632-06ef4f8f175b/go.mod h1:2X2ALNbjYXnVYbkESAzRbpja4i0m4VNuFUQT7F69X/I=
 github.com/maxmcd/dag v0.0.0-20210909010249-5757e2034a95 h1:tBGx3z+FLew3L1BOuklkzm2UEIBcOZCgFa58VcJR9xk=
 github.com/maxmcd/dag v0.0.0-20210909010249-5757e2034a95/go.mod h1:pYGWUsNzYvkcdAOr2py4zGskjw/wdqSGUUwi8jwcikk=
+github.com/maxmcd/reptar v0.0.0-20220507012129-b7cb8d03dbe9 h1:0Pj51hoeko8PuOlkLdNvhLBClYTomOm76gZHCCpxvUY=
+github.com/maxmcd/reptar v0.0.0-20220507012129-b7cb8d03dbe9/go.mod h1:GPjLc3cTgkXFhW5v7bq6ejs6ELsj/ukIL4pR2bLW/Tc=
 github.com/maxmcd/starlark-go v0.0.0-20201021154825-b2f805d0d122 h1:5ntUQ6qQLi3wcHdxEVWud2Q5z11Pj6cQFA6OHbCvbRU=
 github.com/maxmcd/starlark-go v0.0.0-20201021154825-b2f805d0d122/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
@@ -148,11 +151,11 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rhnvrm/simples3 v0.7.0 h1:KSEuKw0eGC5vltLW8ChLvjko+aUr0HbGet+bZHdwfMo=
-github.com/rhnvrm/simples3 v0.7.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rlmcpherson/s3gof3r v0.5.1-0.20170210004045-864ae0bf7cf2 h1:O472FrMcTREIDyg5eBZhWDIQZit6GZ9gE6Ieq+k1Pbw=
+github.com/rlmcpherson/s3gof3r v0.5.1-0.20170210004045-864ae0bf7cf2/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8=
@@ -223,6 +226,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -248,6 +252,8 @@ golang.org/x/term v0.0.0-20210422114643-f5beecf764ed h1:Ei4bQjjpYUsS4efOUz+5Nz++
 golang.org/x/term v0.0.0-20210422114643-f5beecf764ed/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
+golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=

internal/command/bramble.go

@@ -25,6 +25,7 @@ func newBramble(wd string, bramblePath string) (b bramble, err error) {
 		dependency.NewManager(
 			filepath.Join(b.store.BramblePath, "var/dependencies"),
 			"https://store.bramble.run",
+			nil,
 		),
 	)
 	return b, nil

internal/command/build.go

@@ -87,7 +87,8 @@ func (b bramble) runBuild(ctx context.Context, output project.ExecModuleOutput,
 	err = output.WalkAndPatch(8, func(dep project.Dependency, drv project.Derivation) (addGraph *project.ExecModuleOutput, buildOutputs []project.BuildOutput, err error) {
 		select {
 		case <-ctx.Done():
-			return
+			fmt.Println("context cancelled")
+			return nil, nil, context.Canceled
 		default:
 		}
 		dependencies := []store.DerivationOutput{}
@@ -107,7 +108,7 @@ func (b bramble) runBuild(ctx context.Context, output project.ExecModuleOutput,
 		derivationDataLock.Unlock()
 
 		source, err := b.store.StoreLocalSources(ctx, store.SourceFiles{
-			ProjectLocation: b.project.Location(),
+			ProjectLocation: drv.Sources.ProjectLocation,
 			Location:        drv.Sources.Location,
 			Files:           drv.Sources.Files,
 		}) // TODO: delete this if the build fails?