Skip to content

Commit

Permalink
fix: build with go 1.23 by tracking modules.txt
Browse files Browse the repository at this point in the history
Fixes: #117

Since go 1.23, `vendor/modules.txt` is [required](golang/go@38ee0c7#diff-61fb6e44eac25bd4d6a8a64b3f38ee8a41faaefd1ef481170a011ecfc0f7c76bR344).

Hence, this is a first attempt to track `modules.txt` via gomod2nix.

Since, unfortunately, `module.txt` employs a bespoke custom format in
which
[varying](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/cmd/go/internal/modload/vendor.go;l=57)
[numbers](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/cmd/go/internal/modload/vendor.go;l=103)
of `#` hold meaning I deemed it more stable to not reverse engineer it.

Instead, we generate it when running `gomod2nix` by executing `go mod
vendor` and tracking its content in `gomod2nix.toml`.

Not the most elegant solution, but it works and enables us to use
gomod2nix with go v1.23.

Any comments appreached…
  • Loading branch information
obreitwi committed Nov 17, 2024
1 parent 5d38709 commit 928f240
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 12 deletions.
6 changes: 5 additions & 1 deletion builder/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,13 @@ let

sources = toJSON (filterAttrs (n: _: n != defaultPackage) sources);

vendorModules = modulesStruct.vendorModulesTxt or "";

passthru = {
inherit sources;
};

passAsFile = [ "json" "sources" ];
passAsFile = [ "json" "sources" "vendorModules" ];
}
(
''
Expand All @@ -122,6 +124,8 @@ let
${internal.symlink}
${concatStringsSep "\n" localReplaceCommands}
cp $vendorModulesPath vendor/modules.txt
mv vendor $out
''
);
Expand Down
3 changes: 2 additions & 1 deletion gomod2nix.toml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
schema = 3
schema = 4
vendorModulesTxt = "# github.com/BurntSushi/toml v1.3.2\n## explicit; go 1.16\ngithub.com/BurntSushi/toml\ngithub.com/BurntSushi/toml/internal\n# github.com/inconshreveable/mousetrap v1.1.0\n## explicit; go 1.18\ngithub.com/inconshreveable/mousetrap\n# github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47\n## explicit; go 1.15\ngithub.com/nix-community/go-nix/pkg/nar\ngithub.com/nix-community/go-nix/pkg/wire\n# github.com/sirupsen/logrus v1.9.3\n## explicit; go 1.13\ngithub.com/sirupsen/logrus\n# github.com/spf13/cobra v1.8.0\n## explicit; go 1.15\ngithub.com/spf13/cobra\n# github.com/spf13/pflag v1.0.5\n## explicit; go 1.12\ngithub.com/spf13/pflag\n# golang.org/x/mod v0.14.0\n## explicit; go 1.18\ngolang.org/x/mod/internal/lazyregexp\ngolang.org/x/mod/modfile\ngolang.org/x/mod/module\ngolang.org/x/mod/semver\n# golang.org/x/sys v0.14.0\n## explicit; go 1.18\ngolang.org/x/sys/execabs\ngolang.org/x/sys/unix\ngolang.org/x/sys/windows\n# golang.org/x/tools/go/vcs v0.1.0-deprecated\n## explicit; go 1.19\ngolang.org/x/tools/go/vcs\n"

[mod]
[mod."github.com/BurntSushi/toml"]
Expand Down
8 changes: 4 additions & 4 deletions internal/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ import (
"os"
"path/filepath"

log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
generate "github.com/nix-community/gomod2nix/internal/generate"
schema "github.com/nix-community/gomod2nix/internal/schema"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

const directoryDefault = "./"
Expand Down Expand Up @@ -62,7 +62,7 @@ func generateFunc(cmd *cobra.Command, args []string) {
{
goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
outFile := goMod2NixPath
pkgs, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs)
generated, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs)
if err != nil {
panic(fmt.Errorf("error generating pkgs: %v", err))
}
Expand All @@ -75,7 +75,7 @@ func generateFunc(cmd *cobra.Command, args []string) {
goPackagePath = tmpProj.GoPackagePath
}

output, err := schema.Marshal(pkgs, goPackagePath, subPackages)
output, err := schema.Marshal(generated, goPackagePath, subPackages)
if err != nil {
panic(fmt.Errorf("error marshaling output: %v", err))
}
Expand Down
38 changes: 36 additions & 2 deletions internal/generate/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ builtins.filterSource (name: type: baseNameOf name != ".DS_Store") (
return executor.Wait()
}

func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*schema.Package, error) {
func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) (*schema.GeneratePkgsResult, error) {
modDownloads, replace, err := common(directory)
if err != nil {
return nil, err
Expand Down Expand Up @@ -208,10 +208,44 @@ func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*sc
return nil, err
}

// Dependencies is downloaded, run `go mod vendor` to obtain `vendor/modules.txt` without reverse engineering
tmpVendorEnv := filepath.Join(directory, "vendor-gomod2nix")
err = os.RemoveAll(tmpVendorEnv)
if err != nil {
return nil, err
}
defer func() {
_ = os.RemoveAll(tmpVendorEnv)
}()

var modulesTxt string
{
log.Info("Obtaining modules.txt")
cmd := exec.Command(
"go", "mod", "vendor", "-o", tmpVendorEnv,
)
cmd.Dir = directory
err = cmd.Run()
if err != nil {
return nil, err
}
modulesTxtBytes, err := os.ReadFile(filepath.Join(tmpVendorEnv, "modules.txt"))
if err != nil {
return nil, err
}
if len(modulesTxtBytes) == 0 {
return nil, fmt.Errorf("modules.txt has no content")
}
modulesTxt = string(modulesTxtBytes)
}

sort.Slice(packages, func(i, j int) bool {
return packages[i].GoPackagePath < packages[j].GoPackagePath
})

return packages, nil
return &schema.GeneratePkgsResult{
Packages: packages,
ModulesTxt: string(modulesTxt),
}, nil

}
16 changes: 12 additions & 4 deletions internal/schema/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ package types

import (
"bytes"
"github.com/BurntSushi/toml"
"os"

"github.com/BurntSushi/toml"
)

const SchemaVersion = 3
const SchemaVersion = 4

type Package struct {
GoPackagePath string `toml:"-"`
Expand All @@ -18,6 +19,7 @@ type Package struct {
type Output struct {
SchemaVersion int `toml:"schema"`
Mod map[string]*Package `toml:"mod"`
VendorModules string `toml:"vendorModulesTxt"`

// Packages with passed import paths trigger `go install` based on this list
SubPackages []string `toml:"subPackages,omitempty"`
Expand All @@ -26,15 +28,21 @@ type Output struct {
GoPackagePath string `toml:"goPackagePath,omitempty"`
}

func Marshal(pkgs []*Package, goPackagePath string, subPackages []string) ([]byte, error) {
type GeneratePkgsResult struct {
Packages []*Package
ModulesTxt string
}

func Marshal(generated *GeneratePkgsResult, goPackagePath string, subPackages []string) ([]byte, error) {
out := &Output{
SchemaVersion: SchemaVersion,
Mod: make(map[string]*Package),
SubPackages: subPackages,
GoPackagePath: goPackagePath,
VendorModules: generated.ModulesTxt,
}

for _, pkg := range pkgs {
for _, pkg := range generated.Packages {
out.Mod[pkg.GoPackagePath] = pkg
}

Expand Down

0 comments on commit 928f240

Please sign in to comment.