From 0ee7ea1977072ecd235393f1c34dfa4c7c37fe70 Mon Sep 17 00:00:00 2001 From: meghfossa <86321858+meghfossa@users.noreply.github.com> Date: Thu, 11 Nov 2021 14:16:34 -0700 Subject: [PATCH] Migrates to using `go list -m -json all` for godep tactic (pass 1) (#443) --- Changelog.md | 4 + .../strategies/languages/golang/gomodules.md | 50 ++++++----- src/Strategy/Go/GoList.hs | 89 +++++++++++++------ src/Strategy/Go/Transitive.hs | 1 + test/Go/GoListSpec.hs | 25 ++---- test/Go/testdata/golist-stdout | 22 +++++ test/Go/testdata/golist-stdout.complex | 13 --- test/Go/testdata/golist-stdout.gomod | 8 ++ test/Go/testdata/golist-stdout.trivial | 3 - 9 files changed, 131 insertions(+), 84 deletions(-) create mode 100644 test/Go/testdata/golist-stdout delete mode 100644 test/Go/testdata/golist-stdout.complex create mode 100644 test/Go/testdata/golist-stdout.gomod delete mode 100644 test/Go/testdata/golist-stdout.trivial diff --git a/Changelog.md b/Changelog.md index 7f249d011..2fd51daca 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # Spectrometer Changelog +## v2.19.9 + +- Go: Fixes a regression, where deep dependencies were reported as direct dependencies. ([#443](https://github.com/fossas/spectrometer/pull/443/)) + ## v2.19.8 - Perl: Adds support for Perl with parsing of `META.json`, `META.yml`, `MYMETA.yml`, `MYMETA.json`. ([#428](https://github.com/fossas/spectrometer/pull/428)) diff --git a/docs/references/strategies/languages/golang/gomodules.md b/docs/references/strategies/languages/golang/gomodules.md index 608b6142f..4599829a2 100644 --- a/docs/references/strategies/languages/golang/gomodules.md +++ b/docs/references/strategies/languages/golang/gomodules.md @@ -11,29 +11,30 @@ Find all files named `go.mod` Discovery: find go.mod files -We run `go list -m all`, which produces, e.g.,: +We run `go list -m -json all`, which produces, e.g.,: +```json +{ + "Path": "example.com/foo/bar", + "Main": true, + "Dir": "/Users/example/Codes/golang/simple", + "GoMod": "/Users/example/Codes/golang/simple/go.mod", + "GoVersion": "1.16" +} +{ + "Path": "github.com/kr/pretty", + "Version": "v0.1.0", + "Time": "2018-05-06T08:33:45Z", + "Indirect": true, + "Dir": "/Users/example/go/pkg/mod/github.com/kr/pretty@v0.1.0", + "GoMod": "/Users/example/go/pkg/mod/cache/download/github.com/kr/pretty/@v/v0.1.0.mod" +} ``` -github.com/skyrocknroll/go-mod-example -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf -github.com/davecgh/go-spew v1.1.1 -github.com/gorilla/mux v1.6.2 -github.com/konsorten/go-windows-terminal-sequences v1.0.1 -github.com/pmezard/go-difflib v1.0.0 -github.com/sirupsen/logrus v1.2.0 -github.com/stretchr/objx v0.1.1 -github.com/stretchr/testify v1.2.2 -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 -gopkg.in/alecthomas/kingpin.v2 v2.2.6 -``` -where the first line references the current module, and the remaining lines are -package imports and pinned version separated by a space. For package -dependencies that aren't using gomodules, a pseudo-version -(`v0.0.0-TIMESTAMP-COMMITID`) is present instead. We use the commit ID as the -version. +- To infer direct dependencies, we filter out any module, which has `Main` field with value of true, and `Indirect` field with value of true. +- To infer transitive dependencies, we execute `go list -json all`, and parse it's output for `Imports`, `ImportPath`, `Module`, `Standard` data, and fill in the transitive dependencies. + +For package dependencies that aren't using gomodules, a pseudo-version (`v0.0.0-TIMESTAMP-COMMITID`) is present instead. We use the commit ID as the version. ## Analysis: gomod @@ -54,3 +55,12 @@ where: - `replace` rewrites `require`s. In this example, our requires resolve to `[github.com/example/one v1.2.3, github.com/example/other v2.0.0]` + + +## FAQ + +### Why `go list -m -json all` is used instead of `go list -json -deps` to infer dependencies? + +We use `go list -m -json all` in combination with the `go list -json all`, to infer direct and transitive dependencies. The reason, we do not use solely use `go list -json -deps` command at this moment, is because it does not include transitive dependencies imported with test imports. + +This go module functionality is actively being worked on, such that we can label dependencies environment (e.g. Test) correctly, for all types of golang project configurations. \ No newline at end of file diff --git a/src/Strategy/Go/GoList.hs b/src/Strategy/Go/GoList.hs index c91b23fbe..f5c155444 100644 --- a/src/Strategy/Go/GoList.hs +++ b/src/Strategy/Go/GoList.hs @@ -5,36 +5,67 @@ module Strategy.Go.GoList ( Require (..), ) where -import Control.Effect.Diagnostics -import Data.ByteString.Lazy qualified as BL +import Control.Effect.Diagnostics hiding (fromMaybe) +import Data.Aeson (FromJSON, withObject, (.!=), (.:), (.:?)) +import Data.Aeson.Internal (formatError) +import Data.Aeson.Types (parseJSON) import Data.Foldable (traverse_) -import Data.Maybe (mapMaybe) -import Data.String.Conversion (decodeUtf8) +import Data.Maybe (fromMaybe) +import Data.String.Conversion (toText) import Data.Text (Text) -import Data.Text qualified as Text -import DepTypes -import Effect.Exec -import Effect.Grapher +import DepTypes (Dependency) +import Effect.Exec ( + AllowErr (Never), + Command (..), + Exec, + ExecErr (CommandParseError), + execThrow, + ) +import Effect.Grapher (deep, direct, label) import Graphing (Graphing) import Path -import Strategy.Go.Transitive (fillInTransitive) -import Strategy.Go.Types +import Strategy.Go.Transitive (decodeMany, fillInTransitive) +import Strategy.Go.Types ( + GolangGrapher, + graphingGolang, + mkGolangPackage, + mkGolangVersion, + ) import Types (GraphBreadth (..)) data Require = Require { reqPackage :: Text , reqVersion :: Text + , isDirect :: Bool } deriving (Eq, Ord, Show) -golistCmd :: Command -golistCmd = +goListJsonCmd :: Command +goListJsonCmd = Command { cmdName = "go" - , cmdArgs = ["list", "-m", "all"] + , cmdArgs = ["list", "-m", "-json", "all"] , cmdAllowErr = Never } +data GoListModule = GoListModule + { path :: Text + , version :: Maybe Text + , isMain :: Bool + , isIndirect :: Bool + } + deriving (Show, Eq, Ord) + +instance FromJSON GoListModule where + parseJSON = withObject "GoListModule" $ \obj -> + GoListModule <$> obj .: "Path" + <*> obj .:? "Version" + <*> (obj .:? "Main" .!= False) + <*> (obj .:? "Indirect" .!= False) +-- | Analyze using `go list`, and build dependency graph. +-- +-- Since, sometimes go list directive includes test transitive dependencies in the listing +-- We may include test transitive dependencies as deep dependencies on the graph without any edges. analyze' :: ( Has Exec sig m , Has Diagnostics sig m @@ -43,22 +74,20 @@ analyze' :: m (Graphing Dependency, GraphBreadth) analyze' dir = do graph <- graphingGolang $ do - stdout <- context "Getting direct dependencies" $ execThrow dir golistCmd - - let gomodLines = drop 1 . Text.lines . Text.filter (/= '\r') . decodeUtf8 . BL.toStrict $ stdout -- the first line is our package - requires = mapMaybe toRequire gomodLines - - toRequire :: Text -> Maybe Require - toRequire line = - case Text.splitOn " " line of - [package, version] -> Just (Require package version) - _ -> Nothing - - context "Adding direct dependencies" $ buildGraph requires - - _ <- recover (fillInTransitive dir) - pure () + stdout <- context ("Getting direct dependencies using, " <> toText (show goListJsonCmd)) $ execThrow dir goListJsonCmd + case decodeMany stdout of + Left (path, err) -> fatal (CommandParseError goListJsonCmd (toText (formatError path err))) + Right (mods :: [GoListModule]) -> do + context "Adding direct dependencies" $ buildGraph (toRequires mods) + _ <- recover (fillInTransitive dir) + pure () pure (graph, Complete) + where + toRequires :: [GoListModule] -> [Require] + toRequires src = map (\m -> Require (path m) (fromMaybe "LATEST" $ version m) (not $ isIndirect m)) (withoutMain src) + + withoutMain :: [GoListModule] -> [GoListModule] + withoutMain = filter (not . isMain) buildGraph :: Has GolangGrapher sig m => [Require] -> m () buildGraph = traverse_ go @@ -66,5 +95,7 @@ buildGraph = traverse_ go go :: Has GolangGrapher sig m => Require -> m () go Require{..} = do let pkg = mkGolangPackage reqPackage - direct pkg + if isDirect + then direct pkg + else deep pkg label pkg (mkGolangVersion reqVersion) diff --git a/src/Strategy/Go/Transitive.hs b/src/Strategy/Go/Transitive.hs index 13ab4ccf7..48bfdecaf 100644 --- a/src/Strategy/Go/Transitive.hs +++ b/src/Strategy/Go/Transitive.hs @@ -4,6 +4,7 @@ module Strategy.Go.Transitive ( normalizeImportsToModules, Module (..), Package (..), + decodeMany, ) where import Control.Algebra diff --git a/test/Go/GoListSpec.hs b/test/Go/GoListSpec.hs index 9f6f46080..f2c68a0d5 100644 --- a/test/Go/GoListSpec.hs +++ b/test/Go/GoListSpec.hs @@ -15,7 +15,6 @@ import DepTypes import Effect.Exec import Effect.Grapher import Graphing (Graphing) -import Graphing qualified import Path.IO (getCurrentDir) import Strategy.Go.GoList import Test.Hspec @@ -31,17 +30,17 @@ expected = run . evalGrapher $ do direct $ Dependency { dependencyType = GoType - , dependencyName = "github.com/pkg/one" - , dependencyVersion = Just (CEq "commithash") + , dependencyName = "gopkg.in/yaml.v3" + , dependencyVersion = Just (CEq "496545a6307b") , dependencyLocations = [] , dependencyEnvironments = mempty , dependencyTags = Map.empty } - direct $ + deep $ Dependency { dependencyType = GoType - , dependencyName = "github.com/pkg/two" - , dependencyVersion = Just (CEq "v2.0.0") + , dependencyName = "gopkg.in/check.v1" + , dependencyVersion = Just (CEq "788fd7840127") , dependencyLocations = [] , dependencyEnvironments = mempty , dependencyTags = Map.empty @@ -49,8 +48,7 @@ expected = run . evalGrapher $ do spec :: Spec spec = do - outputTrivial <- runIO (BL.readFile "test/Go/testdata/golist-stdout.trivial") - outputComplex <- runIO (BL.readFile "test/Go/testdata/golist-stdout.complex") + outputTrivial <- runIO (BL.readFile "test/Go/testdata/golist-stdout") testdir <- runIO getCurrentDir describe "golist analyze" $ do @@ -63,14 +61,3 @@ spec = do case result of Left err -> expectationFailure ("analyze failed: " <> show (renderFailureBundle err)) Right (graph, _) -> graph `shouldBe` expected - - it "can handle complex inputs" $ do - let result = - analyze' testdir - & runConstExec outputComplex - & runDiagnostics - & run - - case result of - Left err -> fail $ "failed to build graph" <> show (renderFailureBundle err) - Right (graph, _) -> length (Graphing.directList graph) `shouldBe` 12 diff --git a/test/Go/testdata/golist-stdout b/test/Go/testdata/golist-stdout new file mode 100644 index 000000000..ce5e99d62 --- /dev/null +++ b/test/Go/testdata/golist-stdout @@ -0,0 +1,22 @@ +{ + "Path": "example.com/foo/bar", + "Main": true, + "Dir": "/Users/megh/Codes/golang/simple", + "GoMod": "/Users/megh/Codes/golang/simple/go.mod", + "GoVersion": "1.16" +} +{ + "Path": "gopkg.in/check.v1", + "Version": "v1.0.0-20180628173108-788fd7840127", + "Time": "2018-06-28T17:31:08Z", + "Indirect": true, + "Dir": "/Users/megh/go/pkg/mod/gopkg.in/check.v1@v1.0.0-20180628173108-788fd7840127", + "GoMod": "/Users/megh/go/pkg/mod/cache/download/gopkg.in/check.v1/@v/v1.0.0-20180628173108-788fd7840127.mod" +} +{ + "Path": "gopkg.in/yaml.v3", + "Version": "v3.0.0-20210107192922-496545a6307b", + "Time": "2021-01-07T19:29:22Z", + "Dir": "/Users/megh/go/pkg/mod/gopkg.in/yaml.v3@v3.0.0-20210107192922-496545a6307b", + "GoMod": "/Users/megh/go/pkg/mod/cache/download/gopkg.in/yaml.v3/@v/v3.0.0-20210107192922-496545a6307b.mod" +} \ No newline at end of file diff --git a/test/Go/testdata/golist-stdout.complex b/test/Go/testdata/golist-stdout.complex deleted file mode 100644 index 03de28444..000000000 --- a/test/Go/testdata/golist-stdout.complex +++ /dev/null @@ -1,13 +0,0 @@ -github.com/skyrocknroll/go-mod-example -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf -github.com/davecgh/go-spew v1.1.1 -github.com/gorilla/mux v1.6.2 -github.com/konsorten/go-windows-terminal-sequences v1.0.1 -github.com/pmezard/go-difflib v1.0.0 -github.com/sirupsen/logrus v1.2.0 -github.com/stretchr/objx v0.1.1 -github.com/stretchr/testify v1.2.2 -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 -gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/test/Go/testdata/golist-stdout.gomod b/test/Go/testdata/golist-stdout.gomod new file mode 100644 index 000000000..543387d0f --- /dev/null +++ b/test/Go/testdata/golist-stdout.gomod @@ -0,0 +1,8 @@ +module example.com/foo/bar + +go 1.16 + +require ( + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b +) \ No newline at end of file diff --git a/test/Go/testdata/golist-stdout.trivial b/test/Go/testdata/golist-stdout.trivial deleted file mode 100644 index b29bed3cf..000000000 --- a/test/Go/testdata/golist-stdout.trivial +++ /dev/null @@ -1,3 +0,0 @@ -github.com/my/pkg -github.com/pkg/one v0.0.0-XXXXXXXXXXXX-commithash -github.com/pkg/two v2.0.0