Migrates to using go list -m -json all for godep tactic (pass 1) (#443

)

Changelog.md

@@ -1,5 +1,9 @@
 # Spectrometer Changelog
 
+## v2.19.9
+
+- Go: Fixes a regression, where deep dependencies were reported as direct dependencies. ([#443](https://github.com/fossas/spectrometer/pull/443/))
+
 ## v2.19.8
 
 - Perl: Adds support for Perl with parsing of `META.json`, `META.yml`, `MYMETA.yml`, `MYMETA.json`. ([#428](https://github.com/fossas/spectrometer/pull/428))

docs/references/strategies/languages/golang/gomodules.md

@@ -11,29 +11,30 @@ Find all files named `go.mod`
 
 Discovery: find go.mod files
 
-We run `go list -m all`, which produces, e.g.,:
+We run `go list -m -json all`, which produces, e.g.,:
 
+```json
+{
+        "Path": "example.com/foo/bar",
+        "Main": true,
+        "Dir": "/Users/example/Codes/golang/simple",
+        "GoMod": "/Users/example/Codes/golang/simple/go.mod",
+        "GoVersion": "1.16"
+}
+{
+        "Path": "github.com/kr/pretty",
+        "Version": "v0.1.0",
+        "Time": "2018-05-06T08:33:45Z",
+        "Indirect": true,
+        "Dir": "/Users/example/go/pkg/mod/github.com/kr/[email protected]",
+        "GoMod": "/Users/example/go/pkg/mod/cache/download/github.com/kr/pretty/@v/v0.1.0.mod"
+}
 ```
-github.com/skyrocknroll/go-mod-example
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf
-github.com/davecgh/go-spew v1.1.1
-github.com/gorilla/mux v1.6.2
-github.com/konsorten/go-windows-terminal-sequences v1.0.1
-github.com/pmezard/go-difflib v1.0.0
-github.com/sirupsen/logrus v1.2.0
-github.com/stretchr/objx v0.1.1
-github.com/stretchr/testify v1.2.2
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33
-gopkg.in/alecthomas/kingpin.v2 v2.2.6
-```
 
-where the first line references the current module, and the remaining lines are
-package imports and pinned version separated by a space. For package
-dependencies that aren't using gomodules, a pseudo-version
-(`v0.0.0-TIMESTAMP-COMMITID`) is present instead. We use the commit ID as the
-version.
+- To infer direct dependencies, we filter out any module, which has `Main` field with value of true, and `Indirect` field with value of true.
+- To infer transitive dependencies, we execute `go list -json all`, and parse it's output for `Imports`, `ImportPath`, `Module`, `Standard` data, and fill in the transitive dependencies.
+
+For package dependencies that aren't using gomodules, a pseudo-version (`v0.0.0-TIMESTAMP-COMMITID`) is present instead. We use the commit ID as the version.
 
 ## Analysis: gomod
 
@@ -54,3 +55,12 @@ where:
 
 - `replace` rewrites `require`s. In this example, our requires resolve to
   `[github.com/example/one v1.2.3, github.com/example/other v2.0.0]`
+
+
+## FAQ
+
+### Why `go list -m -json all` is used instead of `go list -json -deps` to infer dependencies?
+
+We use `go list -m -json all` in combination with the `go list -json all`, to infer direct and transitive dependencies. The reason, we do not use solely use `go list -json -deps` command at this moment, is because it does not include transitive dependencies imported with test imports. 
+
+This go module functionality is actively being worked on, such that we can label dependencies environment (e.g. Test) correctly, for all types of golang project configurations.

src/Strategy/Go/GoList.hs

@@ -5,36 +5,67 @@ module Strategy.Go.GoList (
   Require (..),
 ) where
 
-import Control.Effect.Diagnostics
-import Data.ByteString.Lazy qualified as BL
+import Control.Effect.Diagnostics hiding (fromMaybe)
+import Data.Aeson (FromJSON, withObject, (.!=), (.:), (.:?))
+import Data.Aeson.Internal (formatError)
+import Data.Aeson.Types (parseJSON)
 import Data.Foldable (traverse_)
-import Data.Maybe (mapMaybe)
-import Data.String.Conversion (decodeUtf8)
+import Data.Maybe (fromMaybe)
+import Data.String.Conversion (toText)
 import Data.Text (Text)
-import Data.Text qualified as Text
-import DepTypes
-import Effect.Exec
-import Effect.Grapher
+import DepTypes (Dependency)
+import Effect.Exec (
+  AllowErr (Never),
+  Command (..),
+  Exec,
+  ExecErr (CommandParseError),
+  execThrow,
+ )
+import Effect.Grapher (deep, direct, label)
 import Graphing (Graphing)
 import Path
-import Strategy.Go.Transitive (fillInTransitive)
-import Strategy.Go.Types
+import Strategy.Go.Transitive (decodeMany, fillInTransitive)
+import Strategy.Go.Types (
+  GolangGrapher,
+  graphingGolang,
+  mkGolangPackage,
+  mkGolangVersion,
+ )
 import Types (GraphBreadth (..))
 
 data Require = Require
   { reqPackage :: Text
   , reqVersion :: Text
+  , isDirect :: Bool
   }
   deriving (Eq, Ord, Show)
 
-golistCmd :: Command
-golistCmd =
+goListJsonCmd :: Command
+goListJsonCmd =
   Command
     { cmdName = "go"
-    , cmdArgs = ["list", "-m", "all"]
+    , cmdArgs = ["list", "-m", "-json", "all"]
     , cmdAllowErr = Never
     }
+data GoListModule = GoListModule
+  { path :: Text
+  , version :: Maybe Text
+  , isMain :: Bool
+  , isIndirect :: Bool
+  }
+  deriving (Show, Eq, Ord)
+
+instance FromJSON GoListModule where
+  parseJSON = withObject "GoListModule" $ \obj ->
+    GoListModule <$> obj .: "Path"
+      <*> obj .:? "Version"
+      <*> (obj .:? "Main" .!= False)
+      <*> (obj .:? "Indirect" .!= False)
 
+-- | Analyze using `go list`, and build dependency graph.
+--
+-- Since, sometimes go list directive includes test transitive dependencies in the listing
+-- We may include test transitive dependencies as deep dependencies on the graph without any edges.
 analyze' ::
   ( Has Exec sig m
   , Has Diagnostics sig m
@@ -43,28 +74,28 @@ analyze' ::
   m (Graphing Dependency, GraphBreadth)
 analyze' dir = do
   graph <- graphingGolang $ do
-    stdout <- context "Getting direct dependencies" $ execThrow dir golistCmd
-
-    let gomodLines = drop 1 . Text.lines . Text.filter (/= '\r') . decodeUtf8 . BL.toStrict $ stdout -- the first line is our package
-        requires = mapMaybe toRequire gomodLines
-
-        toRequire :: Text -> Maybe Require
-        toRequire line =
-          case Text.splitOn " " line of
-            [package, version] -> Just (Require package version)
-            _ -> Nothing
-
-    context "Adding direct dependencies" $ buildGraph requires
-
-    _ <- recover (fillInTransitive dir)
-    pure ()
+    stdout <- context ("Getting direct dependencies using, " <> toText (show goListJsonCmd)) $ execThrow dir goListJsonCmd
+    case decodeMany stdout of
+      Left (path, err) -> fatal (CommandParseError goListJsonCmd (toText (formatError path err)))
+      Right (mods :: [GoListModule]) -> do
+        context "Adding direct dependencies" $ buildGraph (toRequires mods)
+        _ <- recover (fillInTransitive dir)
+        pure ()
   pure (graph, Complete)
+  where
+    toRequires :: [GoListModule] -> [Require]
+    toRequires src = map (\m -> Require (path m) (fromMaybe "LATEST" $ version m) (not $ isIndirect m)) (withoutMain src)
+
+    withoutMain :: [GoListModule] -> [GoListModule]
+    withoutMain = filter (not . isMain)
 
 buildGraph :: Has GolangGrapher sig m => [Require] -> m ()
 buildGraph = traverse_ go
   where
     go :: Has GolangGrapher sig m => Require -> m ()
     go Require{..} = do
       let pkg = mkGolangPackage reqPackage
-      direct pkg
+      if isDirect
+        then direct pkg
+        else deep pkg
       label pkg (mkGolangVersion reqVersion)

src/Strategy/Go/Transitive.hs

@@ -4,6 +4,7 @@ module Strategy.Go.Transitive (
   normalizeImportsToModules,
   Module (..),
   Package (..),
+  decodeMany,
 ) where
 
 import Control.Algebra

test/Go/GoListSpec.hs

@@ -15,7 +15,6 @@ import DepTypes
 import Effect.Exec
 import Effect.Grapher
 import Graphing (Graphing)
-import Graphing qualified
 import Path.IO (getCurrentDir)
 import Strategy.Go.GoList
 import Test.Hspec
@@ -31,26 +30,25 @@ expected = run . evalGrapher $ do
   direct $
     Dependency
       { dependencyType = GoType
-      , dependencyName = "github.com/pkg/one"
-      , dependencyVersion = Just (CEq "commithash")
+      , dependencyName = "gopkg.in/yaml.v3"
+      , dependencyVersion = Just (CEq "496545a6307b")
       , dependencyLocations = []
       , dependencyEnvironments = mempty
       , dependencyTags = Map.empty
       }
-  direct $
+  deep $
     Dependency
       { dependencyType = GoType
-      , dependencyName = "github.com/pkg/two"
-      , dependencyVersion = Just (CEq "v2.0.0")
+      , dependencyName = "gopkg.in/check.v1"
+      , dependencyVersion = Just (CEq "788fd7840127")
       , dependencyLocations = []
       , dependencyEnvironments = mempty
       , dependencyTags = Map.empty
       }
 
 spec :: Spec
 spec = do
-  outputTrivial <- runIO (BL.readFile "test/Go/testdata/golist-stdout.trivial")
-  outputComplex <- runIO (BL.readFile "test/Go/testdata/golist-stdout.complex")
+  outputTrivial <- runIO (BL.readFile "test/Go/testdata/golist-stdout")
   testdir <- runIO getCurrentDir
 
   describe "golist analyze" $ do
@@ -63,14 +61,3 @@ spec = do
       case result of
         Left err -> expectationFailure ("analyze failed: " <> show (renderFailureBundle err))
         Right (graph, _) -> graph `shouldBe` expected
-
-    it "can handle complex inputs" $ do
-      let result =
-            analyze' testdir
-              & runConstExec outputComplex
-              & runDiagnostics
-              & run
-
-      case result of
-        Left err -> fail $ "failed to build graph" <> show (renderFailureBundle err)
-        Right (graph, _) -> length (Graphing.directList graph) `shouldBe` 12

test/Go/testdata/golist-stdout

@@ -0,0 +1,22 @@
+{
+        "Path": "example.com/foo/bar",
+        "Main": true,
+        "Dir": "/Users/megh/Codes/golang/simple",
+        "GoMod": "/Users/megh/Codes/golang/simple/go.mod",
+        "GoVersion": "1.16"
+}
+{
+        "Path": "gopkg.in/check.v1",
+        "Version": "v1.0.0-20180628173108-788fd7840127",
+        "Time": "2018-06-28T17:31:08Z",
+        "Indirect": true,
+        "Dir": "/Users/megh/go/pkg/mod/gopkg.in/[email protected]",
+        "GoMod": "/Users/megh/go/pkg/mod/cache/download/gopkg.in/check.v1/@v/v1.0.0-20180628173108-788fd7840127.mod"
+}
+{
+        "Path": "gopkg.in/yaml.v3",
+        "Version": "v3.0.0-20210107192922-496545a6307b",
+        "Time": "2021-01-07T19:29:22Z",
+        "Dir": "/Users/megh/go/pkg/mod/gopkg.in/[email protected]",
+        "GoMod": "/Users/megh/go/pkg/mod/cache/download/gopkg.in/yaml.v3/@v/v3.0.0-20210107192922-496545a6307b.mod"
+}

test/Go/testdata/golist-stdout.gomod

@@ -0,0 +1,8 @@
+module example.com/foo/bar
+
+go 1.16
+
+require (
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+)