From a9e5c5e2b0513d85c16982bfe50296f7235938df Mon Sep 17 00:00:00 2001 From: Jeffrey Huynh Date: Sun, 18 Aug 2024 11:18:35 -0700 Subject: [PATCH] Draft implementation of strict mode --- src/App/Fossa/Analyze.hs | 6 +- src/App/Fossa/Analyze/Discover.hs | 1 + src/App/Fossa/Analyze/Types.hs | 5 +- src/App/Fossa/Config/Analyze.hs | 6 ++ .../Fossa/Container/Sources/DockerArchive.hs | 9 +- src/App/Fossa/ListTargets.hs | 8 +- src/Strategy/Bundler.hs | 63 +++++++++++--- src/Strategy/Cocoapods.hs | 82 ++++++++++++------- src/Strategy/Conda.hs | 12 ++- src/Strategy/Godep.hs | 18 ++-- src/Strategy/Gomodules.hs | 13 ++- src/Strategy/Maven.hs | 65 +++++++++++++-- src/Strategy/Pub.hs | 70 +++++++++++----- src/Strategy/Scala.hs | 16 ++-- 14 files changed, 283 insertions(+), 91 deletions(-) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 6079feaf04..2c8be42d8f 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -45,6 +45,7 @@ import App.Fossa.Config.Analyze ( IncludeAll (IncludeAll), NoDiscoveryExclusion (NoDiscoveryExclusion), ScanDestination (..), + StrictMode (..), UnpackArchives (UnpackArchives), WithoutDefaultFilters (..), ) @@ -189,6 +190,7 @@ runDependencyAnalysis :: , Has Stack sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has (Reader OverrideDynamicAnalysisBinary) sig m , Has Telemetry sig m @@ -294,6 +296,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do shouldAnalyzePathDependencies = resolvePathDependencies $ Config.experimental cfg allowedTactics = Config.allowedTacticTypes cfg withoutDefaultFilters = Config.withoutDefaultFilters cfg + -- strictMode = Config.strictMode cfg manualSrcUnits <- Diag.errorBoundaryIO . diagToDebug $ @@ -361,7 +364,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do pure Nothing else Diag.context "first-party-scans" . runStickyLogger SevInfo $ runFirstPartyScan basedir maybeApiOpts cfg let firstPartyScanResults = join . resultToMaybe $ maybeFirstPartyScanResults - + -- logDebug $ "Is in strict mode ------------- " <> pretty (show strictMode) let discoveryFilters = if fromFlag NoDiscoveryExclusion noDiscoveryExclusion then mempty else filters (projectScans, ()) <- Diag.context "discovery/analysis tasks" @@ -372,6 +375,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do . runAtomicCounter . runReader (Config.experimental cfg) . runReader (Config.mavenScopeFilterSet cfg) + . runReader (Config.strictMode cfg) . runReader discoveryFilters . runReader (Config.overrideDynamicAnalysis cfg) $ do diff --git a/src/App/Fossa/Analyze/Discover.hs b/src/App/Fossa/Analyze/Discover.hs index 2def7d9daa..697dd6baaa 100644 --- a/src/App/Fossa/Analyze/Discover.hs +++ b/src/App/Fossa/Analyze/Discover.hs @@ -4,6 +4,7 @@ module App.Fossa.Analyze.Discover ( ) where import App.Fossa.Analyze.Types (AnalyzeProject, DiscoverTaskEffs) +import App.Fossa.Config.Analyze (StrictMode) import Control.Effect.Reader (Has, Reader) import Data.Aeson qualified as Aeson import Discovery.Filters (AllFilters) diff --git a/src/App/Fossa/Analyze/Types.hs b/src/App/Fossa/Analyze/Types.hs index 54af27d13b..d21e56a579 100644 --- a/src/App/Fossa/Analyze/Types.hs +++ b/src/App/Fossa/Analyze/Types.hs @@ -11,7 +11,7 @@ module App.Fossa.Analyze.Types ( ) where import App.Fossa.Analyze.Project (ProjectResult) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig, StrictMode (..)) import App.Fossa.Lernie.Types (LernieResults) import App.Fossa.Reachability.Types (SourceUnitReachability (..)) import Control.Effect.Debug (Debug) @@ -19,6 +19,7 @@ import Control.Effect.Diagnostics (Diagnostics, Has) import Control.Effect.Lift (Lift) import Control.Effect.Reader (Reader) import Control.Effect.Telemetry (Telemetry) +import Data.Flag (Flag) import Data.Set (Set) import Data.Text (Text) import Diag.Result (Result (Failure, Success)) @@ -44,6 +45,7 @@ type DiscoverTaskEffs sig m = , Has Debug sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) @@ -69,6 +71,7 @@ type AnalyzeStaticTaskEffs sig m = , Has Debug sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) diff --git a/src/App/Fossa/Config/Analyze.hs b/src/App/Fossa/Config/Analyze.hs index 9a7584715c..6c3e4251da 100644 --- a/src/App/Fossa/Config/Analyze.hs +++ b/src/App/Fossa/Config/Analyze.hs @@ -23,6 +23,7 @@ module App.Fossa.Config.Analyze ( GoDynamicTactic (..), StaticOnlyTactics (..), WithoutDefaultFilters (..), + StrictMode (..), mkSubCommand, loadConfig, cliParser, @@ -135,6 +136,7 @@ data ForceFirstPartyScans = ForceFirstPartyScans deriving (Generic) data ForceNoFirstPartyScans = ForceNoFirstPartyScans deriving (Generic) data IgnoreOrgWideCustomLicenseScanConfigs = IgnoreOrgWideCustomLicenseScanConfigs deriving (Generic) data StaticOnlyTactics = StaticOnlyTactics deriving (Generic) +data StrictMode = StrictMode deriving (Generic, Show) data BinaryDiscovery = BinaryDiscovery deriving (Generic) data IncludeAll = IncludeAll deriving (Generic) @@ -226,6 +228,7 @@ data AnalyzeCliOpts = AnalyzeCliOpts , analyzeCustomFossaDepsFile :: Maybe FilePath , analyzeStaticOnlyTactics :: Flag StaticOnlyTactics , analyzeWithoutDefaultFilters :: Flag WithoutDefaultFilters + , analyzeStrictMode :: Flag StrictMode } deriving (Eq, Ord, Show) @@ -264,6 +267,7 @@ data AnalyzeConfig = AnalyzeConfig , allowedTacticTypes :: AnalysisTacticTypes , reachabilityConfig :: ReachabilityConfig , withoutDefaultFilters :: Flag WithoutDefaultFilters + , strictMode :: Flag StrictMode } deriving (Eq, Ord, Show, Generic) @@ -333,6 +337,7 @@ cliParser = <*> optional (strOption (applyFossaStyle <> long "fossa-deps-file" <> helpDoc fossaDepsFileHelp <> metavar "FILEPATH")) <*> flagOpt StaticOnlyTactics (applyFossaStyle <> long "static-only-analysis" <> stringToHelpDoc "Only analyze the project using static strategies.") <*> withoutDefaultFilterParser fossaAnalyzeDefaultFilterDocUrl + <*> flagOpt StrictMode (applyFossaStyle <> long "strict" <> stringToHelpDoc "Strict mode") where fossaDepsFileHelp :: Maybe (Doc AnsiStyle) fossaDepsFileHelp = @@ -541,6 +546,7 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do <*> pure allowedTacticType <*> resolveReachabilityOptions reachabilityConfig <*> pure analyzeWithoutDefaultFilters + <*> pure analyzeStrictMode collectMavenScopeFilters :: ( Has Diagnostics sig m diff --git a/src/App/Fossa/Container/Sources/DockerArchive.hs b/src/App/Fossa/Container/Sources/DockerArchive.hs index 1de4ab80bb..ab43fdbcde 100644 --- a/src/App/Fossa/Container/Sources/DockerArchive.hs +++ b/src/App/Fossa/Container/Sources/DockerArchive.hs @@ -16,7 +16,7 @@ import App.Fossa.Analyze.Types ( DiscoveredProjectIdentifier (..), DiscoveredProjectScan (..), ) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (ExperimentalAnalyzeConfig), GoDynamicTactic (GoModulesBasedTactic), WithoutDefaultFilters (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (ExperimentalAnalyzeConfig), GoDynamicTactic (GoModulesBasedTactic), StrictMode (..), WithoutDefaultFilters (..)) import App.Fossa.Container.Sources.Discovery (layerAnalyzers, renderLayerTarget) import App.Fossa.Container.Sources.JarAnalysis (analyzeContainerJars) import Codec.Archive.Tar.Index (TarEntryOffset) @@ -59,7 +59,7 @@ import Control.Monad (join, void, when) import Data.Bifunctor (bimap) import Data.ByteString.Lazy qualified as BS import Data.FileTree.IndexFileTree (SomeFileTree, fixedVfsRoot) -import Data.Flag (Flag, fromFlag) +import Data.Flag (Flag, fromFlag, toFlag) import Data.Foldable (traverse_) import Data.Map qualified as Map import Data.Maybe (isNothing, listToMaybe, mapMaybe) @@ -202,6 +202,7 @@ analyzeLayer systemDepsOnly filters withoutDefaultFilters capabilities osInfo la runTarballReadFSIO layerFs tarball . runReader noExperimental . runReader noMavenScopeFilters + . runReader testFlag . Diag.context "discovery/analysis tasks" . runOutput @DiscoveredProjectScan . runStickyLogger SevInfo @@ -213,6 +214,8 @@ analyzeLayer systemDepsOnly filters withoutDefaultFilters capabilities osInfo la pure projectResults ) where + testFlag :: Flag StrictMode + testFlag = toFlag StrictMode False noMavenScopeFilters :: MavenScopeFilters noMavenScopeFilters = MavenScopeIncludeFilters mempty noExperimental :: ExperimentalAnalyzeConfig @@ -256,6 +259,7 @@ runDependencyAnalysis :: , Has (Output DiscoveredProjectScan) sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Stack sig m , Has Telemetry sig m @@ -373,6 +377,7 @@ listTargetLayer capabilities osInfo layerFs tarball layerType = do False -- Targets are not impacted by path dependencies. ) . runReader (MavenScopeIncludeFilters mempty) + . runReader (toFlag StrictMode False) . runReader (mempty :: AllFilters) $ run where diff --git a/src/App/Fossa/ListTargets.hs b/src/App/Fossa/ListTargets.hs index 23c6fcf0c4..35ba7cdfe3 100644 --- a/src/App/Fossa/ListTargets.hs +++ b/src/App/Fossa/ListTargets.hs @@ -6,7 +6,7 @@ module App.Fossa.ListTargets ( ) where import App.Fossa.Analyze.Discover (DiscoverFunc (DiscoverFunc), discoverFuncs) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig, StrictMode (..)) import App.Fossa.Config.ListTargets ( ListTargetOutputFormat (..), ListTargetsCliOpts, @@ -36,6 +36,7 @@ import Control.Effect.Stack (Stack) import Control.Effect.Telemetry (Telemetry) import Data.Aeson (ToJSON, encode, object, (.=)) import Data.Aeson.Extra (encodeJSONToText) +import Data.Flag (Flag (..), toFlag) import Data.Foldable (for_, traverse_) import Data.Set.NonEmpty (toSet) import Data.String.Conversion (decodeUtf8, toText) @@ -83,10 +84,14 @@ listTargetsMain ListTargetsConfig{..} = do . runReader experimental -- `fossa list-targets` does not support maven scope filters. . runReader (MavenScopeIncludeFilters mempty) + . runReader (testFlag) -- The current version of `fossa list-targets` does not support filters. -- TODO: support both discovery and post-discovery filtering. . runReader (mempty :: AllFilters) $ runAll listTargetOutputFormat (unBaseDir baseDir) + where + testFlag :: Flag StrictMode + testFlag = toFlag StrictMode False runAll :: ( Has ReadFS sig m @@ -99,6 +104,7 @@ runAll :: , Has Stack sig m , Has (Reader ExperimentalAnalyzeConfig) sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m , Has (Reader AllFilters) sig m , Has Telemetry sig m ) => diff --git a/src/Strategy/Bundler.hs b/src/Strategy/Bundler.hs index af7279b858..a02af21f82 100644 --- a/src/Strategy/Bundler.hs +++ b/src/Strategy/Bundler.hs @@ -11,6 +11,7 @@ import App.Fossa.Analyze.LicenseAnalyze ( LicenseAnalyzeProject (licenseAnalyzeProject), ) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Effect.Diagnostics ( Diagnostics, context, @@ -21,8 +22,9 @@ import Control.Effect.Diagnostics ( (<||>), ) import Control.Effect.Diagnostics qualified as Diag -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Glob as Glob (toGlob, ()) import Data.Text (isSuffixOf) import Diag.Common (AllDirectDeps (AllDirectDeps), MissingEdges (MissingEdges)) @@ -35,6 +37,7 @@ import Discovery.Walk ( walkWithFilters', ) import Effect.Exec (Exec, Has) +import Effect.Logger (Logger, logDebug) import Effect.ReadFS (ReadFS, readContentsParser) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path, toFilePath) @@ -119,8 +122,16 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => BundlerProject -> m DependencyResults -getDeps project = analyzeGemfileLock project <||> context "Bundler" (analyzeBundleShow project) +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m, Has Logger sig m) => BundlerProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then do + logDebug "Ruby strict mode" + analyzeGemfileLock project + else do + logDebug "Ruby NOT IN STRICT MODE ===========" + analyzeGemfileLock project <||> context "Bundler" (analyzeBundleShow project) analyzeBundleShow :: (Has Exec sig m, Has Diagnostics sig m) => BundlerProject -> m DependencyResults analyzeBundleShow project = do @@ -132,15 +143,11 @@ analyzeBundleShow project = do , dependencyManifestFiles = maybe [bundlerGemfile project] pure (bundlerGemfileLock project) } -analyzeGemfileLock :: (Has ReadFS sig m, Has Diagnostics sig m) => BundlerProject -> m DependencyResults -analyzeGemfileLock project = - warnOnErr AllDirectDeps - . warnOnErr MissingEdges - . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) - . errHelp BundlerMissingLockFileHelp - . errDoc bundlerLockFileRationaleUrl - . errDoc rubyFossaDocUrl - $ do +analyzeGemfileLock :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => BundlerProject -> m DependencyResults +analyzeGemfileLock project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then do lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile pure $ @@ -149,3 +156,35 @@ analyzeGemfileLock project = , dependencyGraphBreadth = Complete , dependencyManifestFiles = [lockFile] } + else do + warnOnErr AllDirectDeps + . warnOnErr MissingEdges + . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) + . errHelp BundlerMissingLockFileHelp + . errDoc bundlerLockFileRationaleUrl + . errDoc rubyFossaDocUrl + $ do + lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) + graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile + pure $ + DependencyResults + { dependencyGraph = graph + , dependencyGraphBreadth = Complete + , dependencyManifestFiles = [lockFile] + } + +-- warnOnErr AllDirectDeps +-- . warnOnErr MissingEdges +-- . errCtx (BundlerMissingLockFileCtx $ bundlerGemfile project) +-- . errHelp BundlerMissingLockFileHelp +-- . errDoc bundlerLockFileRationaleUrl +-- . errDoc rubyFossaDocUrl +-- $ do +-- lockFile <- context "Retrieve Gemfile.lock" (Diag.fromMaybeText "No Gemfile.lock present in the project" (bundlerGemfileLock project)) +-- graph <- context "Gemfile.lock analysis" . GemfileLock.analyze' $ lockFile +-- pure $ +-- DependencyResults +-- { dependencyGraph = graph +-- , dependencyGraphBreadth = Complete +-- , dependencyManifestFiles = [lockFile] +-- } diff --git a/src/Strategy/Cocoapods.hs b/src/Strategy/Cocoapods.hs index 58df4ef8d9..85efc6117d 100644 --- a/src/Strategy/Cocoapods.hs +++ b/src/Strategy/Cocoapods.hs @@ -8,12 +8,14 @@ module Strategy.Cocoapods ( import App.Fossa.Analyze.LicenseAnalyze (LicenseAnalyzeProject, licenseAnalyzeProject) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Applicative ((<|>)) import Control.Carrier.Diagnostics (errHelp) import Control.Effect.Diagnostics (Diagnostics, context, errCtx, errDoc, warnOnErr, (<||>)) import Control.Effect.Diagnostics qualified as Diag -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Glob qualified as Glob import Data.List (find) import Data.List.Extra (singleton) @@ -110,33 +112,57 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Diagnostics sig m, Has Exec sig m, Has Logger sig m) => CocoapodsProject -> m DependencyResults -getDeps project = - context "Cocoapods" $ - context - "Podfile.lock analysis" - ( warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - . errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLock project) - ) - <||> context "Podfile analysis" (analyzePodfile project) - -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m) => CocoapodsProject -> m DependencyResults -getDeps' project = - context "Cocoapods" $ - context - "Podfile.lock analysis" - ( warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - . errCtx MissingPodLockFileCtx - . errHelp MissingPodLockFileHelp - . errDoc refPodDocUrl - $ (analyzePodfileLockStatically project) - ) - <||> context "Podfile analysis" (analyzePodfile project) +getDeps :: (Has ReadFS sig m, Has Diagnostics sig m, Has Exec sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => CocoapodsProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLock project) + ) + else + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( warnOnErr MissingEdges + . warnOnErr MissingDeepDeps + . errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLock project) + ) + <||> context "Podfile analysis" (analyzePodfile project) + +getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => CocoapodsProject -> m DependencyResults +getDeps' project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then + context "Cocoapods" $ + context + "Strict mode Podfile.lock analysis" + ( errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLockStatically project) + ) + else + context "Cocoapods" $ + context + "Podfile.lock analysis" + ( warnOnErr MissingEdges + . warnOnErr MissingDeepDeps + . errCtx MissingPodLockFileCtx + . errHelp MissingPodLockFileHelp + . errDoc refPodDocUrl + $ (analyzePodfileLockStatically project) + ) + <||> context "Podfile analysis" (analyzePodfile project) analyzePodfile :: (Has ReadFS sig m, Has Diagnostics sig m) => CocoapodsProject -> m DependencyResults analyzePodfile project = do diff --git a/src/Strategy/Conda.hs b/src/Strategy/Conda.hs index 7496deecd3..581564a1b1 100644 --- a/src/Strategy/Conda.hs +++ b/src/Strategy/Conda.hs @@ -5,14 +5,16 @@ module Strategy.Conda ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Effect.Diagnostics ( Diagnostics, Has, fatalText, (<||>), ) -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -73,8 +75,12 @@ mkProject project = -- There might be a dep with a version spec in an environment.yml file: i.e. conda+foo$1.2.*, and perhaps -- the same dep resolved to a known version in the users virtual environment: i.e. conda+'conda-forge':foo$1.2.4 (we get that from conda env create). -- If we combined the results then we would include both of those deps in the result, which is not correct behavior. -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => CondaProject -> m DependencyResults -getDeps project = analyzeCondaEnvCreate project <||> analyzeEnvironmentYml project +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => CondaProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then analyzeCondaEnvCreate project + else analyzeCondaEnvCreate project <||> analyzeEnvironmentYml project analyzeCondaEnvCreate :: (Has Exec sig m, Has Diagnostics sig m) => CondaProject -> m DependencyResults analyzeCondaEnvCreate CondaProject{..} = do diff --git a/src/Strategy/Godep.hs b/src/Strategy/Godep.hs index 9613d3c52e..eb76119186 100644 --- a/src/Strategy/Godep.hs +++ b/src/Strategy/Godep.hs @@ -3,11 +3,13 @@ module Strategy.Godep ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Applicative ((<|>)) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, (<||>)) import Control.Effect.Diagnostics qualified as Diag -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk ( @@ -69,11 +71,15 @@ mkProject project = , projectData = project } -getDeps :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m) => GodepProject -> m DependencyResults -getDeps project = - context "Godep" $ - context "Gopkg.lock analysis" (analyzeGopkgLock project) - <||> context "Gopkg.toml analysis" (analyzeGopkgToml project) +getDeps :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => GodepProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then context "Strict mode Godep" $ context "Gopkg.lock analysis" (analyzeGopkgLock project) + else + context "Godep" $ + context "Gopkg.lock analysis" (analyzeGopkgLock project) + <||> context "Gopkg.toml analysis" (analyzeGopkgToml project) analyzeGopkgLock :: (Has ReadFS sig m, Has Exec sig m, Has Diagnostics sig m) => GodepProject -> m DependencyResults analyzeGopkgLock project = do diff --git a/src/Strategy/Gomodules.hs b/src/Strategy/Gomodules.hs index c8653dae4b..7b14e87edb 100644 --- a/src/Strategy/Gomodules.hs +++ b/src/Strategy/Gomodules.hs @@ -7,12 +7,13 @@ module Strategy.Gomodules ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) -import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (useV3GoResolver), GoDynamicTactic (..)) +import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig (useV3GoResolver), GoDynamicTactic (..), StrictMode (..)) import Control.Carrier.Diagnostics (warn) import Control.Effect.Diagnostics (Diagnostics, context, fatalText, recover, (<||>)) -import Control.Effect.Reader (Reader, asks) +import Control.Effect.Reader (Reader, ask, asks) import Control.Monad (when) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Text (Text) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) @@ -67,9 +68,13 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has (Reader (Flag StrictMode)) sig m) => GomodulesProject -> GoDynamicTactic -> m DependencyResults getDeps project goDynamicTactic = do - (graph, graphBreadth) <- context "Gomodules" $ dynamicAnalysis <||> staticAnalysis + strictMode <- ask @((Flag StrictMode)) + (graph, graphBreadth) <- + if fromFlag StrictMode strictMode + then context "Strict mode Gomodules" dynamicAnalysis + else context "Gomodules" $ dynamicAnalysis <||> staticAnalysis stdlib <- recover . context "Collect go standard library information" . listGoStdlibPackages $ gomodulesDir project pure $ DependencyResults diff --git a/src/Strategy/Maven.hs b/src/Strategy/Maven.hs index 5953ad9d21..87585cc6e1 100644 --- a/src/Strategy/Maven.hs +++ b/src/Strategy/Maven.hs @@ -7,11 +7,14 @@ module Strategy.Maven ( import App.Fossa.Analyze.LicenseAnalyze (LicenseAnalyzeProject, licenseAnalyzeProject) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Algebra (Has) -import Control.Effect.Diagnostics (Diagnostics, context, warnOnErr, (<||>)) +import Control.Carrier.Diagnostics qualified as Diag +import Control.Effect.Diagnostics (Diag (..), Diagnostics, context, warnOnErr, (<||>)) import Control.Effect.Lift (Lift) import Control.Effect.Reader (Reader, ask) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Data.Set (Set) import Data.Set qualified as Set import Data.Set.NonEmpty (nonEmpty, toSet) @@ -21,6 +24,7 @@ import Diag.Common (MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdge import Discovery.Filters (AllFilters, MavenScopeFilters, mavenScopeFilterSet) import Discovery.Simple (simpleDiscover) import Effect.Exec (CandidateCommandEffs) +import Effect.Logger (Logger, logDebug, pretty) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Graphing (Graphing, gmap, shrinkRoots) @@ -31,6 +35,7 @@ import Strategy.Maven.PluginStrategy qualified as Plugin import Strategy.Maven.Pom qualified as Pom import Strategy.Maven.Pom.Closure (MavenProjectClosure (..)) import Strategy.Maven.Pom.Closure qualified as PomClosure +import Text.Pretty.Simple (pShow) import Types (BuildTarget (..), DependencyResults (..), DiscoveredProject (..), DiscoveredProjectType (MavenProjectType), FoundTargets (..), GraphBreadth (..)) discover :: @@ -75,13 +80,22 @@ getDeps :: , Has ReadFS sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m + , Has Logger sig m ) => FoundTargets -> MavenProject -> m DependencyResults getDeps foundTargets (MavenProject closure) = do let submoduleTargets = submoduleTargetSet foundTargets - (graph, graphBreadth) <- context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure + strictMode <- ask @((Flag StrictMode)) + (graph, graphBreadth) <- + if fromFlag StrictMode strictMode + then do + logDebug "In switch case for strict mode (getDeps) &&&&&&&&&" + context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure + else context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure + -- (graph, graphBreadth) <- context "Maven" $ getDepsDynamicAnalysis submoduleTargets closure <||> getStaticAnalysis submoduleTargets closure pure $ DependencyResults { dependencyGraph = graph @@ -93,6 +107,8 @@ getDeps' :: ( Has (Lift IO) sig m , Has Diagnostics sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m + , Has Logger sig m ) => FoundTargets -> MavenProject -> @@ -113,17 +129,34 @@ getDepsDynamicAnalysis :: , Has ReadFS sig m , CandidateCommandEffs sig m , Has (Reader MavenScopeFilters) sig m + , Has (Reader (Flag StrictMode)) sig m + , Has Logger sig m ) => Set Text -> MavenProjectClosure -> m (Graphing Dependency, GraphBreadth) getDepsDynamicAnalysis submoduleTargets closure = do let allSubmodules = PomClosure.closureSubmodules closure + strictMode <- ask @((Flag StrictMode)) + logDebug $ "Is in strict mode for maven strategy ------------- " <> pretty (pShow strictMode) (graph, graphBreadth) <- - context "Dynamic Analysis" - $ warnOnErr MissingEdges - . warnOnErr MissingDeepDeps - $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) + if fromFlag StrictMode strictMode + then do + logDebug "In switch case for strict mode ===========" + context "Dynamic Analysis" $ getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure + else do + logDebug "NOT IN STRICT MODE ===========" + context "Dynamic Analysis" + $ warnOnErr MissingEdges + . warnOnErr MissingDeepDeps + $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) + + logDebug $ "Is in strict mode for maven strategy ------------- " <> pretty (pShow strictMode) + -- (graph, graphBreadth) <- + -- context "Dynamic Analysis" + -- $ warnOnErr MissingEdges + -- . warnOnErr MissingDeepDeps + -- $ (getDepsPlugin closure <||> getDepsTreeCmd closure <||> getDepsPluginLegacy closure) filteredGraph <- applyMavenFilters submoduleTargets allSubmodules graph pure (withoutProjectAsDep filteredGraph, graphBreadth) where @@ -137,28 +170,42 @@ getDepsPlugin :: ( CandidateCommandEffs sig m , Has (Lift IO) sig m , Has ReadFS sig m + , Has Diagnostics sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsPlugin closure = context "Plugin analysis" (Plugin.analyze' . parent $ PomClosure.closurePath closure) +getDepsPlugin closure = do + Diag.fatal MissingDeepDeps + +-- context "Plugin analysis" (Plugin.analyze' . parent $ PomClosure.closurePath closure) getDepsPluginLegacy :: ( CandidateCommandEffs sig m , Has (Lift IO) sig m , Has ReadFS sig m + , Has Diagnostics sig m + , Has Logger sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsPluginLegacy closure = context "Legacy Plugin analysis" (Plugin.analyzeLegacy' . parent $ PomClosure.closurePath closure) +getDepsPluginLegacy closure = do + logDebug $ "In plugin command ++++++++ " + -- Diag.fatal MissingDeepDeps + context "Legacy Plugin analysis" (Plugin.analyzeLegacy' . parent $ PomClosure.closurePath closure) getDepsTreeCmd :: ( Has (Lift IO) sig m , Has ReadFS sig m , CandidateCommandEffs sig m + , Has Diagnostics sig m + , Has Logger sig m ) => MavenProjectClosure -> m (Graphing MavenDependency, GraphBreadth) -getDepsTreeCmd closure = +getDepsTreeCmd closure = do + logDebug $ "In deps tree comand ++++++++ " + Diag.fatal MissingDeepDeps + context "Dynamic analysis" $ DepTreeCmd.analyze . parent $ PomClosure.closurePath closure diff --git a/src/Strategy/Pub.hs b/src/Strategy/Pub.hs index 55c66db080..2d9b991319 100644 --- a/src/Strategy/Pub.hs +++ b/src/Strategy/Pub.hs @@ -2,11 +2,13 @@ module Strategy.Pub (discover) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..), strictMode) import Control.Carrier.Diagnostics (errDoc, errHelp) import Control.Effect.Diagnostics (Diagnostics, errCtx, fatalText, recover, warnOnErr, (<||>)) -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Control.Monad (void) import Data.Aeson (ToJSON) +import Data.Flag (Flag, fromFlag) import Diag.Common ( MissingDeepDeps (MissingDeepDeps), MissingEdges (MissingEdges), @@ -15,7 +17,7 @@ import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) import Discovery.Walk (WalkStep (WalkContinue), findFileNamed, walkWithFilters') import Effect.Exec (Exec, Has) -import Effect.Logger (Logger) +import Effect.Logger (Logger, logDebug) import Effect.ReadFS (ReadFS) import GHC.Generics (Generic) import Path (Abs, Dir, File, Path) @@ -64,18 +66,39 @@ mkProject project = , projectData = project } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => PubProject -> m DependencyResults +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => PubProject -> m DependencyResults getDeps project = do + strictMode <- ask @((Flag StrictMode)) (graph, graphBreadth) <- case pubLock project of - Just lockFile -> analyzeDepsCmd lockFile (pubSpecDir project) <||> analyzePubLockFile lockFile + Just lockFile -> do + if fromFlag StrictMode strictMode + then do + logDebug "Pub Analysis in strict mode" + analyzeDepsCmd lockFile (pubSpecDir project) + else do + logDebug "Pub Analysis NOT in strict mode" + analyzeDepsCmd lockFile (pubSpecDir project) <||> analyzePubLockFile lockFile Nothing -> do - void . recover - $ warnOnErr MissingDeepDeps - . warnOnErr MissingEdges - $ errCtx PubspecLimitationCtx - $ errHelp PubspecLimitationHelp - $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - analyzePubSpecFile $ pubSpec project + if fromFlag StrictMode strictMode + then do + logDebug "Pub Analysis (NOTHING) in strict mode" + analyzePubSpecFile $ pubSpec project + else do + logDebug "Pub Analysis (NOTHING) NOT in strict mode" + void . recover + $ warnOnErr MissingDeepDeps + . warnOnErr MissingEdges + $ errCtx PubspecLimitationCtx + $ errHelp PubspecLimitationHelp + $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + analyzePubSpecFile $ pubSpec project + -- void . recover + -- $ warnOnErr MissingDeepDeps + -- . warnOnErr MissingEdges + -- $ errCtx PubspecLimitationCtx + -- $ errHelp PubspecLimitationHelp + -- $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + -- analyzePubSpecFile $ pubSpec project pure $ DependencyResults { dependencyGraph = graph @@ -83,18 +106,27 @@ getDeps project = do , dependencyManifestFiles = [pubSpec project] } -getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => PubProject -> m DependencyResults +getDeps' :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => PubProject -> m DependencyResults getDeps' project = do + strictMode <- ask @((Flag StrictMode)) (graph, graphBreadth) <- case pubLock project of Just lockFile -> analyzePubLockFile lockFile Nothing -> do - void . recover - $ warnOnErr MissingDeepDeps - . warnOnErr MissingEdges - $ errCtx PubspecLimitationCtx - $ errHelp PubspecLimitationHelp - $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") - analyzePubSpecFile $ pubSpec project + if fromFlag StrictMode strictMode + then do + void $ + errCtx PubspecLimitationCtx $ + errHelp PubspecLimitationHelp $ + errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + analyzePubSpecFile $ pubSpec project + else do + void . recover + $ warnOnErr MissingDeepDeps + . warnOnErr MissingEdges + $ errCtx PubspecLimitationCtx + $ errHelp PubspecLimitationHelp + $ errDoc refPubDocUrl (fatalText "Missing pubspec.lock file") + analyzePubSpecFile $ pubSpec project pure $ DependencyResults { dependencyGraph = graph diff --git a/src/Strategy/Scala.hs b/src/Strategy/Scala.hs index 0c332476e3..5ffb816924 100644 --- a/src/Strategy/Scala.hs +++ b/src/Strategy/Scala.hs @@ -13,12 +13,14 @@ module Strategy.Scala ( ) where import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import App.Fossa.Config.Analyze (StrictMode (..)) import Control.Carrier.Diagnostics (errDoc) import Control.Effect.Diagnostics (Diagnostics, errCtx, errHelp, fatalText, fromMaybeText, recover, warnOnErr, (<||>)) -import Control.Effect.Reader (Reader) +import Control.Effect.Reader (Reader, ask) import Control.Effect.Stack (context) import Data.Aeson (KeyValue ((.=)), ToJSON (toJSON), object) import Data.ByteString.Lazy (ByteString) +import Data.Flag (Flag, fromFlag) import Data.Maybe (fromMaybe, listToMaybe, mapMaybe) import Data.String.Conversion (ConvertUtf8 (decodeUtf8), toString, toText) import Data.Text (Text) @@ -127,10 +129,14 @@ mkProject (ScalaProject sbtBuildDir sbtTreeJson closure) = , projectData = ScalaProject sbtBuildDir sbtTreeJson closure } -getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => ScalaProject -> m DependencyResults -getDeps project = - warnOnErr MissingDeepDeps (analyzeWithDepTreeJson project <||> analyzeWithSbtDepTree project) - <||> analyzeWithPoms project +getDeps :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has (Reader (Flag StrictMode)) sig m) => ScalaProject -> m DependencyResults +getDeps project = do + strictMode <- ask @((Flag StrictMode)) + if fromFlag StrictMode strictMode + then analyzeWithDepTreeJson project + else + warnOnErr MissingDeepDeps (analyzeWithDepTreeJson project <||> analyzeWithSbtDepTree project) + <||> analyzeWithPoms project pathToText :: Path ar fd -> Text pathToText = toText . toFilePath