Implementation to add custom fossa-deps file on analyze run

spectrometer.cabal

@@ -118,6 +118,7 @@ common deps
     , retry                        ^>=0.9.0.0
     , safe-exceptions              ^>=0.1.7
     , semver                       ^>=0.4.0.1
+    , split                        ^>=0.2.3.5
     , stm                          ^>=2.5.0
     , stm-chans                    ^>=3.0.0
     , tar                          ^>=0.6.0.0

src/App/Fossa/Analyze.hs

@@ -254,7 +254,6 @@ analyze ::
   m Aeson.Value
 analyze cfg = Diag.context "fossa-analyze" $ do
   capabilities <- sendIO getNumCapabilities
-
   let maybeApiOpts = case destination of
         OutputStdout -> Nothing
         UploadScan opts _ -> Just opts
@@ -269,11 +268,16 @@ analyze cfg = Diag.context "fossa-analyze" $ do
       skipResolutionSet = Config.vsiSkipSet $ Config.vsiOptions cfg
       vendoredDepsOptions = Config.vendoredDeps cfg
       grepOptions = Config.grepOptions cfg
+      BaseDir fossaDepsDir = Config.fossaDepsDir cfg
 
   -- additional source units are built outside the standard strategy flow, because they either
   -- require additional information (eg API credentials), or they return additional information (eg user deps).
-  vsiResults <- Diag.errorBoundaryIO . diagToDebug $
-    Diag.context "analyze-vsi" . runStickyLogger SevInfo . runFinally $ do
+  vsiResults <- Diag.errorBoundaryIO
+    . diagToDebug
+    $ Diag.context "analyze-vsi"
+      . runStickyLogger SevInfo
+      . runFinally
+    $ do
       let shouldRunVSI = fromFlag Config.VSIAnalysis $ Config.vsiAnalysisEnabled $ Config.vsiOptions cfg
       case (shouldRunVSI, maybeApiOpts) of
         (True, Just apiOpts') ->
@@ -284,24 +288,29 @@ analyze cfg = Diag.context "fossa-analyze" $ do
     Diag.errorBoundaryIO
       . diagToDebug
       . runReader filters
-      $ Diag.context "discover-dynamic-linking" . doAnalyzeDynamicLinkedBinary basedir . Config.dynamicLinkingTarget
+      $ Diag.context "discover-dynamic-linking"
+        . doAnalyzeDynamicLinkedBinary basedir
+        . Config.dynamicLinkingTarget
       $ Config.vsiOptions cfg
   binarySearchResults <-
-    Diag.errorBoundaryIO . diagToDebug $
-      Diag.context "discover-binaries" $
-        if (fromFlag BinaryDiscovery $ Config.binaryDiscoveryEnabled $ Config.vsiOptions cfg)
-          then analyzeDiscoverBinaries basedir filters
-          else pure Nothing
+    Diag.errorBoundaryIO
+      . diagToDebug
+      $ Diag.context "discover-binaries"
+      $ if (fromFlag BinaryDiscovery $ Config.binaryDiscoveryEnabled $ Config.vsiOptions cfg)
+        then analyzeDiscoverBinaries basedir filters
+        else pure Nothing
   manualSrcUnits <-
-    Diag.errorBoundaryIO . diagToDebug $
-      if filterIsVSIOnly filters
+    Diag.errorBoundaryIO
+      . diagToDebug
+      $ if filterIsVSIOnly filters
         then do
           logInfo "Running in VSI only mode, skipping manual source units"
           pure Nothing
-        else Diag.context "fossa-deps" . runStickyLogger SevInfo $ analyzeFossaDepsFile basedir maybeApiOpts vendoredDepsOptions
+        else Diag.context "fossa-deps" . runStickyLogger SevInfo $ analyzeFossaDepsFile fossaDepsDir maybeApiOpts vendoredDepsOptions
   maybeLernieResults <-
-    Diag.errorBoundaryIO . diagToDebug $
-      if filterIsVSIOnly filters
+    Diag.errorBoundaryIO
+      . diagToDebug
+      $ if filterIsVSIOnly filters
         then do
           logInfo "Running in VSI only mode, skipping keyword search and custom-license search"
           pure Nothing
@@ -323,8 +332,9 @@ analyze cfg = Diag.context "fossa-analyze" $ do
   traverse_ (Diag.flushLogs SevError SevDebug) [maybeLernieResults]
 
   maybeFirstPartyScanResults <-
-    Diag.errorBoundaryIO . diagToDebug $
-      if firstPartyScansFlag cfg == FirstPartyScansOffFromFlag
+    Diag.errorBoundaryIO
+      . diagToDebug
+      $ if firstPartyScansFlag cfg == FirstPartyScansOffFromFlag
         then do
           logInfo "first party scans forced off by the --experimental-block-first-party-scans flag. Skipping first party scans"
           pure Nothing
@@ -345,9 +355,10 @@ analyze cfg = Diag.context "fossa-analyze" $ do
       $ do
         runAnalyzers filters basedir Nothing
         when (fromFlag UnpackArchives $ Config.unpackArchives cfg) $
-          forkTask $ do
-            res <- Diag.runDiagnosticsIO . diagToDebug . stickyLogStack . withEmptyStack $ Archive.discover (runAnalyzers filters) basedir ancestryDirect
-            Diag.withResult SevError SevWarn res (const (pure ()))
+          forkTask $
+            do
+              res <- Diag.runDiagnosticsIO . diagToDebug . stickyLogStack . withEmptyStack $ Archive.discover (runAnalyzers filters) basedir ancestryDirect
+              Diag.withResult SevError SevWarn res (const (pure ()))
 
   let projectResults = mapMaybe toProjectResult projectScans
   let filteredProjects = mapMaybe toProjectResult projectScans

src/App/Fossa/Config/Analyze.hs

@@ -74,6 +74,8 @@ import Control.Effect.Lift (Lift)
 import Control.Monad (when)
 import Data.Aeson (ToJSON (toEncoding), defaultOptions, genericToEncoding)
 import Data.Flag (Flag, flagOpt, fromFlag)
+import Data.List.Extra
+import Data.List.Split
 import Data.Monoid.Extra (isMempty)
 import Data.Set (Set)
 import Data.Set qualified as Set
@@ -205,6 +207,7 @@ data AnalyzeCliOpts = AnalyzeCliOpts
   , analyzeForceFirstPartyScans :: Flag ForceFirstPartyScans
   , analyzeForceNoFirstPartyScans :: Flag ForceNoFirstPartyScans
   , analyzeIgnoreOrgWideCustomLicenseScanConfigs :: Flag IgnoreOrgWideCustomLicenseScanConfigs
+  , fossaDepsConfig :: Maybe FilePath
   }
   deriving (Eq, Ord, Show)
 
@@ -233,6 +236,7 @@ data AnalyzeConfig = AnalyzeConfig
   , overrideDynamicAnalysis :: OverrideDynamicAnalysisBinary
   , firstPartyScansFlag :: FirstPartyScansFlag
   , grepOptions :: GrepOptions
+  , fossaDepsDir :: BaseDir
   }
   deriving (Eq, Ord, Show, Generic)
 
@@ -283,6 +287,7 @@ cliParser =
     <*> flagOpt ForceFirstPartyScans (long "experimental-force-first-party-scans" <> help "Force first party scans")
     <*> flagOpt ForceNoFirstPartyScans (long "experimental-block-first-party-scans" <> help "Block first party scans. This can be used to forcibly turn off first-party scans if your organization defaults to first-party scans.")
     <*> flagOpt IgnoreOrgWideCustomLicenseScanConfigs (long "ignore-org-wide-custom-license-scan-configs" <> help "Ignore custom-license scan configurations for your organization. These configurations are defined in the \"Integrations\" section of the Admin settings in the FOSSA web app")
+    <*> optional (strOption (long "fossa-deps-config" <> help "Path to fossa-deps configuration file including filename (default: .fossa.yml)"))
 
 data GoDynamicTactic
   = GoModulesBasedTactic
@@ -406,6 +411,9 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do
       vendoredDepsOptions = collectVendoredDeps maybeConfig cliOpts
       dynamicAnalysisOverrides = OverrideDynamicAnalysisBinary $ envCmdOverrides envvars
       grepOptions = collectGrepOptions maybeConfig cliOpts
+      fossaDepsDir = case fossaDepsConfig of
+        Just filePath -> collectBaseDir (head (splitOn "fossa-deps" filePath))
+        _ -> collectBaseDir analyzeBaseDir
   firstPartyScansFlag <-
     case (fromFlag ForceFirstPartyScans analyzeForceFirstPartyScans, fromFlag ForceNoFirstPartyScans analyzeForceNoFirstPartyScans) of
       (True, True) -> fatalText "You provided both the --experimental-force-first-party-scans and --experimental-block-first-party-scans flags. Only one of these flags may be used"
@@ -429,6 +437,7 @@ mergeStandardOpts maybeConfig envvars cliOpts@AnalyzeCliOpts{..} = do
     <*> pure dynamicAnalysisOverrides
     <*> pure firstPartyScansFlag
     <*> pure grepOptions
+    <*> fossaDepsDir
 
 collectFilters ::
   ( Has Diagnostics sig m

src/App/Fossa/ManualDeps.hs

@@ -103,7 +103,6 @@ findAndReadFossaDepsFile root = do
     Just depsFile -> do
       manualDeps <- readFoundDeps depsFile
       pure $ Just manualDeps
-
 readFoundDeps :: (Has Diagnostics sig m, Has ReadFS sig m) => FoundDepsFile -> m ManualDependencies
 readFoundDeps (ManualJSON path) = readContentsJson path
 readFoundDeps (ManualYaml path) = readContentsYaml path
@@ -145,7 +144,6 @@ toSourceUnit ::
 toSourceUnit root depsFile manualDeps@ManualDependencies{..} maybeApiOpts vendoredDepsOptions = do
   -- If the file exists and we have no dependencies to report, that's a failure.
   when (hasNoDeps manualDeps) $ fatalText "No dependencies found in fossa-deps file"
-
   archiveLocators <- case (maybeApiOpts, NE.nonEmpty vendoredDependencies) of
     (Just apiOpts, Just vdeps) -> NE.toList <$> runFossaApiClient apiOpts (scanAndUpload root vdeps vendoredDepsOptions)
     (Nothing, Just vdeps) -> pure $ noSourceUnits $ NE.toList vdeps
@@ -433,12 +431,13 @@ instance FromJSON ReferencedDependency where
       parseOS :: Object -> Parser Text
       parseOS obj = do
         os <- requiredFieldMsg "os" $ obj .: "os"
-        unless (toLower os `elem` supportedOSs) $
-          fail . toString $
-            "Provided os: "
-              <> (toLower os)
-              <> " is not supported! Please provide oneOf: "
-              <> Text.intercalate ", " supportedOSs
+        unless (toLower os `elem` supportedOSs)
+          $ fail
+            . toString
+          $ "Provided os: "
+            <> (toLower os)
+            <> " is not supported! Please provide oneOf: "
+            <> Text.intercalate ", " supportedOSs
         pure os
 
       requiredFieldMsg :: String -> Parser a -> Parser a
@@ -476,7 +475,8 @@ instance FromJSON CustomDependency where
       <$> (obj `neText` "name")
       <*> (unTextLike <$> obj `neText` "version")
       <*> (obj `neText` "license")
-      <*> obj .:? "metadata"
+      <*> obj
+        .:? "metadata"
       <* forbidMembers "custom dependencies" ["type", "path", "url"] obj
 
 instance FromJSON RemoteDependency where
@@ -485,7 +485,8 @@ instance FromJSON RemoteDependency where
       <$> (obj `neText` "name")
       <*> (unTextLike <$> obj `neText` "version")
       <*> (obj `neText` "url")
-      <*> obj .:? "metadata"
+      <*> obj
+        .:? "metadata"
       <* forbidMembers "remote dependencies" ["license", "path", "type"] obj
 
 validateRemoteDep :: (Has Diagnostics sig m) => RemoteDependency -> Organization -> m RemoteDependency
@@ -537,8 +538,10 @@ instance ToDiagnostic RemoteDepLengthIsGtThanAllowed where
 instance FromJSON DependencyMetadata where
   parseJSON = withObject "metadata" $ \obj ->
     DependencyMetadata
-      <$> obj .:? "description"
-      <*> obj .:? "homepage"
+      <$> obj
+        .:? "description"
+      <*> obj
+        .:? "homepage"
       <* forbidMembers "metadata" ["url"] obj
 
 -- Parse supported dependency types into their respective type or return Nothing.

test/Test/Fixtures.hs

@@ -162,10 +162,11 @@ baseDir = do
 
 contributors :: API.Contributors
 contributors =
-  API.Contributors . Map.fromList $
-    [ ("testContributor1", "testContributor1")
-    , ("testContributor2", "testContributor2")
-    ]
+  API.Contributors
+    . Map.fromList
+    $ [ ("testContributor1", "testContributor1")
+      , ("testContributor2", "testContributor2")
+      ]
 
 build :: API.Build
 build =
@@ -404,4 +405,5 @@ standardAnalyzeConfig =
     , ANZ.overrideDynamicAnalysis = App.OverrideDynamicAnalysisBinary{unOverrideDynamicAnalysisBinary = mempty}
     , ANZ.firstPartyScansFlag = App.FirstPartyScansUseDefault
     , ANZ.grepOptions = grepOptions
+    , ANZ.fossaDepsDir = App.BaseDir absDir
     }