New line parser for requirements.txt (#183)

* handle new lines

* update changelog

Changelog.md

@@ -1,3 +1,10 @@
+# v2.4.9
+- Fix a bug with `requirements.txt` parsing line extensions ([#183](https://github.com/fossas/spectrometer/pull/183))
+- Fix a bug where we didn't read the cached fossa revision for projects without VCS ([#182](https://github.com/fossas/spectrometer/pull/182))
+- Fix a bug with project URL output when no branch is supplied in instances where VCS does not exist ([#181](https://github.com/fossas/spectrometer/pull/181))
+# v2.4.8
+- Introduce a new hidden `fossa compatibility` command which runs fossa v1 `fossa analyze` and allows users to access the archive uploader([#179]https://github.com/fossas/spectrometer/pull/179)
+
 # v2.4.7
 
 - Fixes an issue where `fossa test` would always succeed for push-only API keys ([#170](https://github.com/fossas/spectrometer/pull/170))

src/Strategy/Python/ReqTxt.hs

@@ -1,5 +1,6 @@
 module Strategy.Python.ReqTxt
   ( analyze'
+  , requirementsTxtParser
   )
   where
 
@@ -25,27 +26,32 @@ type Parser = Parsec Void Text
 
 -- https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
 requirementsTxtParser :: Parser [Req]
-requirementsTxtParser = concat <$> ((line `sepBy` eol) <* eof)
+requirementsTxtParser =  concat <$> manyTill reqParser eof
+
+reqParser :: Parser [Req]
+reqParser = [] <$ char '-' <* ignored -- pip options
+     <|> [] <$ char '.' <* ignored -- relative path
+     <|> [] <$ char '/' <* ignored -- absolute path
+     <|> [] <$ oneOfS ["http:", "https:", "git+", "hg+", "svn+", "bzr+"] <* ignored -- URLs
+     <|> [] <$ comment
+     <|> (pure <$> requirementParser <* ignored)
+     <|> pure [] -- empty line
+
   where
   isEndLine :: Char -> Bool
   isEndLine '\n' = True
   isEndLine '\r' = True
+  isEndLine '\\' = True
   isEndLine _    = False
 
   -- ignore content until the end of the line
   ignored :: Parser ()
-  ignored = () <$ takeWhileP (Just "ignored") (not . isEndLine)
+  ignored = () <$ takeWhileP (Just "ignored") (not . isEndLine) <* (try newLine <|> (() <$ takeWhileP (Just "end of line") isEndLine))
+
+  newLine :: Parser ()
+  newLine = char '\\' <* takeWhileP (Just "endLine") isEndLine *> ignored 
 
   comment :: Parser ()
   comment = char '#' *> ignored
 
-  -- FUTURE: we can case split / sum-type this for better analysis
-  line = [] <$ char '-' <* ignored -- pip options
-     <|> [] <$ char '.' <* ignored -- relative path
-     <|> [] <$ char '/' <* ignored -- absolute path
-     <|> [] <$ oneOfS ["http:", "https:", "git+", "hg+", "svn+", "bzr+"] <* ignored -- URLs
-     <|> [] <$ comment
-     <|> (pure <$> requirementParser <* optional comment)
-     <|> pure [] -- empty line
-
-  oneOfS = asum . map string
+  oneOfS = asum . map string

test/Python/RequirementsSpec.hs

@@ -3,19 +3,18 @@ module Python.RequirementsSpec
   )
 where
 
-import Data.Foldable
+import Data.Foldable ( traverse_ )
 import Data.Text (Text)
-import Strategy.Python.Util (requirementParser)
+import Strategy.Python.Util (requirementParser, buildGraph)
+import Strategy.Python.ReqTxt (requirementsTxtParser)
 import Test.Hspec.Megaparsec
 import Text.Megaparsec
 import Prelude
-import Test.Hspec
-
-spec :: Spec
-spec =
-  describe "requirementParser"
-    $ it "can parse the edge case examples"
-    $ traverse_ (\input -> runParser requirementParser "" `shouldSucceedOn` input) examples
+import DepTypes
+import qualified Data.Map.Strict as M
+import qualified Data.Text.IO as TIO
+import qualified Test.Hspec as T
+import GraphUtil (expectDeps)
 
 examples :: [Text]
 examples =
@@ -35,3 +34,57 @@ examples =
     "name; os_name=='a' or os_name=='b' and os_name=='c'",
     "name; (os_name=='a' or os_name=='b') and os_name=='c'"
   ]
+
+depOne :: Dependency
+depOne = Dependency { dependencyType = PipType
+                        , dependencyName = "one"
+                        , dependencyVersion = Just (CEq "1.0")
+                        , dependencyLocations = []
+                        , dependencyEnvironments = []
+                        , dependencyTags = M.empty
+                        }
+
+depTwo :: Dependency
+depTwo = Dependency { dependencyType = PipType
+                        , dependencyName = "two"
+                        , dependencyVersion = Just (CLessOrEq "2.0.0")
+                        , dependencyLocations = []
+                        , dependencyEnvironments = []
+                        , dependencyTags = M.empty
+                        }
+
+depThree :: Dependency
+depThree = Dependency { dependencyType = PipType
+                        , dependencyName = "three"
+                        , dependencyVersion = Just (CEq "3.0.0")
+                        , dependencyLocations = []
+                        , dependencyEnvironments = []
+                        , dependencyTags = M.empty
+                        }
+
+depFour :: Dependency
+depFour = Dependency { dependencyType = PipType
+                        , dependencyName = "four"
+                        , dependencyVersion = Just (CEq "4.0.0")
+                        , dependencyLocations = []
+                        , dependencyEnvironments = []
+                        , dependencyTags = M.empty
+                        }
+
+spec :: T.Spec
+spec = do
+  T.describe "requirementParser"
+    $ T.it "can parse the edge case examples"
+    $ traverse_ (\input -> runParser requirementParser "" `shouldSucceedOn` input) examples
+
+  requirementsTextFile <- T.runIO (TIO.readFile "test/Python/testdata/req.txt")
+  T.describe "req file" $
+    T.it "can parse" $
+      case runParser requirementsTxtParser "" requirementsTextFile of 
+        Left r -> do
+          T.expectationFailure $ "failed to parse: error:" ++ show r
+        Right res -> do
+          let result = buildGraph res 
+          expectDeps [depOne, depTwo, depThree, depFour] result
+
+

test/Python/testdata/req.txt

@@ -0,0 +1,7 @@
+one==1.0
+two<=2.0.0
+three==3.0.0 \
+  --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+  --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+  # via python-dateuti
+four==4.0.0 #help