diff --git a/Changelog.md b/Changelog.md index c2c565a13..7f249d011 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # Spectrometer Changelog +## v2.19.8 + +- Perl: Adds support for Perl with parsing of `META.json`, `META.yml`, `MYMETA.yml`, `MYMETA.json`. ([#428](https://github.com/fossas/spectrometer/pull/428)) + ## v2.19.7 - Resolves a regression when parsing npm `package-lock.json` files that do not contain a `version` field ([#445](https://github.com/fossas/spectrometer/pull/445)) diff --git a/docs/references/files/fossa-yml.v3.schema.json b/docs/references/files/fossa-yml.v3.schema.json index 392bc1509..3d84cde5a 100644 --- a/docs/references/files/fossa-yml.v3.schema.json +++ b/docs/references/files/fossa-yml.v3.schema.json @@ -201,6 +201,10 @@ { "const": "setuptools", "description": "For setuptools targets (python)" + }, + { + "const": "perl", + "description": "For perl targets (using *META.{json,yml})" } ], "description": "Target (package manager)" diff --git a/docs/references/strategies/README.md b/docs/references/strategies/README.md index b2d74a62f..366c92553 100644 --- a/docs/references/strategies/README.md +++ b/docs/references/strategies/README.md @@ -68,6 +68,10 @@ - [carthage](platforms/ios/carthage.md) - [cocoapods](platforms/ios/cocoapods.md) +### perl + +- [perl](languages/perl/perl.md) + ### php - [php](languages/php/composer.md) diff --git a/docs/references/strategies/languages/perl/perl.md b/docs/references/strategies/languages/perl/perl.md new file mode 100644 index 000000000..4c4f307e5 --- /dev/null +++ b/docs/references/strategies/languages/perl/perl.md @@ -0,0 +1,38 @@ +# Perl Analysis + +| Strategy | Direct Deps | Deep Deps | Edges | Classifies Dev Dependencies | +| ------------------ | ------------------ | ------------------ | ----- | --------------------------- | +| `*META.{yml, json} | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | + +## Project Discovery + +Find a file named `MYMETA.json`, `MYMETA.yml`, `META.json`, or `META.yml`. + +## Analysis + +1. Parse `MYMETA.{yml, json}` or `META.{yml, json}` to identify dependencies. + +## Limitation + +- Dependency required for `runtime` only will be reported. +- Reported analysis will not have any edges. + +## Example + +1. Build your perl target. When you do this, you should have `MYMETA.yml` and `MYMETA.json`. +2. Execute `fossa analyze -o` on the project to print analyzed dependency graphing (this will not upload any analysis to any endpoint) + +## FAQ + +### How do I *only perform analysis* for Perl? + +You can explicitly specify an analysis target in `.fossa.yml` file. The example below will exclude all analysis targets except for the composer. + +```yaml +# .fossa.yml + +version: 3 +targets: + only: + - type: perl +``` \ No newline at end of file diff --git a/spectrometer.cabal b/spectrometer.cabal index 8eaae770a..acad54804 100644 --- a/spectrometer.cabal +++ b/spectrometer.cabal @@ -295,6 +295,7 @@ library Strategy.NuGet.Paket Strategy.NuGet.ProjectAssetsJson Strategy.NuGet.ProjectJson + Strategy.Perl Strategy.Pub Strategy.Python.Pipenv Strategy.Python.Poetry @@ -399,6 +400,7 @@ test-suite unit-tests NuGet.PaketSpec NuGet.ProjectAssetsJsonSpec NuGet.ProjectJsonSpec + Perl.PerlSpec Python.PipenvSpec Python.Poetry.CommonSpec Python.Poetry.PoetryLockSpec diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 466b2ad2e..7e9ff431c 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -125,6 +125,7 @@ import Strategy.NuGet.PackagesConfig qualified as PackagesConfig import Strategy.NuGet.Paket qualified as Paket import Strategy.NuGet.ProjectAssetsJson qualified as ProjectAssetsJson import Strategy.NuGet.ProjectJson qualified as ProjectJson +import Strategy.Perl qualified as Perl import Strategy.Pub qualified as Pub import Strategy.Python.Pipenv qualified as Pipenv import Strategy.Python.Poetry qualified as Poetry @@ -285,6 +286,7 @@ discoverFuncs = , DiscoverFunc PackageReference.discover , DiscoverFunc PackagesConfig.discover , DiscoverFunc Paket.discover + , DiscoverFunc Perl.discover , DiscoverFunc Pipenv.discover , DiscoverFunc Poetry.discover , DiscoverFunc ProjectAssetsJson.discover diff --git a/src/Path/Extra.hs b/src/Path/Extra.hs index 15f339a50..d1d8e9097 100644 --- a/src/Path/Extra.hs +++ b/src/Path/Extra.hs @@ -1,11 +1,12 @@ module Path.Extra ( tryMakeRelative, renderRelative, + extensionOf, ) where import Data.String.Conversion (toText) import Data.Text (Text) -import Path (Abs, Dir, File, Path, SomeBase (..), stripProperPrefix) +import Path (Abs, Dir, File, Path, SomeBase (..), fileExtension, stripProperPrefix) -- tryMakeRelative returns the path of an absolute file (Path Abs File) relative to an absolute directory (Path Abs Dir). -- If the file is not within the directory, then the absolute file path will be returned @@ -19,3 +20,6 @@ tryMakeRelative absDir absFile = -- Intended for convenience when displaying the newly relative path; to interact with it use `tryMakeRelative` instead. renderRelative :: Path Abs Dir -> Path Abs File -> Text renderRelative absDir absFile = toText $ tryMakeRelative absDir absFile + +extensionOf :: Path Abs File -> Maybe Text +extensionOf absFile = toText <$> fileExtension absFile diff --git a/src/Strategy/Perl.hs b/src/Strategy/Perl.hs new file mode 100644 index 000000000..7e3ebb714 --- /dev/null +++ b/src/Strategy/Perl.hs @@ -0,0 +1,202 @@ +module Strategy.Perl ( + discover, + + -- * for testing + PackageName (..), + PerlMeta (..), + buildGraph, +) where + +import App.Fossa.Analyze.Types (AnalyzeProject, analyzeProject) +import Control.Applicative ((<|>)) +import Control.Effect.Diagnostics (Diagnostics, context) +import Data.Aeson (Object, ToJSON) +import Data.Aeson.Types (FromJSONKey, Parser, withObject) +import Data.Aeson.Types qualified as AesonTypes +import Data.Foldable (asum) +import Data.Map (Map, toList) +import Data.Maybe (fromMaybe) +import Data.Set qualified as Set +import Data.Text (Text) +import Data.Yaml (FromJSON (parseJSON), (.:), (.:?)) +import DepTypes ( + DepEnvironment (..), + DepType (CpanType), + Dependency (..), + VerConstraint (CEq), + ) +import Discovery.Walk ( + WalkStep (WalkContinue), + findFileNamed, + walk', + ) +import Effect.Exec (Has) +import Effect.ReadFS (ReadFS, readContentsJson, readContentsYaml) +import GHC.Generics (Generic) +import Graphing (Graphing, deeps) +import Path +import Path.Extra (extensionOf) +import Text.Read (readMaybe) +import Types ( + DependencyResults (..), + DiscoveredProject (..), + GraphBreadth (Partial), + ) + +discover :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [DiscoveredProject PerlProject] +discover dir = context "Perl" $ do + projects <- context "Finding projects" $ findProjects dir + pure (map mkProject projects) + +findProjects :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [PerlProject] +findProjects = walk' $ \dir _ files -> do + -- We prefer MYMETA over META. + -- Reference: https://metacpan.org/dist/App-mymeta_requires/view/bin/mymeta-requires + case asum $ map (`findFileNamed` files) ["MYMETA.json", "MYMETA.yml", "META.json", "META.yml"] of + Nothing -> pure ([], WalkContinue) + Just f -> pure ([PerlProject dir f], WalkContinue) + +data PerlProject = PerlProject + { perlDir :: Path Abs Dir + , perlMetaFile :: Path Abs File + } + deriving (Eq, Ord, Show, Generic) + +instance ToJSON PerlProject +instance AnalyzeProject PerlProject where + analyzeProject _ = getDeps + +mkProject :: PerlProject -> DiscoveredProject PerlProject +mkProject project = + DiscoveredProject + { projectType = "perl" + , projectBuildTargets = mempty + , projectPath = perlDir project + , projectData = project + } + +getDeps :: (Has ReadFS sig m, Has Diagnostics sig m) => PerlProject -> m DependencyResults +getDeps project = do + graph <- analyze (perlMetaFile project) + pure $ + DependencyResults + { dependencyGraph = graph + , dependencyGraphBreadth = Partial + , dependencyManifestFiles = [perlMetaFile project] + } + +analyze :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> m (Graphing Dependency) +analyze metaFile = do + content <- context "Identifying dependencies in meta file" $ + case extensionOf metaFile of + Just "json" -> readContentsJson metaFile + _ -> readContentsYaml metaFile + + pure $ buildGraph (content) + +newtype PackageName = PackageName {unPackageName :: Text} deriving (Show, Eq, Ord, FromJSONKey) + +-- | Represents Metafile for various versions. +-- References: +-- v1.0: http://module-build.sourceforge.net/META-spec-v1.0.html +-- v1.1: http://module-build.sourceforge.net/META-spec-v1.1.html +-- v1.2: http://module-build.sourceforge.net/META-spec-v1.2.html +-- v1.3: http://module-build.sourceforge.net/META-spec-v1.3.html +-- v1.4: https://metacpan.org/release/DAGOLDEN/CPAN-Meta-2.101090 +-- v2.0: https://metacpan.org/release/DAGOLDEN/CPAN-Meta-2.150010 +data PerlMeta = PerlMeta + { version :: Double + , runtimeRequires :: Maybe (Map PackageName (Maybe Text)) + , buildRequires :: Maybe (Map PackageName (Maybe Text)) + , testRequires :: Maybe (Map PackageName (Maybe Text)) + , developRequires :: Maybe (Map PackageName (Maybe Text)) + , configureRequires :: Maybe (Map PackageName (Maybe Text)) + } + deriving (Generic, Show, Eq, Ord) + +instance FromJSON PackageName where + parseJSON (AesonTypes.String packageName) = pure $ PackageName packageName + parseJSON _ = fail "failed to parse package's name" + +instance FromJSON PerlMeta where + parseJSON = withObject "meta content" $ \o -> do + -- spec_version can be either be string or number + -- in yaml, version is provided as string, where as in json, it is numeric + specVersion :: Double <- + (o .: "meta-spec" |> "version") + <|> ( do + v <- o .: "meta-spec" |> "version" + case readMaybe v of + Nothing -> fail ("Expected numeric value for version field, but got: " <> show v) + Just x -> pure x + ) + + if specVersion > 1.4 + then parseAboveV1_4 o specVersion + else parseBelowV1_5 o specVersion + where + (|>) :: FromJSON a => Parser Object -> Text -> Parser a + (|>) parser key = do + obj <- parser + obj .: key + + (|?>) :: FromJSON a => Parser (Maybe Object) -> Text -> Parser (Maybe a) + (|?>) parser key = do + obj <- parser + case obj of + Nothing -> pure Nothing + Just o -> o .:? key + + -- Reference: https://metacpan.org/release/DAGOLDEN/CPAN-Meta-2.101090 + parseAboveV1_4 obj version = do + runtimeRequires <- obj .:? "prereqs" |?> "runtime" |?> "requires" + buildRequires <- obj .:? "prereqs" |?> "build" |?> "requires" + testRequires <- obj .:? "prereqs" |?> "test" |?> "requires" + developRequires <- obj .:? "prereqs" |?> "develop" |?> "requires" + configureRequires <- obj .:? "prereqs" |?> "configure" |?> "requires" + + pure $ + PerlMeta + version + runtimeRequires + buildRequires + testRequires + developRequires + configureRequires + + -- Reference: http://module-build.sourceforge.net/META-spec-v1.4.html + parseBelowV1_5 obj version = do + runtimeRequires <- obj .:? "requires" + buildRequires <- obj .:? "build_requires" + configureRequires1 <- obj .:? "configure_requires" + pure $ PerlMeta version runtimeRequires buildRequires Nothing Nothing configureRequires1 + +buildGraph :: PerlMeta -> Graphing Dependency +buildGraph meta = + deeps $ + filter + notNamedPerl + (runtimeDeps ++ testDeps ++ developDeps ++ buildDeps ++ configureDeps) + where + runtimeDeps = getDepsOf EnvProduction runtimeRequires + testDeps = getDepsOf EnvTesting testRequires + developDeps = getDepsOf EnvDevelopment developRequires + buildDeps = getDepsOf EnvDevelopment buildRequires + configureDeps = getDepsOf EnvDevelopment configureRequires + + notNamedPerl :: Dependency -> Bool + notNamedPerl dep = dependencyName dep /= "perl" + + getDepsOf :: DepEnvironment -> (PerlMeta -> Maybe (Map PackageName (Maybe Text))) -> [Dependency] + getDepsOf env getter = map (toDependency env) (toList $ fromMaybe mempty (getter meta)) + + toDependency :: DepEnvironment -> (PackageName, Maybe Text) -> Dependency + toDependency env (pkgName, version) = + Dependency + { dependencyName = unPackageName pkgName + , dependencyType = CpanType + , dependencyVersion = CEq <$> version + , dependencyLocations = [] + , dependencyEnvironments = Set.singleton env + , dependencyTags = mempty + } diff --git a/test/App/Fossa/AnalyzeSpec.hs b/test/App/Fossa/AnalyzeSpec.hs index ba8b040ba..81ad12cad 100644 --- a/test/App/Fossa/AnalyzeSpec.hs +++ b/test/App/Fossa/AnalyzeSpec.hs @@ -16,5 +16,5 @@ spec :: Spec spec = -- this test only exists to prevent merging the commented out analyzers describe "Discovery function list" $ - it "should be length 32" $ - length (discoverFuncs :: [DiscoverFunc SomeMonad]) `shouldBe` 32 + it "should be length 33" $ + length (discoverFuncs :: [DiscoverFunc SomeMonad]) `shouldBe` 33 diff --git a/test/Perl/PerlSpec.hs b/test/Perl/PerlSpec.hs new file mode 100644 index 000000000..eb8ceb742 --- /dev/null +++ b/test/Perl/PerlSpec.hs @@ -0,0 +1,80 @@ +module Perl.PerlSpec ( + spec, +) where + +import Data.Aeson (decodeFileStrict') +import Data.Map.Strict (empty, fromList) +import Data.Set (singleton) +import Data.Text (Text) +import Data.Yaml (decodeFileEither, prettyPrintParseException) +import DepTypes +import GraphUtil (expectDeps) +import Strategy.Perl (PackageName (PackageName), PerlMeta (..), buildGraph) +import Test.Hspec ( + Spec, + describe, + expectationFailure, + it, + shouldBe, + ) + +perl :: (PackageName, Maybe Text) +perl = (PackageName "perl", Just "5.006") + +expectedContentFromV2 :: PerlMeta +expectedContentFromV2 = + PerlMeta + { version = 2.0 + , runtimeRequires = Just $ fromList [(PackageName "Carp", Just "0"), perl] + , buildRequires = Just $ fromList [perl] + , testRequires = Just $ fromList [perl] + , developRequires = Just $ fromList [(PackageName "Dist::Zilla", Just "5"), perl] + , configureRequires = Just $ fromList [(PackageName "ExtUtils::MakeMaker", Just "0"), perl] + } + +expectedContentFromV1_4 :: PerlMeta +expectedContentFromV1_4 = + PerlMeta + { version = 1.4 + , runtimeRequires = Just $ fromList [(PackageName "Archive::Zip", Just "0"), perl] + , buildRequires = Just $ fromList [(PackageName "Compress::Zlib", Just "0")] + , testRequires = Nothing + , developRequires = Nothing + , configureRequires = Just $ fromList [(PackageName "ExtUtils::MakeMaker", Just "0")] + } + +mkDependency :: Text -> Text -> DepEnvironment -> Dependency +mkDependency name version env = Dependency CpanType name (Just $ CEq version) [] (singleton env) empty + +spec :: Spec +spec = do + describe "parse" $ do + it "should parse meta json (v2) file correctly" $ do + resolvedFile <- decodeFileStrict' "test/Perl/testdata/MetaV2.json" + resolvedFile `shouldBe` Just expectedContentFromV2 + + it "should parse meta json (v1.4) file correctly" $ do + resolvedFile <- decodeFileStrict' "test/Perl/testdata/MetaV1_4.json" + resolvedFile `shouldBe` Just expectedContentFromV1_4 + + it "should parse yaml file (v2) correctly" $ do + resolvedFile <- decodeFileEither "test/Perl/testdata/MetaV2.yml" + case resolvedFile of + Left err -> expectationFailure ("failed to parse yaml file" <> show (prettyPrintParseException err)) + Right val -> val `shouldBe` Just expectedContentFromV2 + + it "should parse yaml file (v1.4) correctly" $ do + resolvedFile <- decodeFileEither "test/Perl/testdata/MetaV1_4.yml" + case resolvedFile of + Left err -> expectationFailure ("failed to parse yaml file" <> show (prettyPrintParseException err)) + Right val -> val `shouldBe` Just expectedContentFromV1_4 + + describe "buildGraph" $ + it "should build graph" $ do + let graph = buildGraph expectedContentFromV2 + let expectedDeps = + [ mkDependency "Carp" "0" EnvProduction + , mkDependency "Dist::Zilla" "5" EnvDevelopment + , mkDependency "ExtUtils::MakeMaker" "0" EnvDevelopment + ] + expectDeps expectedDeps graph diff --git a/test/Perl/testdata/MetaV1_4.json b/test/Perl/testdata/MetaV1_4.json new file mode 100644 index 000000000..09d0d966a --- /dev/null +++ b/test/Perl/testdata/MetaV1_4.json @@ -0,0 +1,24 @@ +{ + "abstract": "Some Description", + "author": [ + "Some Author" + ], + "build_requires": { + "Compress::Zlib": "0" + }, + "configure_requires": { + "ExtUtils::MakeMaker": "0" + }, + "dynamic_config": 1, + "license": "perl", + "meta-spec": { + "url": "http://module-build.sourceforge.net/META-spec-v1.4.html", + "version": "1.4" + }, + "name": "Some-App", + "requires": { + "Archive::Zip": "0", + "perl": "5.006" + }, + "version": 1.16 + } \ No newline at end of file diff --git a/test/Perl/testdata/MetaV1_4.yml b/test/Perl/testdata/MetaV1_4.yml new file mode 100644 index 000000000..74cc74a83 --- /dev/null +++ b/test/Perl/testdata/MetaV1_4.yml @@ -0,0 +1,17 @@ +abstract: 'Some Description' +author: + - 'Some Author' +build_requires: + Compress::Zlib: '0' +configure_requires: + ExtUtils::MakeMaker: '0' +dynamic_config: 1 +license: perl +meta-spec: + url: http://module-build.sourceforge.net/META-spec-v1.4.html + version: '1.4' +name: Some-App +requires: + Archive::Zip: '0' + perl: '5.006' +version: 1.16 \ No newline at end of file diff --git a/test/Perl/testdata/MetaV2.json b/test/Perl/testdata/MetaV2.json new file mode 100644 index 000000000..75e027052 --- /dev/null +++ b/test/Perl/testdata/MetaV2.json @@ -0,0 +1,40 @@ +{ + "license": [ + "mit" + ], + "meta-spec": { + "url": "http://search.cpan.org/perldoc?CPAN::Meta::Spec", + "version": 2 + }, + "name": "Some-App", + "prereqs": { + "configure": { + "requires": { + "ExtUtils::MakeMaker": "0", + "perl": "5.006" + } + }, + "develop": { + "requires": { + "Dist::Zilla": "5", + "perl": "5.006" + } + }, + "runtime": { + "requires": { + "Carp": "0", + "perl": "5.006" + } + }, + "test": { + "requires": { + "perl": "5.006" + } + }, + "build": { + "requires": { + "perl": "5.006" + } + } + } +} \ No newline at end of file diff --git a/test/Perl/testdata/MetaV2.yml b/test/Perl/testdata/MetaV2.yml new file mode 100644 index 000000000..00bdb7263 --- /dev/null +++ b/test/Perl/testdata/MetaV2.yml @@ -0,0 +1,23 @@ +name: Some-App +meta-spec: + url: http://search.cpan.org/perldoc?CPAN::Meta::Spec + version: 2 +prereqs: + configure: + requires: + ExtUtils::MakeMaker: '0' + perl: '5.006' + develop: + requires: + Dist::Zilla: '5' + perl: '5.006' + runtime: + requires: + Carp: '0' + perl: '5.006' + test: + requires: + perl: '5.006' + build: + requires: + perl: '5.006'