From 71f045fc33b8750ade631fb2b00e308ad20e880f Mon Sep 17 00:00:00 2001 From: Anten Skrabec Date: Tue, 2 Mar 2021 17:49:25 -0700 Subject: [PATCH] Add Ruby Support (#13) * add test and testfiles * implement ruby(bundler) Gemfile parsing * add test target in makefile * update test to include ruby * add check for bundler binary * remove debug code * test all packages and add -cover to github go test --- .github/workflows/go.yml | 2 +- Makefile | 5 + deplist.go | 24 +++++ deplist_test.go | 88 ++++++++++++++++- internal/scan/ruby.go | 45 +++++++++ internal/scan/ruby_test.go | 99 +++++++++++++++++++ test/testRepo/Gemfile | 25 +++++ test/testRepo/Gemfile.lock | 188 +++++++++++++++++++++++++++++++++++++ 8 files changed, 473 insertions(+), 3 deletions(-) create mode 100644 internal/scan/ruby.go create mode 100644 internal/scan/ruby_test.go create mode 100644 test/testRepo/Gemfile create mode 100644 test/testRepo/Gemfile.lock diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 01acf86..e293ae7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -29,4 +29,4 @@ jobs: run: go build -v . - name: Test - run: go test -v . + run: go test -v ./... -cover diff --git a/Makefile b/Makefile index c84b0b6..b7a8b7b 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,7 @@ +.PHONY: test + build: go build cmd/deplist.go + +test: + go test ./... -cover -covermode=atomic diff --git a/deplist.go b/deplist.go index 21a7306..1b91a1a 100644 --- a/deplist.go +++ b/deplist.go @@ -33,6 +33,10 @@ func init() { if _, err := exec.LookPath("mvn"); err != nil { log.Fatal("maven is required") } + + if _, err := exec.LookPath("bundle"); err != nil { + log.Fatal("bundler gem is required") + } } // GetDeps scans a given repository and returns all dependencies found in a DependencyList struct. @@ -47,6 +51,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) { pomPath := filepath.Join(fullPath, "pom.xml") goPath := filepath.Join(fullPath, "go.mod") + rubyPath := filepath.Join(fullPath, "Gemfile.lock") // point at the parent repo, but can't assume where the indicators will be err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error { @@ -125,7 +130,26 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) { Version: strings.Replace(version, "v", "", 1), }) } + case rubyPath: + pkgs, err := scan.GetRubyDeps(path) + if err != nil { + return err + } + + if len(pkgs) > 0 { + foundTypes.DepFoundAddFlag(LangRuby) + } + + for name, version := range pkgs { + deps = append(deps, + Dependency{ + DepType: LangRuby, + Path: name, + Version: strings.Replace(version, "v", "", 1), + }) + } } + } return nil }) diff --git a/deplist_test.go b/deplist_test.go index f0ec1e2..36969ff 100644 --- a/deplist_test.go +++ b/deplist_test.go @@ -107,6 +107,82 @@ func BuildWant() []Dependency { "tether", } + rubySet := []string{ + "fluent-plugin-kafka", + "fluent-plugin-rewrite-tag-filter", + "faraday", + "concurrent-ruby", + "elasticsearch", + "sigdump", + "syslog_protocol", + "uuidtools", + "aws-partitions", + "http-cookie", + "ltsv", + "quantile", + "connection_pool", + "tzinfo-data", + "unf", + "aws-sdk-core", + "fluent-plugin-cloudwatch-logs", + "fluent-plugin-kubernetes_metadata_filter", + "http-form_data", + "jmespath", + "kubeclient", + "msgpack", + "rest-client", + "dig_rb", + "unf_ext", + "to_regexp", + "cool.io", + "ethon", + "ffi", + "fluent-plugin-remote-syslog", + "lru_redux", + "prometheus-client", + "typhoeus", + "aws-sdk-cloudwatchlogs", + "fluent-plugin-prometheus", + "mime-types", + "mime-types-data", + "public_suffix", + "domain_name", + "aws-sigv4", + "elasticsearch-api", + "excon", + "fluentd", + "http_parser.rb", + "netrc", + "recursive-open-struct", + "aws-eventstream", + "systemd-journal", + "net-http-persistent", + "elasticsearch-transport", + "ffi-compiler", + "fluent-plugin-record-modifier", + "http-accept", + "http-parser", + "rake", + "digest-crc", + "fluent-plugin-splunk-hec", + "fluent-plugin-systemd", + "multi_json", + "multipart-post", + "ruby-kafka", + "strptime", + "fluent-plugin-concat", + "serverengine", + "fluent-plugin-multi-format-parser", + "tzinfo", + "fluent-mixin-config-placeholders", + "jsonpath", + "fluent-config-regexp-type", + "fluent-plugin-elasticsearch", + "http", + "yajl-ruby", + "addressable", + } + for _, n := range golangPaths { d := Dependency{ DepType: 1, @@ -135,6 +211,14 @@ func BuildWant() []Dependency { } deps = append(deps, d) } + + for _, n := range rubySet { + d := Dependency{ + DepType: LangRuby, + Path: n, + } + deps = append(deps, d) + } return deps } @@ -143,8 +227,8 @@ func TestGetDeps(t *testing.T) { got, gotBitmask, _ := GetDeps("test/testRepo") - if gotBitmask != 7 { - t.Errorf("GotBitmask() != 7") + if gotBitmask != 23 { + t.Errorf("GotBitmask() != 7; got: %d", gotBitmask) } // iterate thru and compare diff --git a/internal/scan/ruby.go b/internal/scan/ruby.go new file mode 100644 index 0000000..e4c21ff --- /dev/null +++ b/internal/scan/ruby.go @@ -0,0 +1,45 @@ +package scan + +import ( + "os/exec" + "path/filepath" + "strings" +) + +func GetRubyDeps(path string) (map[string]string, error) { + gathered = make(map[string]string) + + dirPath := filepath.Dir(path) + + //Make sure that the Gemfile we are loading is supported by the version of bundle currently installed. + cmd := exec.Command("bundle", "update", "--bundler") + cmd.Dir = dirPath + _, err := cmd.Output() + if err != nil { + return nil, err + } + + cmd = exec.Command("bundle", "list") + + cmd.Dir = dirPath + + data, err := cmd.Output() + + splitOutput := strings.Split(string(data), "\n") + + for _, line := range splitOutput { + if !strings.HasPrefix(line, " *") { + continue + } + rawDep := strings.TrimPrefix(line, " * ") + dep := strings.Split(rawDep, " ") + dep[1] = dep[1][1 : len(dep[1])-1] + gathered[dep[0]] = dep[1] + } + + if err != nil { + return nil, err + } + + return gathered, nil +} diff --git a/internal/scan/ruby_test.go b/internal/scan/ruby_test.go new file mode 100644 index 0000000..99c8373 --- /dev/null +++ b/internal/scan/ruby_test.go @@ -0,0 +1,99 @@ +package scan + +import ( + "testing" +) + +var want map[string]string = map[string]string{ + "fluent-plugin-splunk-hec": "1.1.2", + "http-form_data": "2.3.0", + "prometheus-client": "0.9.0", + "tzinfo": "2.0.2", + "fluent-plugin-rewrite-tag-filter": "2.3.0", + "fluent-plugin-kafka": "0.13.1", + "fluent-plugin-prometheus": "1.7.3", + "http_parser.rb": "0.6.0", + "lru_redux": "1.1.0", + "rest-client": "2.1.0", + "tzinfo-data": "1.2020.1", + "unf": "0.1.4", + "aws-eventstream": "1.1.0", + "yajl-ruby": "1.4.1", + "jmespath": "1.4.0", + "http-cookie": "1.0.3", + "fluent-plugin-multi-format-parser": "1.0.0", + "elasticsearch": "7.8.0", + "fluent-plugin-elasticsearch": "4.1.1", + "fluent-plugin-kubernetes_metadata_filter": "2.5.2", + "dig_rb": "1.0.1", + "elasticsearch-transport": "7.8.0", + "ffi": "1.11.3", + "mime-types": "3.3.1", + "to_regexp": "0.2.1", + "aws-sdk-cloudwatchlogs": "1.38.0", + "msgpack": "1.3.3", + "typhoeus": "1.4.0", + "digest-crc": "0.6.1", + "http-parser": "1.2.1", + "ruby-kafka": "1.1.0", + "serverengine": "2.2.1", + "http": "4.4.1", + "ethon": "0.12.0", + "multipart-post": "2.1.1", + "concurrent-ruby": "1.1.6", + "fluent-plugin-concat": "2.4.0", + "public_suffix": "4.0.5", + "sigdump": "0.2.4", + "syslog_protocol": "0.9.2", + "aws-sigv4": "1.2.2", + "fluent-plugin-record-modifier": "2.1.0", + "addressable": "2.7.0", + "elasticsearch-api": "7.8.0", + "excon": "0.75.0", + "fluent-plugin-cloudwatch-logs": "0.7.6", + "fluent-plugin-systemd": "1.0.2", + "ltsv": "0.1.2", + "quantile": "0.2.1", + "recursive-open-struct": "1.1.2", + "cool.io": "1.6.0", + "unf_ext": "0.0.7.7", + "strptime": "0.2.4", + "fluent-plugin-remote-syslog": "1.1", + "fluent-config-regexp-type": "1.0.0", + "fluent-mixin-config-placeholders": "0.4.0", + "fluentd": "1.7.4", + "http-accept": "1.7.0", + "systemd-journal": "1.3.3", + "uuidtools": "2.1.5", + "aws-sdk-core": "3.109.3", + "connection_pool": "2.2.3", + "domain_name": "0.5.20190701", + "faraday": "1.0.1", + "jsonpath": "1.0.5", + "mime-types-data": "3.2020.0512", + "netrc": "0.11.0", + "aws-partitions": "1.396.0", + "kubeclient": "4.8.0", + "multi_json": "1.15.0", + "net-http-persistent": "3.1.0", + "rake": "13.0.1", + "ffi-compiler": "1.0.1", +} + +func Test_GetRubyDeps(t *testing.T) { + got, err := GetRubyDeps("../../test/testRepo/") + + if err != nil { + t.Errorf("GetRubyDeps() error thrown: %+v", err) + } + + for k, v := range want { + if got[k] != v { + t.Errorf("GetRubyDeps() - deps missing entry: %s = %s ", k, v) + } + } + + if len(got) != len(want) { + t.Errorf("GetRubyDeps() = %d; want %d", len(got), len(want)) + } +} diff --git a/test/testRepo/Gemfile b/test/testRepo/Gemfile new file mode 100644 index 0000000..97e46b5 --- /dev/null +++ b/test/testRepo/Gemfile @@ -0,0 +1,25 @@ +source "https://rubygems.org" +gem 'fluentd', '1.7.4', :source=>"https://rubygems.org" +gem 'elasticsearch-transport' +gem 'elasticsearch-api' +gem 'elasticsearch' +gem 'fluent-plugin-kubernetes_metadata_filter' +gem 'fluent-plugin-cloudwatch-logs' +gem 'fluent-plugin-concat' +gem 'fluent-plugin-elasticsearch' +gem 'fluent-plugin-kafka' +gem 'fluent-plugin-multi-format-parser' +gem 'fluent-plugin-record-modifier' +gem 'fluent-plugin-rewrite-tag-filter' +gem 'fluent-plugin-systemd' +gem 'fluent-plugin-remote-syslog' +gem 'fluent-plugin-prometheus' +gem 'fluent-plugin-splunk-hec' +gem 'typhoeus' + +#Below gem(and two of its dependencies) are moved to O_A_L/fluentd/lib, but its dependencies still exist in O_A_L/fluentd/vendored_gem_src/ +#gem 'fluent-plugin-remote_syslog', '1.0.0' + +gem 'ffi', '1.11.3' +gem 'uuidtools', '2.1.5' +gem 'rake' diff --git a/test/testRepo/Gemfile.lock b/test/testRepo/Gemfile.lock new file mode 100644 index 0000000..8361270 --- /dev/null +++ b/test/testRepo/Gemfile.lock @@ -0,0 +1,188 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + aws-eventstream (1.1.0) + aws-partitions (1.396.0) + aws-sdk-cloudwatchlogs (1.38.0) + aws-sdk-core (~> 3, >= 3.109.0) + aws-sigv4 (~> 1.1) + aws-sdk-core (3.109.3) + aws-eventstream (~> 1, >= 1.0.2) + aws-partitions (~> 1, >= 1.239.0) + aws-sigv4 (~> 1.1) + jmespath (~> 1.0) + aws-sigv4 (1.2.2) + aws-eventstream (~> 1, >= 1.0.2) + concurrent-ruby (1.1.6) + connection_pool (2.2.3) + cool.io (1.6.0) + dig_rb (1.0.1) + digest-crc (0.6.1) + rake (~> 13.0) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + elasticsearch (7.8.0) + elasticsearch-api (= 7.8.0) + elasticsearch-transport (= 7.8.0) + elasticsearch-api (7.8.0) + multi_json + elasticsearch-transport (7.8.0) + faraday (~> 1) + multi_json + ethon (0.12.0) + ffi (>= 1.3.0) + excon (0.75.0) + faraday (1.0.1) + multipart-post (>= 1.2, < 3) + ffi (1.11.3) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-mixin-config-placeholders (0.4.0) + fluentd + uuidtools (>= 2.1.5) + fluent-plugin-cloudwatch-logs (0.7.6) + aws-sdk-cloudwatchlogs (~> 1.0) + fluentd (>= 0.14.15) + fluent-plugin-concat (2.4.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-elasticsearch (4.1.1) + elasticsearch + excon + fluentd (>= 0.14.22) + fluent-plugin-kafka (0.13.1) + fluentd (>= 0.10.58, < 2) + ltsv + ruby-kafka (>= 0.7.8, < 2) + fluent-plugin-kubernetes_metadata_filter (2.5.2) + fluentd (>= 0.14.0, < 1.12) + kubeclient (< 5) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-prometheus (1.7.3) + fluentd (>= 0.14.20, < 2) + prometheus-client (< 0.10) + fluent-plugin-record-modifier (2.1.0) + fluentd (>= 1.0, < 2) + fluent-plugin-remote-syslog (1.1) + fluent-mixin-config-placeholders + fluentd + syslog_protocol + fluent-plugin-rewrite-tag-filter (2.3.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-splunk-hec (1.1.2) + fluentd (~> 1.4) + multi_json (~> 1.13) + net-http-persistent (~> 3.0) + fluent-plugin-systemd (1.0.2) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 1.3.2) + fluentd (1.7.4) + cool.io (>= 1.4.5, < 2.0.0) + dig_rb (~> 1.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 1.2.0, < 2.0.0) + serverengine (>= 2.0.4, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (~> 2.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http (4.4.1) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + http-parser (~> 1.2.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.3.0) + http-parser (1.2.1) + ffi-compiler (>= 1.0, < 2.0) + http_parser.rb (0.6.0) + jmespath (1.4.0) + jsonpath (1.0.5) + multi_json + to_regexp (~> 0.2.1) + kubeclient (4.8.0) + http (>= 3.0, < 5.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + lru_redux (1.1.0) + ltsv (0.1.2) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2020.0512) + msgpack (1.3.3) + multi_json (1.15.0) + multipart-post (2.1.1) + net-http-persistent (3.1.0) + connection_pool (~> 2.2) + netrc (0.11.0) + prometheus-client (0.9.0) + quantile (~> 0.2.1) + public_suffix (4.0.5) + quantile (0.2.1) + rake (13.0.1) + recursive-open-struct (1.1.2) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + ruby-kafka (1.1.0) + digest-crc + serverengine (2.2.1) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.4) + syslog_protocol (0.9.2) + systemd-journal (1.3.3) + ffi (~> 1.9) + to_regexp (0.2.1) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (2.0.2) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2020.1) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + uuidtools (2.1.5) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + elasticsearch + elasticsearch-api + elasticsearch-transport + ffi (= 1.11.3) + fluent-plugin-cloudwatch-logs + fluent-plugin-concat + fluent-plugin-elasticsearch + fluent-plugin-kafka + fluent-plugin-kubernetes_metadata_filter + fluent-plugin-multi-format-parser + fluent-plugin-prometheus + fluent-plugin-record-modifier + fluent-plugin-remote-syslog + fluent-plugin-rewrite-tag-filter + fluent-plugin-splunk-hec + fluent-plugin-systemd + fluentd (= 1.7.4)! + rake + typhoeus + uuidtools (= 2.1.5) + +BUNDLED WITH + 2.1.4