Add Ruby Support (#13)

* add test and testfiles

* implement ruby(bundler) Gemfile parsing

* add test target in makefile

* update test to include ruby

* add check for bundler binary

* remove debug code

* test all packages and add -cover to github go test

.github/workflows/go.yml

@@ -29,4 +29,4 @@ jobs:
       run: go build -v .
 
     - name: Test
-      run: go test -v .
+      run: go test -v ./... -cover

Makefile

@@ -1,2 +1,7 @@
+.PHONY: test
+
 build:
 	go build cmd/deplist.go
+
+test:
+	go test ./... -cover -covermode=atomic

deplist.go

@@ -33,6 +33,10 @@ func init() {
 	if _, err := exec.LookPath("mvn"); err != nil {
 		log.Fatal("maven is required")
 	}
+
+	if _, err := exec.LookPath("bundle"); err != nil {
+		log.Fatal("bundler gem is required")
+	}
 }
 
 // GetDeps scans a given repository and returns all dependencies found in a DependencyList struct.
@@ -47,6 +51,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 
 	pomPath := filepath.Join(fullPath, "pom.xml")
 	goPath := filepath.Join(fullPath, "go.mod")
+	rubyPath := filepath.Join(fullPath, "Gemfile.lock")
 
 	// point at the parent repo, but can't assume where the indicators will be
 	err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
@@ -125,7 +130,26 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							Version: strings.Replace(version, "v", "", 1),
 						})
 				}
+			case rubyPath:
+				pkgs, err := scan.GetRubyDeps(path)
+				if err != nil {
+					return err
+				}
+
+				if len(pkgs) > 0 {
+					foundTypes.DepFoundAddFlag(LangRuby)
+				}
+
+				for name, version := range pkgs {
+					deps = append(deps,
+						Dependency{
+							DepType: LangRuby,
+							Path:    name,
+							Version: strings.Replace(version, "v", "", 1),
+						})
+				}
 			}
+
 		}
 		return nil
 	})

deplist_test.go

@@ -107,6 +107,82 @@ func BuildWant() []Dependency {
 		"tether",
 	}
 
+	rubySet := []string{
+		"fluent-plugin-kafka",
+		"fluent-plugin-rewrite-tag-filter",
+		"faraday",
+		"concurrent-ruby",
+		"elasticsearch",
+		"sigdump",
+		"syslog_protocol",
+		"uuidtools",
+		"aws-partitions",
+		"http-cookie",
+		"ltsv",
+		"quantile",
+		"connection_pool",
+		"tzinfo-data",
+		"unf",
+		"aws-sdk-core",
+		"fluent-plugin-cloudwatch-logs",
+		"fluent-plugin-kubernetes_metadata_filter",
+		"http-form_data",
+		"jmespath",
+		"kubeclient",
+		"msgpack",
+		"rest-client",
+		"dig_rb",
+		"unf_ext",
+		"to_regexp",
+		"cool.io",
+		"ethon",
+		"ffi",
+		"fluent-plugin-remote-syslog",
+		"lru_redux",
+		"prometheus-client",
+		"typhoeus",
+		"aws-sdk-cloudwatchlogs",
+		"fluent-plugin-prometheus",
+		"mime-types",
+		"mime-types-data",
+		"public_suffix",
+		"domain_name",
+		"aws-sigv4",
+		"elasticsearch-api",
+		"excon",
+		"fluentd",
+		"http_parser.rb",
+		"netrc",
+		"recursive-open-struct",
+		"aws-eventstream",
+		"systemd-journal",
+		"net-http-persistent",
+		"elasticsearch-transport",
+		"ffi-compiler",
+		"fluent-plugin-record-modifier",
+		"http-accept",
+		"http-parser",
+		"rake",
+		"digest-crc",
+		"fluent-plugin-splunk-hec",
+		"fluent-plugin-systemd",
+		"multi_json",
+		"multipart-post",
+		"ruby-kafka",
+		"strptime",
+		"fluent-plugin-concat",
+		"serverengine",
+		"fluent-plugin-multi-format-parser",
+		"tzinfo",
+		"fluent-mixin-config-placeholders",
+		"jsonpath",
+		"fluent-config-regexp-type",
+		"fluent-plugin-elasticsearch",
+		"http",
+		"yajl-ruby",
+		"addressable",
+	}
+
 	for _, n := range golangPaths {
 		d := Dependency{
 			DepType: 1,
@@ -135,6 +211,14 @@ func BuildWant() []Dependency {
 		}
 		deps = append(deps, d)
 	}
+
+	for _, n := range rubySet {
+		d := Dependency{
+			DepType: LangRuby,
+			Path:    n,
+		}
+		deps = append(deps, d)
+	}
 	return deps
 }
 
@@ -143,8 +227,8 @@ func TestGetDeps(t *testing.T) {
 
 	got, gotBitmask, _ := GetDeps("test/testRepo")
 
-	if gotBitmask != 7 {
-		t.Errorf("GotBitmask() != 7")
+	if gotBitmask != 23 {
+		t.Errorf("GotBitmask() != 7; got: %d", gotBitmask)
 	}
 
 	// iterate thru and compare

internal/scan/ruby.go

@@ -0,0 +1,45 @@
+package scan
+
+import (
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func GetRubyDeps(path string) (map[string]string, error) {
+	gathered = make(map[string]string)
+
+	dirPath := filepath.Dir(path)
+
+	//Make sure that the Gemfile we are loading is supported by the version of bundle currently installed.
+	cmd := exec.Command("bundle", "update", "--bundler")
+	cmd.Dir = dirPath
+	_, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd = exec.Command("bundle", "list")
+
+	cmd.Dir = dirPath
+
+	data, err := cmd.Output()
+
+	splitOutput := strings.Split(string(data), "\n")
+
+	for _, line := range splitOutput {
+		if !strings.HasPrefix(line, "  *") {
+			continue
+		}
+		rawDep := strings.TrimPrefix(line, "  * ")
+		dep := strings.Split(rawDep, " ")
+		dep[1] = dep[1][1 : len(dep[1])-1]
+		gathered[dep[0]] = dep[1]
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return gathered, nil
+}

internal/scan/ruby_test.go

@@ -0,0 +1,99 @@
+package scan
+
+import (
+	"testing"
+)
+
+var want map[string]string = map[string]string{
+	"fluent-plugin-splunk-hec":          "1.1.2",
+	"http-form_data":                    "2.3.0",
+	"prometheus-client":                 "0.9.0",
+	"tzinfo":                            "2.0.2",
+	"fluent-plugin-rewrite-tag-filter":  "2.3.0",
+	"fluent-plugin-kafka":               "0.13.1",
+	"fluent-plugin-prometheus":          "1.7.3",
+	"http_parser.rb":                    "0.6.0",
+	"lru_redux":                         "1.1.0",
+	"rest-client":                       "2.1.0",
+	"tzinfo-data":                       "1.2020.1",
+	"unf":                               "0.1.4",
+	"aws-eventstream":                   "1.1.0",
+	"yajl-ruby":                         "1.4.1",
+	"jmespath":                          "1.4.0",
+	"http-cookie":                       "1.0.3",
+	"fluent-plugin-multi-format-parser": "1.0.0",
+	"elasticsearch":                     "7.8.0",
+	"fluent-plugin-elasticsearch":       "4.1.1",
+	"fluent-plugin-kubernetes_metadata_filter": "2.5.2",
+	"dig_rb":                           "1.0.1",
+	"elasticsearch-transport":          "7.8.0",
+	"ffi":                              "1.11.3",
+	"mime-types":                       "3.3.1",
+	"to_regexp":                        "0.2.1",
+	"aws-sdk-cloudwatchlogs":           "1.38.0",
+	"msgpack":                          "1.3.3",
+	"typhoeus":                         "1.4.0",
+	"digest-crc":                       "0.6.1",
+	"http-parser":                      "1.2.1",
+	"ruby-kafka":                       "1.1.0",
+	"serverengine":                     "2.2.1",
+	"http":                             "4.4.1",
+	"ethon":                            "0.12.0",
+	"multipart-post":                   "2.1.1",
+	"concurrent-ruby":                  "1.1.6",
+	"fluent-plugin-concat":             "2.4.0",
+	"public_suffix":                    "4.0.5",
+	"sigdump":                          "0.2.4",
+	"syslog_protocol":                  "0.9.2",
+	"aws-sigv4":                        "1.2.2",
+	"fluent-plugin-record-modifier":    "2.1.0",
+	"addressable":                      "2.7.0",
+	"elasticsearch-api":                "7.8.0",
+	"excon":                            "0.75.0",
+	"fluent-plugin-cloudwatch-logs":    "0.7.6",
+	"fluent-plugin-systemd":            "1.0.2",
+	"ltsv":                             "0.1.2",
+	"quantile":                         "0.2.1",
+	"recursive-open-struct":            "1.1.2",
+	"cool.io":                          "1.6.0",
+	"unf_ext":                          "0.0.7.7",
+	"strptime":                         "0.2.4",
+	"fluent-plugin-remote-syslog":      "1.1",
+	"fluent-config-regexp-type":        "1.0.0",
+	"fluent-mixin-config-placeholders": "0.4.0",
+	"fluentd":                          "1.7.4",
+	"http-accept":                      "1.7.0",
+	"systemd-journal":                  "1.3.3",
+	"uuidtools":                        "2.1.5",
+	"aws-sdk-core":                     "3.109.3",
+	"connection_pool":                  "2.2.3",
+	"domain_name":                      "0.5.20190701",
+	"faraday":                          "1.0.1",
+	"jsonpath":                         "1.0.5",
+	"mime-types-data":                  "3.2020.0512",
+	"netrc":                            "0.11.0",
+	"aws-partitions":                   "1.396.0",
+	"kubeclient":                       "4.8.0",
+	"multi_json":                       "1.15.0",
+	"net-http-persistent":              "3.1.0",
+	"rake":                             "13.0.1",
+	"ffi-compiler":                     "1.0.1",
+}
+
+func Test_GetRubyDeps(t *testing.T) {
+	got, err := GetRubyDeps("../../test/testRepo/")
+
+	if err != nil {
+		t.Errorf("GetRubyDeps() error thrown: %+v", err)
+	}
+
+	for k, v := range want {
+		if got[k] != v {
+			t.Errorf("GetRubyDeps() - deps missing entry: %s = %s ", k, v)
+		}
+	}
+
+	if len(got) != len(want) {
+		t.Errorf("GetRubyDeps() = %d; want %d", len(got), len(want))
+	}
+}

test/testRepo/Gemfile

@@ -0,0 +1,25 @@
+source "https://rubygems.org"
+gem 'fluentd', '1.7.4', :source=>"https://rubygems.org"
+gem 'elasticsearch-transport'
+gem 'elasticsearch-api'
+gem 'elasticsearch'
+gem 'fluent-plugin-kubernetes_metadata_filter'
+gem 'fluent-plugin-cloudwatch-logs'
+gem 'fluent-plugin-concat'
+gem 'fluent-plugin-elasticsearch'
+gem 'fluent-plugin-kafka'
+gem 'fluent-plugin-multi-format-parser'
+gem 'fluent-plugin-record-modifier'
+gem 'fluent-plugin-rewrite-tag-filter'
+gem 'fluent-plugin-systemd'
+gem 'fluent-plugin-remote-syslog'
+gem 'fluent-plugin-prometheus'
+gem 'fluent-plugin-splunk-hec'
+gem 'typhoeus'
+
+#Below gem(and two of its dependencies) are moved to O_A_L/fluentd/lib, but its dependencies still exist in O_A_L/fluentd/vendored_gem_src/
+#gem 'fluent-plugin-remote_syslog', '1.0.0'
+
+gem 'ffi', '1.11.3'
+gem 'uuidtools', '2.1.5'
+gem 'rake'