diff --git a/.github/workflows/crystal_build.yml b/.github/workflows/crystal_build.yml index 43638d28..4eab45be 100644 --- a/.github/workflows/crystal_build.yml +++ b/.github/workflows/crystal_build.yml @@ -2,9 +2,9 @@ name: Crystal Build on: push: - branches: [ "main" ] + branches: [ "main", "dev" ] pull_request: - branches: [ "main" ] + branches: [ "main", "dev" ] jobs: build: diff --git a/.github/workflows/crystal_lint.yml b/.github/workflows/crystal_lint.yml index 95d86969..8442e4be 100644 --- a/.github/workflows/crystal_lint.yml +++ b/.github/workflows/crystal_lint.yml @@ -2,7 +2,7 @@ name: Crystal Lint on: push: - branches: [ "main" ] + branches: [ "main", "dev" ] jobs: build: diff --git a/.github/workflows/crystal_test.yml b/.github/workflows/crystal_test.yml index 47cd657d..02eca602 100644 --- a/.github/workflows/crystal_test.yml +++ b/.github/workflows/crystal_test.yml @@ -2,9 +2,9 @@ name: Crystal Test on: push: - branches: [ "main" ] + branches: [ "main", "dev" ] pull_request: - branches: [ "main" ] + branches: [ "main", "dev" ] jobs: build: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af4cce28..c31d0c9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,10 @@ +## ❤️ Contribute +1. Write code in forked repo +2. Make Pull Request to `dev` branch +3. Finish :D + +![](https://github.com/hahwul/noir/assets/13212227/23989dab-6b4d-4f18-904f-7f5cfd172b04) + ## 🛠️ How to Build and Test? ### Clone and Install Dependencies ```bash @@ -30,16 +37,14 @@ ameba --fix # https://github.com/crystal-ameba/ameba#installation ``` -## ❤️ Contribute -1. Write code in forked repo -2. Make Pull Request -3. Finish :D - ## 🧭 Code structure -- spec: unit-test codes +- spec (for `crystal spec`) + - unit_test: unit-test codes + - functional_test: functional test codes - src - analyzer: Code analyzers for Endpoint URL and Parameter analysis - detector: Codes for language, framework identification - models: Everything for the model, such as class, structure, etc - utils: Utility codes + - etc... - noir.cr: main and command-line parser diff --git a/README.md b/README.md index 50e1f35e..c9c7c6a9 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,15 @@ | Language | Framework | URL | Method | Param | Header | WS | |----------|-----------|-----|--------|-------|--------|----| -| Go | Echo | ✅ | ✅ | ✅ | X | X | +| Go | Echo | ✅ | ✅ | ✅ | ✅ | X | +| Go | Gin | ✅ | ✅ | ✅ | ✅ | X | | Python | Django | ✅ | X | X | X | X | | Python | Flask | ✅ | X | X | X | X | | Ruby | Rails | ✅ | ✅ | ✅ | ✅ | X | | Ruby | Sinatra | ✅ | ✅ | ✅ | ✅ | X | | Php | | ✅ | ✅ | ✅ | ✅ | X | | Java | Spring | ✅ | ✅ | X | X | X | -| Java | Jsp | X | X | X | X | X | +| Java | Jsp | ✅ | ✅ | ✅ | X | X | | Crystal | Kemal | ✅ | ✅ | ✅ | ✅ | ✅ | | JS | Express | ✅ | ✅ | X | X | X | | JS | Next | X | X | X | X | X | diff --git a/shard.yml b/shard.yml index 42dcda32..c107f9e6 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: noir -version: 0.5.4 +version: 0.6.0 authors: - hahwul diff --git a/spec/functional_test/fixtures/go_echo/server.go b/spec/functional_test/fixtures/go_echo/server.go index d0ebb4e2..b7fdb117 100644 --- a/spec/functional_test/fixtures/go_echo/server.go +++ b/spec/functional_test/fixtures/go_echo/server.go @@ -13,6 +13,7 @@ func main() { }) e.GET("/pet", func(c echo.Context) error { _ = c.QueryParam("query") + _ = c.Request().Header.Get("X-API-Key") return c.String(http.StatusOK, "Hello, Pet!") }) e.POST("/pet", func(c echo.Context) error { diff --git a/spec/functional_test/fixtures/go_gin/go.mod b/spec/functional_test/fixtures/go_gin/go.mod new file mode 100644 index 00000000..2ed33e21 --- /dev/null +++ b/spec/functional_test/fixtures/go_gin/go.mod @@ -0,0 +1,32 @@ +module github.com/hahwul/test-go-app + +go 1.20 + +require ( + github.com/bytedance/sonic v1.10.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.15.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + golang.org/x/arch v0.5.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/spec/functional_test/fixtures/go_gin/public/secret.html b/spec/functional_test/fixtures/go_gin/public/secret.html new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/go_gin/server.go b/spec/functional_test/fixtures/go_gin/server.go new file mode 100644 index 00000000..0fd415b5 --- /dev/null +++ b/spec/functional_test/fixtures/go_gin/server.go @@ -0,0 +1,29 @@ +package main + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +func main() { + r := gin.Default() + r.GET("/ping", func(c *gin.Context) { + _ = c.DefaultQuery("name", "Guest") + _ = c.Query("age") + + c.JSON(http.StatusOK, gin.H{ + "message": "pong", + }) + }) + r.POST("/submit", func(c *gin.Context) { + username := c.PostForm("username") + password := c.DefaultPostForm("password", "default_password") + userAgent := c.GetHeader("User-Agent") + + c.String(http.StatusOK, "Submitted data: Username=%s, Password=%s, userAgent=%s", username, password, userAgent) + }) + + r.Static("/public", "public") + r.Run() // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") +} diff --git a/spec/functional_test/fixtures/jsp/el.jsp b/spec/functional_test/fixtures/jsp/el.jsp new file mode 100644 index 00000000..c55c8fb7 --- /dev/null +++ b/spec/functional_test/fixtures/jsp/el.jsp @@ -0,0 +1,3 @@ +<% + String username = ${param.username} +%> \ No newline at end of file diff --git a/spec/functional_test/fixtures/jsp/get_param.jsp b/spec/functional_test/fixtures/jsp/get_param.jsp new file mode 100644 index 00000000..f5f945ba --- /dev/null +++ b/spec/functional_test/fixtures/jsp/get_param.jsp @@ -0,0 +1,4 @@ +<% + String username = request.getParameter("username"); + String password = request.getParameter("password"); +%> diff --git a/spec/functional_test/fixtures/raml/docs.yaml b/spec/functional_test/fixtures/raml/docs.yaml new file mode 100644 index 00000000..b2cce8a9 --- /dev/null +++ b/spec/functional_test/fixtures/raml/docs.yaml @@ -0,0 +1,34 @@ +#%RAML 1.0 +title: User API +version: v1 + +/users/{userId}: + get: + description: Retrieve user information + queryParameters: + userId: + description: The ID of the user to retrieve + type: integer + example: 123 + headers: + Authorization: + description: API key or token for authentication + type: string + example: Bearer my_api_token + responses: + 200: + description: User information retrieved successfully + +/users: + post: + description: Create a new user + body: + application/json: + example: + { + "name": "John Doe", + "email": "johndoe@example.com" + } + responses: + 201: + description: User created successfully diff --git a/spec/functional_test/testers/go_echo_spec.cr b/spec/functional_test/testers/go_echo_spec.cr index e523032b..e3f774ef 100644 --- a/spec/functional_test/testers/go_echo_spec.cr +++ b/spec/functional_test/testers/go_echo_spec.cr @@ -4,6 +4,7 @@ extected_endpoints = [ Endpoint.new("/", "GET"), Endpoint.new("/pet", "GET", [ Param.new("query", "", "query"), + Param.new("X-API-Key", "", "header"), ]), Endpoint.new("/pet", "POST", [ Param.new("name", "", "json"), diff --git a/spec/functional_test/testers/go_gin_spec.cr b/spec/functional_test/testers/go_gin_spec.cr new file mode 100644 index 00000000..dbbd8af3 --- /dev/null +++ b/spec/functional_test/testers/go_gin_spec.cr @@ -0,0 +1,18 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("/ping", "GET", [ + Param.new("name", "", "query"), + Param.new("age", "", "query"), + ]), + Endpoint.new("/submit", "POST", [ + Param.new("username", "", "form"), + Param.new("password", "", "form"), + Param.new("User-Agent", "", "header"), + ]), +] + +FunctionalTester.new("fixtures/go_gin/", { + :techs => 1, + :endpoints => 2, +}, extected_endpoints).test_all diff --git a/spec/functional_test/testers/jsp_spec.cr b/spec/functional_test/testers/jsp_spec.cr new file mode 100644 index 00000000..43b7d0ba --- /dev/null +++ b/spec/functional_test/testers/jsp_spec.cr @@ -0,0 +1,14 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("/get_param.jsp", "GET", [ + Param.new("username", "", "query"), + Param.new("password", "", "query"), + ]), + Endpoint.new("/el.jsp", "GET", [Param.new("username", "", "query")]), +] + +FunctionalTester.new("fixtures/jsp/", { + :techs => 1, + :endpoints => 2, +}, extected_endpoints).test_all diff --git a/spec/functional_test/testers/raml_spec.cr b/spec/functional_test/testers/raml_spec.cr new file mode 100644 index 00000000..59d8d377 --- /dev/null +++ b/spec/functional_test/testers/raml_spec.cr @@ -0,0 +1,17 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("/users/{userId}", "GET", [ + Param.new("userId", "", "query"), + Param.new("Authorization", "", "header"), + ]), + Endpoint.new("/users", "POST", [ + Param.new("name", "", "json"), + Param.new("email", "", "json"), + ]), +] + +FunctionalTester.new("fixtures/raml/", { + :techs => 1, + :endpoints => 2, +}, extected_endpoints).test_all diff --git a/src/analyzer/analyzer.cr b/src/analyzer/analyzer.cr index e7d37b68..491d9645 100644 --- a/src/analyzer/analyzer.cr +++ b/src/analyzer/analyzer.cr @@ -7,12 +7,15 @@ def initialize_analyzers(logger : NoirLogger) analyzers["java_spring"] = ->analyzer_spring(Hash(Symbol, String)) analyzers["php_pure"] = ->analyzer_php_pure(Hash(Symbol, String)) analyzers["go_echo"] = ->analyzer_go_echo(Hash(Symbol, String)) + analyzers["go_gin"] = ->analyzer_go_gin(Hash(Symbol, String)) analyzers["python_flask"] = ->analyzer_flask(Hash(Symbol, String)) analyzers["python_django"] = ->analyzer_django(Hash(Symbol, String)) analyzers["js_express"] = ->analyzer_express(Hash(Symbol, String)) analyzers["crystal_kemal"] = ->analyzer_kemal(Hash(Symbol, String)) analyzers["oas2"] = ->analyzer_oas2(Hash(Symbol, String)) analyzers["oas3"] = ->analyzer_oas3(Hash(Symbol, String)) + analyzers["raml"] = ->analyzer_raml(Hash(Symbol, String)) + analyzers["java_jsp"] = ->analyzer_jsp(Hash(Symbol, String)) logger.info_sub "#{analyzers.size} Analyzers initialized" logger.debug "Analyzers:" diff --git a/src/analyzer/analyzers/analyzer_go_echo.cr b/src/analyzer/analyzers/analyzer_go_echo.cr index 1e29a014..856ee2be 100644 --- a/src/analyzer/analyzers/analyzer_go_echo.cr +++ b/src/analyzer/analyzers/analyzer_go_echo.cr @@ -35,6 +35,14 @@ class AnalyzerGoEcho < Analyzer end end end + + if line.includes?("Request().Header.Get(") + match = line.match(/Request\(\)\.Header\.Get\(\"(.*)\"\)/) + if match + header_name = match[1] + last_endpoint.params << Param.new(header_name, "", "header") + end + end end end end diff --git a/src/analyzer/analyzers/analyzer_go_gin.cr b/src/analyzer/analyzers/analyzer_go_gin.cr new file mode 100644 index 00000000..cfd2146a --- /dev/null +++ b/src/analyzer/analyzers/analyzer_go_gin.cr @@ -0,0 +1,126 @@ +require "../../models/analyzer" + +class AnalyzerGoGin < Analyzer + def analyze + # Source Analysis + public_dirs = [] of (Hash(String, String)) + Dir.glob("#{base_path}/**/*") do |path| + next if File.directory?(path) + if File.exists?(path) && File.extname(path) == ".go" + File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| + last_endpoint = Endpoint.new("", "") + file.each_line do |line| + if line.includes?(".GET(") || line.includes?(".POST(") || line.includes?(".PUT(") || line.includes?(".DELETE(") + get_route_path(line).tap do |route_path| + if route_path.size > 0 + new_endpoint = Endpoint.new("#{url}#{route_path}", line.split(".")[1].split("(")[0]) + result << new_endpoint + last_endpoint = new_endpoint + end + end + end + + ["Query", "PostForm", "GetHeader", "Static"].each do |pattern| + if line.includes?("#{pattern}(") + get_param(line).tap do |param| + if param.name.size > 0 && last_endpoint.method != "" + last_endpoint.params << param + end + end + end + end + end + end + end + end + + public_dirs.each do |p_dir| + full_path = (base_path + "/" + p_dir["file_path"]).gsub("//", "/") + Dir.glob("#{full_path}/**/*") do |path| + next if File.directory?(path) + if File.exists?(path) + if p_dir["static_path"].ends_with?("/") + p_dir["static_path"] = p_dir["static_path"][0..-2] + end + + result << Endpoint.new("#{url}#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET") + end + end + end + + Fiber.yield + + result + end + + def get_param(line : String) : Param + param_type = "json" + if line.includes?("Query(") + param_type = "query" + end + if line.includes?("PostForm(") + param_type = "form" + end + if line.includes?("GetHeader(") + param_type = "header" + end + + first = line.strip.split("(") + if first.size > 1 + second = first[1].split(")") + if second.size > 1 + if line.includes?("DefaultQuery") || line.includes?("DefaultPostForm") + param_name = second[0].split(",")[0].gsub("\"", "") + rtn = Param.new(param_name, "", param_type) + else + param_name = second[0].gsub("\"", "") + rtn = Param.new(param_name, "", param_type) + end + + return rtn + end + end + + Param.new("", "", "") + end + + def get_static_path(line : String) : Hash(String, String) + first = line.strip.split("(") + if first.size > 1 + second = first[1].split(",") + if second.size > 1 + static_path = second[0].gsub("\"", "") + file_path = second[1].gsub("\"", "").gsub(" ", "").gsub(")", "") + rtn = { + "static_path" => static_path, + "file_path" => file_path, + } + + return rtn + end + end + + { + "static_path" => "", + "file_path" => "", + } + end + + def get_route_path(line : String) : String + first = line.strip.split("(") + if first.size > 1 + second = first[1].split(",") + if second.size > 1 + route_path = second[0].gsub("\"", "") + return route_path + end + end + + "" + end +end + +def analyzer_go_gin(options : Hash(Symbol, String)) + instance = AnalyzerGoGin.new(options) + instance.analyze +end diff --git a/src/analyzer/analyzers/analyzer_jsp.cr b/src/analyzer/analyzers/analyzer_jsp.cr new file mode 100644 index 00000000..cd930c04 --- /dev/null +++ b/src/analyzer/analyzers/analyzer_jsp.cr @@ -0,0 +1,56 @@ +require "../../utils/utils.cr" +require "../../models/analyzer" + +class AnalyzerJsp < Analyzer + def analyze + # Source Analysis + Dir.glob("#{base_path}/**/*") do |path| + next if File.directory?(path) + if base_path[-1].to_s == "/" + relative_path = path.sub("#{base_path}", "").sub("./", "").sub("//", "/") + else + relative_path = path.sub("#{base_path}/", "").sub("./", "").sub("//", "/") + end + relative_path = remove_start_slash(relative_path) + + if File.exists?(path) && File.extname(path) == ".jsp" + File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| + params_query = [] of Param + + file.each_line do |line| + if line.includes? "request.getParameter" + match = line.strip.match(/request.getParameter\("(.*?)"\)/) + if match + param_name = match[1] + params_query << Param.new(param_name, "", "query") + end + end + + if line.includes? "${param." + match = line.strip.match(/\$\{param\.(.*?)\}/) + if match + param_name = match[1] + params_query << Param.new(param_name, "", "query") + end + end + rescue + next + end + result << Endpoint.new("#{url}/#{relative_path}", "GET", params_query) + end + end + end + Fiber.yield + + result + end + + def allow_patterns + ["$_GET", "$_POST", "$_REQUEST", "$_SERVER"] + end +end + +def analyzer_jsp(options : Hash(Symbol, String)) + instance = AnalyzerJsp.new(options) + instance.analyze +end diff --git a/src/analyzer/analyzers/analyzer_raml.cr b/src/analyzer/analyzers/analyzer_raml.cr new file mode 100644 index 00000000..60c67597 --- /dev/null +++ b/src/analyzer/analyzers/analyzer_raml.cr @@ -0,0 +1,62 @@ +require "../../models/analyzer" + +class AnalyzerRAML < Analyzer + def analyze + locator = CodeLocator.instance + raml_spec = locator.get("raml-spec") + + if !raml_spec.nil? + if File.exists?(raml_spec) + content = File.read(raml_spec, encoding: "utf-8", invalid: :skip) + yaml_obj = YAML.parse(content) + yaml_obj.as_h.each do |path, path_obj| + begin + path_obj.as_h.each do |method, method_obj| + params = [] of Param + + if method_obj.as_h.has_key? "queryParameters" + method_obj["queryParameters"].as_h.each do |param_name, _| + param = Param.new(param_name.to_s, "", "query") + params << param + end + end + + if method_obj.as_h.has_key? "body" + method_obj["body"].as_h.each do |content_type, content_obj| + if content_type == "application/json" + content_obj["example"].as_h.each do |param_name, _| + param = Param.new(param_name.to_s, "", "json") + params << param + end + elsif content_type == "application/x-www-form-urlencoded" + content_obj["example"].as_h.each do |param_name, _| + param = Param.new(param_name.to_s, "", "form") + params << param + end + end + end + end + + if method_obj.as_h.has_key? "headers" + method_obj["headers"].as_h.each do |param_name, _| + param = Param.new(param_name.to_s, "", "header") + params << param + end + end + + @result << Endpoint.new(path.to_s, method.to_s.upcase, params) + end + rescue + end + end + end + end + + @result + end +end + +def analyzer_raml(options : Hash(Symbol, String)) + instance = AnalyzerRAML.new(options) + instance.analyze +end diff --git a/src/detector/detector.cr b/src/detector/detector.cr index 1efe1c00..684a81fa 100644 --- a/src/detector/detector.cr +++ b/src/detector/detector.cr @@ -15,7 +15,8 @@ def detect_techs(base_path : String, options : Hash(Symbol, String), logger : No defind_detectors([ DetectorCrystalKemal, DetectorGoEcho, DetectorJavaJsp, DetectorJavaSpring, DetectorJsExpress, DetectorPhpPure, DetectorPythonDjango, DetectorPythonFlask, - DetectorRubyRails, DetectorRubySinatra, DetectorOas2, DetectorOas3, + DetectorRubyRails, DetectorRubySinatra, DetectorOas2, DetectorOas3, DetectorRAML, + DetectorGoGin, ]) Dir.glob("#{base_path}/**/*") do |file| spawn do diff --git a/src/detector/detectors/go_gin.cr b/src/detector/detectors/go_gin.cr new file mode 100644 index 00000000..90fddbf9 --- /dev/null +++ b/src/detector/detectors/go_gin.cr @@ -0,0 +1,15 @@ +require "../../models/detector" + +class DetectorGoGin < Detector + def detect(filename : String, file_contents : String) : Bool + if (filename.includes? "go.mod") && (file_contents.includes? "github.com/gin-gonic/gin") + true + else + false + end + end + + def set_name + @name = "go_gin" + end +end diff --git a/src/detector/detectors/raml.cr b/src/detector/detectors/raml.cr new file mode 100644 index 00000000..dc29b7f2 --- /dev/null +++ b/src/detector/detectors/raml.cr @@ -0,0 +1,26 @@ +require "../../models/detector" +require "../../utils/yaml" +require "../../models/code_locator" + +class DetectorRAML < Detector + def detect(filename : String, file_contents : String) : Bool + check = false + if valid_yaml? file_contents + if file_contents.includes? "#%RAML" + begin + YAML.parse(file_contents) + check = true + locator = CodeLocator.instance + locator.set("raml-spec", filename) + rescue + end + end + end + + check + end + + def set_name + @name = "raml" + end +end diff --git a/src/noir.cr b/src/noir.cr index 1d2ddd68..0d42dc55 100644 --- a/src/noir.cr +++ b/src/noir.cr @@ -6,7 +6,7 @@ require "./options.cr" require "./techs/techs.cr" module Noir - VERSION = "0.5.4" + VERSION = "0.6.0" end noir_options = default_options() diff --git a/src/techs/techs.cr b/src/techs/techs.cr index 4e3c298f..209475a8 100644 --- a/src/techs/techs.cr +++ b/src/techs/techs.cr @@ -10,6 +10,11 @@ module NoirTechs :framework => "Echo", :similar => ["echo", "go-echo", "go_echo"], }, + :go_gin => { + :language => "Go", + :framework => "Gin", + :similar => ["gin", "go-gin", "go_gin"], + }, :java_jsp => { :language => "Java", :framework => "JSP", @@ -58,6 +63,10 @@ module NoirTechs :format => ["JSON", "YAML"], :similar => ["oas 3.0", "oas_3_0"], }, + :raml => { + :format => ["YAML"], + :similar => ["raml"], + }, } def self.get_techs