From c9638fe3121c6e91ee02118796b7e5a2a60e6936 Mon Sep 17 00:00:00 2001 From: Ekaterina Rogushkova Date: Fri, 22 May 2020 19:34:27 +0300 Subject: [PATCH] Initial commit --- .gitignore | 17 +++ LICENSE.md | 87 +++++++++++++ README.md | 59 +++++++++ cmd/testrail-cli/testrail-cli.go | 82 ++++++++++++ example_test.log | 59 +++++++++ go.mod | 9 ++ package1/example_test.go | 49 +++++++ test_objects.go | 214 +++++++++++++++++++++++++++++++ testrail.go | 71 ++++++++++ utils.go | 45 +++++++ 10 files changed, 692 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 cmd/testrail-cli/testrail-cli.go create mode 100644 example_test.log create mode 100644 go.mod create mode 100644 package1/example_test.go create mode 100644 test_objects.go create mode 100644 testrail.go create mode 100644 utils.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2fc1550 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +.idea/ + +# Dependency directories (remove the comment below to include it) +/vendor/ diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..12be72b --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,87 @@ +## Insolar License version 1.0 + +https://github.com/insolar/insolar/blob/master/LICENSE.md + +### Acceptance + +In order to get any license under these terms, you must agree to them as both strict obligations and conditions +to all your licenses. + +### Copyright License + +The licensor grants you a copyright license for the software to do everything you might do with the software +that would otherwise infringe the licensor's copyright in it for any permitted purpose. However, you may only +make changes or new works based on the software according to Changes and New Works License, and you may only +use the software as described in Limitations. + +### Changes and New Works License + +The licensor grants you an additional copyright license to make changes and new works based on the software +for any permitted purpose. + +### Patent License + +The licensor grants you a patent license for the software that covers patent claims the licensor can license, +or becomes able to license, that you would infringe by using the software as allowed in this license. + +### Limitations + +You may only use the software for your internal business purposes, and you must not use the software +to provide any service, including any software-as-a-service, to anyone outside your company. + +This license is also intended to enable you to build applications for Insolar Mainnet, a public blockchain +network developed by licensor based on the software. If you build an application that interfaces with the +Insolar Mainnet, and adds substantial value to the software or Insolar Mainnet, you may provide +that application as a service to others outside your company. + +### Notices + +You must ensure that any copy you make of any part of the software includes a copy of these terms or the URL +for them above, as well as copies of any plain-text lines beginning with Required Notice: that the licensor +provided with the software. For example: + + Required Notice: Copyright 2020 Insolar Network Ltd. (https://github.com/insolar/insolar/blob/master/network/LICENSE.md) + +### No Other Rights + +These terms do not allow you to sublicense or transfer any of your licenses to anyone else, or prevent +the licensor from granting licenses to anyone else. These terms do not imply any other licenses. + +### Patent Challenge and Assertion + +Your license for the software ends immediately if your company does any of the following, or assists +anyone else to do so: you make any written claim that the software infringes or contributes to infringement +of any patent, or you challenge the validity of any patent licensed above, or you market or sell a product +or a service that competes, directly or indirectly, with the software or any product or service of licensor +based on the software. + +### Violations + +The first time you are notified in writing that you have violated any of these terms, or done anything +with the software not covered by your licenses, your licenses can nonetheless continue if you come into +full compliance with these terms, and take practical steps to correct past violations, within 30 days +of receiving notice. Otherwise, all your licenses end immediately. + +### No Liability + +_**As far as the law allows, the software comes as is, without any warranty or condition, and the licensor +will not be liable to you for any damages arising out of these terms or the use or nature of the software, +under any kind of legal claim.**_ + +### Definitions + +The **licensor** is Insolar Network Ltd. + +The **software** is the software the licensor makes available under these terms. + +**You** refers to the individual or entity agreeing to these terms. + +**Your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, +plus all organizations that have control over, are under the control of, or are under common control +with that organization. **Control** means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, contract, or otherwise. +Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the software under these terms. + +**Use** means anything you do with the software requiring one of your licenses. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..bb2aa5e --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +#### Go test2json integration to TestRail + +``` +go get github.com/insolar/testrail-cli +go install cmd/testrail-cli/testrail-cli.go +``` + +#### Usage +Every test MUST log testrail case in format +```go +func TestExample(t *testing.T) { + t.Log("C3605 Some testcase description") + ... +} +``` + +```go +func TestSuite(t *testing.T) { + t.Run("TestExample", func(t *testing.T) { + t.Log("C3605 Some testcase description"") + ... + }) +} +``` + +If you want to skip test, you can add issue to skip description +```go +func TestExample3(t *testing.T) { + t.Log("C3607 Some testcase description") + t.Skip("https://example.net/browse/TASK-1 other description") +} +``` + +#### Run +| Param key | Env key | Description | +| ------------- | ------------- | --------------------------- | +| --URL | TR_URL | testrail url | +| --USER | TR_USER | testrail user | +| --PASSWORD | TR_PASSWORD | testrail password | +| --RUN_ID | TR_RUN_ID | testrail run id | +| --FILE | TR_FILE | go test json file | +| --SKIP-DESC | SKIP-DESC | skip description check flag | + +Use params +``` +testrail-cli --URL=https://example.testrail.com/ --USER=example@gmail.com --PASSWORD=${pass} --RUN_ID=57 --FILE=example_test.log +``` +Or env vars with TR prefix +``` +TR_URL=https://example.testrail.com/ TR_USER=example@gmail.com TR_PASSWORD=${pass} TR_RUN_ID=57 TR_FILE=example_test_suite.json testrail-cli +``` +Also you can pipe json in +``` +go test ./... -json | testrail-cli --URL=https://example.testrail.com/ --USER=example@gmail.com --PASSWORD=${pass} --RUN_ID=57 +``` +Or save file using tee for debug +``` +go test ./... -json | tee autotest.log | testrail-cli --URL=https://example.testrail.com/ --USER=example@gmail.com --PASSWORD=${pass} --RUN_ID=57 +``` diff --git a/cmd/testrail-cli/testrail-cli.go b/cmd/testrail-cli/testrail-cli.go new file mode 100644 index 0000000..1911370 --- /dev/null +++ b/cmd/testrail-cli/testrail-cli.go @@ -0,0 +1,82 @@ +// Copyright 2020 Insolar Network Ltd. +// All rights reserved. +// This material is licensed under the Insolar License version 1.0, +// available at https://github.com/insolar/testrail-cli/LICENSE.md. + +package main + +import ( + "flag" + tr "github.com/insolar/testrail-cli" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "io" + "log" + "os" +) + +func main() { + + viper.AutomaticEnv() + viper.SetEnvPrefix("TR") + flag.String("URL", "", "testrail url") + flag.String("USER", "", "testrail username") + flag.String("PASSWORD", "", "testrail password/token") + flag.String("FILE", "", "go test json file") + flag.Int("RUN_ID", 0, "testrail run id") + flag.Bool("SKIP-DESC", false, "skip description check") + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + + url := viper.GetString("URL") + user := viper.GetString("USER") + pass := viper.GetString("PASSWORD") + runID := viper.GetInt("RUN_ID") + file := viper.GetString("FILE") + skipDesc := viper.GetBool("SKIP-DESC") + + if url == "" { + log.Fatal("provide TestRail url") + } + if runID == 0 { + log.Fatal("provide run id, ex.: --RUN_ID=54, or env TR_RUN_ID=54") + } + if user == "" { + log.Fatal("provide user for TestRail authentication") + } + if pass == "" { + log.Fatal("provide password/token for TestRail authentication") + } + + var stream io.Reader + if file != "" { + f, err := os.Open(file) + if err != nil { + log.Fatal(err) + } + //defer f.Close() + stream = f + } else { + stream = os.Stdin + } + events := tr.ReadFile(stream) + + t := tr.NewTestRail(url, user, pass) + run := t.GetRun(runID) + casesWithDescs := t.GetCasesWithDescs(run.ProjectID, run.SuiteID) + // update all cases with N/A status, we store all autotests in ONE run, so in case + // someone delete particular case implementation status must be updated to N/A + untested := t.NAResults(casesWithDescs) + t.UpdateRunForCases(runID, untested) + + testEventsBatch := t.GroupEventsByTest(events) + tObjects := t.EventsToTestObjects(testEventsBatch) + filteredObjects := tr.FilterTestObjects(tObjects, casesWithDescs, skipDesc) + tr.LogInvalidTests(filteredObjects) + + sendableResults := t.TestObjectsToSendableResultsForCase(filteredObjects.Valid) + if sendableResults != nil { + t.UpdateRunForCases(runID, sendableResults) + } +} diff --git a/example_test.log b/example_test.log new file mode 100644 index 0000000..e019a95 --- /dev/null +++ b/example_test.log @@ -0,0 +1,59 @@ +{"Time":"2020-05-22T15:00:50.396397+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample"} +{"Time":"2020-05-22T15:00:50.396686+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample","Output":"=== RUN TestExample\n"} +{"Time":"2020-05-22T15:00:50.396705+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample","Output":"=== PAUSE TestExample\n"} +{"Time":"2020-05-22T15:00:50.396709+03:00","Action":"pause","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample"} +{"Time":"2020-05-22T15:00:50.396716+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2"} +{"Time":"2020-05-22T15:00:50.396721+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2","Output":"=== RUN TestExample2\n"} +{"Time":"2020-05-22T15:00:50.396725+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2","Output":"=== PAUSE TestExample2\n"} +{"Time":"2020-05-22T15:00:50.396728+03:00","Action":"pause","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2"} +{"Time":"2020-05-22T15:00:50.396733+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample3"} +{"Time":"2020-05-22T15:00:50.396736+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample3","Output":"=== RUN TestExample3\n"} +{"Time":"2020-05-22T15:00:50.901681+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample3","Output":" TestExample3: example_test.go:28: C3607 Skip test\n"} +{"Time":"2020-05-22T15:00:50.901733+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample3","Output":" TestExample3: example_test.go:29: sdfs\n"} +{"Time":"2020-05-22T15:00:50.90177+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample3","Output":"--- SKIP: TestExample3 (0.51s)\n"} +{"Time":"2020-05-22T15:00:50.901779+03:00","Action":"skip","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample3","Elapsed":0.51} +{"Time":"2020-05-22T15:00:50.901799+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members"} +{"Time":"2020-05-22T15:00:50.901803+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members","Output":"=== RUN TestMGRGroupCreateWith2Members\n"} +{"Time":"2020-05-22T15:00:50.901818+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members","Output":"=== PAUSE TestMGRGroupCreateWith2Members\n"} +{"Time":"2020-05-22T15:00:50.901824+03:00","Action":"pause","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members"} +{"Time":"2020-05-22T15:00:50.901829+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence"} +{"Time":"2020-05-22T15:00:50.901834+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence","Output":"=== RUN TestMGRGroupCreateCheckEmptySequence\n"} +{"Time":"2020-05-22T15:00:50.901844+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence","Output":"=== PAUSE TestMGRGroupCreateCheckEmptySequence\n"} +{"Time":"2020-05-22T15:00:50.901849+03:00","Action":"pause","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence"} +{"Time":"2020-05-22T15:00:50.901859+03:00","Action":"cont","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample"} +{"Time":"2020-05-22T15:00:50.901865+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample","Output":"=== CONT TestExample\n"} +{"Time":"2020-05-22T15:00:50.901877+03:00","Action":"cont","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence"} +{"Time":"2020-05-22T15:00:50.901882+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence","Output":"=== CONT TestMGRGroupCreateCheckEmptySequence\n"} +{"Time":"2020-05-22T15:00:50.901892+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=200"} +{"Time":"2020-05-22T15:00:50.901898+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=200","Output":"=== RUN TestMGRGroupCreateCheckEmptySequence/groupGoal=200\n"} +{"Time":"2020-05-22T15:00:50.901953+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=200","Output":" TestMGRGroupCreateCheckEmptySequence/groupGoal=200: example_test.go:47: C3702 Create group of 3 members with groupGoal=200 and check empty sequence\n"} +{"Time":"2020-05-22T15:00:50.901986+03:00","Action":"cont","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members"} +{"Time":"2020-05-22T15:00:50.901995+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members","Output":"=== CONT TestMGRGroupCreateWith2Members\n"} +{"Time":"2020-05-22T15:00:50.902003+03:00","Action":"cont","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2"} +{"Time":"2020-05-22T15:00:50.902009+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2","Output":"=== CONT TestExample2\n"} +{"Time":"2020-05-22T15:00:50.902018+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=300"} +{"Time":"2020-05-22T15:00:50.902024+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=300","Output":"=== RUN TestMGRGroupCreateCheckEmptySequence/groupGoal=300\n"} +{"Time":"2020-05-22T15:00:50.90203+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=300","Output":" TestMGRGroupCreateCheckEmptySequence/groupGoal=300: example_test.go:47: C3704 Create group of 3 members with groupGoal=300 and check empty sequence\n"} +{"Time":"2020-05-22T15:00:50.902062+03:00","Action":"run","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=400"} +{"Time":"2020-05-22T15:00:50.902075+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=400","Output":"=== RUN TestMGRGroupCreateCheckEmptySequence/groupGoal=400\n"} +{"Time":"2020-05-22T15:00:50.902084+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=400","Output":" TestMGRGroupCreateCheckEmptySequence/groupGoal=400: example_test.go:47: C3696 Create group of 3 members with groupGoal=400 and check empty sequence\n"} +{"Time":"2020-05-22T15:00:50.902114+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence","Output":"--- PASS: TestMGRGroupCreateCheckEmptySequence (0.00s)\n"} +{"Time":"2020-05-22T15:00:50.902133+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=200","Output":" --- PASS: TestMGRGroupCreateCheckEmptySequence/groupGoal=200 (0.00s)\n"} +{"Time":"2020-05-22T15:00:50.902138+03:00","Action":"pass","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=200","Elapsed":0} +{"Time":"2020-05-22T15:00:50.902144+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=300","Output":" --- PASS: TestMGRGroupCreateCheckEmptySequence/groupGoal=300 (0.00s)\n"} +{"Time":"2020-05-22T15:00:50.90217+03:00","Action":"pass","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=300","Elapsed":0} +{"Time":"2020-05-22T15:00:50.90218+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=400","Output":" --- PASS: TestMGRGroupCreateCheckEmptySequence/groupGoal=400 (0.00s)\n"} +{"Time":"2020-05-22T15:00:51.005646+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=400","Output":" TestExample: example_test.go:16: C9999 Pass test\n"} +{"Time":"2020-05-22T15:00:51.005689+03:00","Action":"pass","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence/groupGoal=400","Elapsed":0} +{"Time":"2020-05-22T15:00:51.005697+03:00","Action":"pass","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateCheckEmptySequence","Elapsed":0} +{"Time":"2020-05-22T15:00:51.005701+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample","Output":"--- PASS: TestExample (0.10s)\n"} +{"Time":"2020-05-22T15:00:51.206031+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample","Output":" TestExample2: example_test.go:22: C3606 Fail testsdf\n"} +{"Time":"2020-05-22T15:00:51.206056+03:00","Action":"pass","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample","Elapsed":0.1} +{"Time":"2020-05-22T15:00:51.206063+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2","Output":"--- FAIL: TestExample2 (0.30s)\n"} +{"Time":"2020-05-22T15:00:51.90453+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2","Output":" TestMGRGroupCreateWith2Members: example_test.go:35: C3703 Error creating group of 2 members\n"} +{"Time":"2020-05-22T15:00:51.904564+03:00","Action":"fail","Package":"github.com/insolar/testrail-cli/package1","Test":"TestExample2","Elapsed":0.3} +{"Time":"2020-05-22T15:00:51.904574+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members","Output":"--- PASS: TestMGRGroupCreateWith2Members (1.00s)\n"} +{"Time":"2020-05-22T15:00:51.904582+03:00","Action":"pass","Package":"github.com/insolar/testrail-cli/package1","Test":"TestMGRGroupCreateWith2Members","Elapsed":1} +{"Time":"2020-05-22T15:00:51.904587+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Output":"FAIL\n"} +{"Time":"2020-05-22T15:00:51.905226+03:00","Action":"output","Package":"github.com/insolar/testrail-cli/package1","Output":"FAIL\tgithub.com/insolar/testrail-cli/package1\t1.750s\n"} +{"Time":"2020-05-22T15:00:51.905257+03:00","Action":"fail","Package":"github.com/insolar/testrail-cli/package1","Elapsed":1.75} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3340cdf --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/insolar/testrail-cli + +go 1.12 + +require ( + github.com/educlos/testrail v0.0.0-20190627213040-ca1b25409ae2 + github.com/spf13/pflag v1.0.3 + github.com/spf13/viper v1.6.2 +) diff --git a/package1/example_test.go b/package1/example_test.go new file mode 100644 index 0000000..859ba87 --- /dev/null +++ b/package1/example_test.go @@ -0,0 +1,49 @@ +// Copyright 2020 Insolar Network Ltd. +// All rights reserved. +// This material is licensed under the Insolar License version 1.0, +// available at https://github.com/insolar/testrail-cli/LICENSE.md. + +package testrail_cli + +import ( + "testing" + "time" +) + +func TestExample(t *testing.T) { + t.Parallel() + time.Sleep(100 * time.Millisecond) + t.Log("C9999 Pass test") +} + +func TestExample2(t *testing.T) { + t.Parallel() + time.Sleep(300 * time.Millisecond) + t.Log("C3606 Fail testsdf") + t.Fail() +} + +func TestExample3(t *testing.T) { + time.Sleep(500 * time.Millisecond) + t.Log("C3607 Skip test") + t.Skip("sdfs") +} + +func TestMGRGroupCreateWith2Members(t *testing.T) { + t.Parallel() + time.Sleep(1 * time.Second) + t.Log("C3703 Error creating group of 2 members") +} + +func TestMGRGroupCreateCheckEmptySequence(t *testing.T) { + t.Parallel() + checkGroup(t, "200", []string{"100", "100", "100"}, "C3702 Create group of 3 members with groupGoal=200 and check empty sequence") + checkGroup(t, "300", []string{"100", "100", "100"}, "C3704 Create group of 3 members with groupGoal=300 and check empty sequence") + checkGroup(t, "400", []string{"100", "100", "100"}, "C3696 Create group of 3 members with groupGoal=400 and check empty sequence") +} + +func checkGroup(t *testing.T, groupGoal string, userGoals []string, testName string) { + t.Run("groupGoal="+groupGoal, func(t *testing.T) { + t.Log(testName) + }) +} diff --git a/test_objects.go b/test_objects.go new file mode 100644 index 0000000..bbb97fa --- /dev/null +++ b/test_objects.go @@ -0,0 +1,214 @@ +// Copyright 2020 Insolar Network Ltd. +// All rights reserved. +// This material is licensed under the Insolar License version 1.0, +// available at https://github.com/insolar/testrail-cli/LICENSE.md. + +package testrail_cli + +import ( + "github.com/educlos/testrail" + "log" + "strconv" + "time" +) + +// TestMatcher represents data differences between implementation and testrail case +type TestMatcher struct { + Status string + CaseID int + Desc string + TRDesc string + GoTestName string + IssueURL string +} + +type TestObjectSummary struct { + Valid []*TestMatcher + NotFound []*TestMatcher + WrongDesc []*TestMatcher + SkippedNoIssue []*TestMatcher +} + +// TestEvent go test2json event object +type TestEvent struct { + Time time.Time // encodes as an RFC3339-format string + Action string + Package string + Test string + Elapsed float64 // seconds + Output string +} + +// EventsToTestObjects parses event batches to construct TestObjects, extracting caseID, Description, Status and IssueURL +func (m *TestRail) EventsToTestObjects(events map[string][]*TestEvent) []*TestMatcher { + tests := make([]*TestMatcher, 0) + for _, eventsBatch := range events { + t := &TestMatcher{} + for _, e := range eventsBatch { + if e.Action == "output" { + t.GoTestName = e.Test + res := testCaseIdRe.FindAllStringSubmatch(e.Output, -1) + if len(res) != 0 && len(res[0]) == 3 { + d, err := strconv.Atoi(res[0][1]) + if err != nil { + log.Fatal(err) + } + //TODO: Bad, should harden regex instead, debatable for now + if t.CaseID != 0 { + continue + } + t.CaseID = d + t.Desc = res[0][2] + } + res = testStatusRe.FindAllStringSubmatch(e.Output, -1) + if len(res) != 0 && len(res[0]) == 2 { + t.Status = res[0][1] + } + res = testSkipIssueRe.FindAllStringSubmatch(e.Output, -1) + if len(res) != 0 && len(res[0]) == 2 { + t.IssueURL = res[0][1] + } + } + } + tests = append(tests, t) + } + return tests +} + +func UniqTestKey(e *TestEvent) string { + return e.Test + "|" + e.Package +} + +// GroupEventsByTest groups test2json events by test + package key +func (m *TestRail) GroupEventsByTest(events []*TestEvent) map[string][]*TestEvent { + eventsByTest := make(map[string][]*TestEvent) + testNames := make(map[string]int) + for _, e := range events { + if _, ok := testNames[UniqTestKey(e)]; !ok { + testNames[UniqTestKey(e)] = 1 + } + } + for uniqTest := range testNames { + for _, e := range events { + if UniqTestKey(e) == uniqTest { + eventsByTest[uniqTest] = append(eventsByTest[uniqTest], e) + } + } + } + return eventsByTest +} + +// JSONEventsToSendable convert events to test rail sendable format +func (m *TestRail) JSONEventsToSendable(events []*TestEvent) *testrail.SendableResultsForCase { + testEventsBatch := m.GroupEventsByTest(events) + tObjects := m.EventsToTestObjects(testEventsBatch) + return m.TestObjectsToSendableResultsForCase(tObjects) +} + +// TestObjectsToSendableResultsForCase converts TestObjects to sendable results +func (m *TestRail) TestObjectsToSendableResultsForCase(objs []*TestMatcher) *testrail.SendableResultsForCase { + if len(objs) == 0 { + log.Println("no valid tests found matching cases, skip sending") + return nil + } + results := make([]testrail.ResultsForCase, 0) + for _, o := range objs { + result := testrail.ResultsForCase{ + CaseID: o.CaseID, + SendableResult: testrail.SendableResult{ + AssignedToID: autotestUserID, + StatusID: statusMap[o.Status], + Comment: "", + Version: "1", + Elapsed: *testrail.TimespanFromDuration(1 * time.Second), + Defects: TicketFromURL(o.IssueURL), + }, + } + results = append(results, result) + } + sendableResults := testrail.SendableResultsForCase{ + Results: results, + } + return &sendableResults +} + +// NAResults generate payload to set all test rail run results to UNTESTED +func (m *TestRail) NAResults(cases []*CaseWithDesc) *testrail.SendableResultsForCase { + results := make([]testrail.ResultsForCase, 0) + for _, c := range cases { + result := testrail.ResultsForCase{ + CaseID: c.CaseID, + SendableResult: testrail.SendableResult{ + AssignedToID: autotestUserID, + StatusID: statusMap["N/A"], + Comment: "", + Version: "1", + Elapsed: *testrail.TimespanFromDuration(1 * time.Second), + Defects: "", + }, + } + results = append(results, result) + } + sendableResults := &testrail.SendableResultsForCase{ + Results: results, + } + return sendableResults +} + +func LogInvalidTests(objs *TestObjectSummary) { + if len(objs.NotFound) > 0 { + log.Println("Tests without testrail case ID:") + for _, o := range objs.NotFound { + log.Printf(" %s", o.GoTestName) + } + } + if len(objs.WrongDesc) > 0 { + log.Println("Test title discrepancy with testrail test-case title:") + for _, o := range objs.WrongDesc { + log.Printf(" %s", o.GoTestName) + log.Printf(" Test: %s", o.Desc) + log.Printf(" Testrail: %s", o.TRDesc) + log.Printf(" Testcase: %s", TRTicket(o.CaseID)) + } + } + if len(objs.SkippedNoIssue) > 0 { + log.Println("Skipped tests without issue:") + for _, o := range objs.SkippedNoIssue { + log.Printf(" %s", o.GoTestName) + } + } +} + +// FilterTestObjects split test objects into groups: valid/not found/wrong description +func FilterTestObjects(objs []*TestMatcher, cases []*CaseWithDesc, skipDesc bool) *TestObjectSummary { + wrongDescObjs := make([]*TestMatcher, 0) + skipNoIssue := make([]*TestMatcher, 0) + notFoundObjs := make([]*TestMatcher, 0) + validObjs := make([]*TestMatcher, 0) + for _, o := range objs { + found := false + for _, c := range cases { + if o.CaseID == c.CaseID { + found = true + if !skipDesc && o.Desc != c.Desc { + o.TRDesc = c.Desc + wrongDescObjs = append(wrongDescObjs, o) + continue + } + validObjs = append(validObjs, o) + } + } + if !found { + notFoundObjs = append(notFoundObjs, o) + } + if o.Status == "SKIP" && o.IssueURL == "" { + skipNoIssue = append(skipNoIssue, o) + } + } + return &TestObjectSummary{ + Valid: validObjs, + NotFound: notFoundObjs, + WrongDesc: wrongDescObjs, + SkippedNoIssue: skipNoIssue, + } +} diff --git a/testrail.go b/testrail.go new file mode 100644 index 0000000..7ec7c9d --- /dev/null +++ b/testrail.go @@ -0,0 +1,71 @@ +// Copyright 2020 Insolar Network Ltd. +// All rights reserved. +// This material is licensed under the Insolar License version 1.0, +// available at https://github.com/insolar/testrail-cli/LICENSE.md. + +package testrail_cli + +import ( + "github.com/educlos/testrail" + "log" + "regexp" +) + +var ( + autotestUserID = 10 + statusMap = map[string]int{ + "PASS": 1, + "FAIL": 5, + "N/A": 7, + "SKIP": 6, + } + testStatusRe = regexp.MustCompile(`--- (.*):`) + testSkipIssueRe = regexp.MustCompile(`insolar\.atlassian\.net/browse/([A-Z]+-\d+)`) + testCaseIdRe = regexp.MustCompile(`C(\d{1,8})\s(.*)`) +) + +type TestRail struct { + c *testrail.Client +} + +func NewTestRail(url string, user string, password string) *TestRail { + return &TestRail{ + c: testrail.NewClient(url, user, password), + } +} + +type CaseWithDesc struct { + CaseID int + Desc string +} + +func (m *TestRail) GetCasesWithDescs(projectID int, suiteID int) []*CaseWithDesc { + casesWithDescs := make([]*CaseWithDesc, 0) + cases, err := m.c.GetCases(projectID, suiteID) + if err != nil { + log.Fatal(err) + } + for _, c := range cases { + casesWithDescs = append(casesWithDescs, &CaseWithDesc{ + CaseID: c.ID, + Desc: c.Title, + }) + } + return casesWithDescs +} + +func (m *TestRail) GetRun(id int) testrail.Run { + run, err := m.c.GetRun(id) + if err != nil { + log.Fatal(err) + } + return run +} + +func (m *TestRail) UpdateRunForCases(runId int, results *testrail.SendableResultsForCase) []testrail.Result { + res, err := m.c.AddResultsForCases(runId, *results) + if err != nil { + log.Fatal(err) + } + return res +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..401ceab --- /dev/null +++ b/utils.go @@ -0,0 +1,45 @@ +// Copyright 2020 Insolar Network Ltd. +// All rights reserved. +// This material is licensed under the Insolar License version 1.0, +// available at https://github.com/insolar/testrail-cli/LICENSE.md. + +package testrail_cli + +import ( + "bufio" + "encoding/json" + "github.com/spf13/viper" + "io" + "log" + "path" + "strconv" + "strings" +) + +func TicketFromURL(url string) string { + if strings.HasPrefix(url, "https") || strings.HasPrefix(url, "http") { + s := strings.Split(url, "/") + return s[len(s)-1] + } + return url +} + +func TRTicket(id int) string { + return path.Join(viper.GetString("URL"), "/index.php?/cases/view/", strconv.Itoa(id)) +} + +func ReadFile(stream io.Reader) []*TestEvent { + testEvents := make([]*TestEvent, 0) + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + var te *TestEvent + if err := json.Unmarshal([]byte(scanner.Text()), &te); err != nil { + log.Fatalf("failed to unmarshal test event json: %s\n", err) + } + testEvents = append(testEvents, te) + } + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + return testEvents +}