From 33c3051da13b5abe85a17fbdb790508e1184bac2 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 16 Dec 2024 19:39:04 +0000 Subject: [PATCH 1/5] Promote JUnit output 'terraform test' feature from experimental status --- internal/command/test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/internal/command/test.go b/internal/command/test.go index 993fce4f2397..38ee31042b9b 100644 --- a/internal/command/test.go +++ b/internal/command/test.go @@ -103,23 +103,6 @@ func (c *TestCommand) Run(rawArgs []string) int { view := views.NewTest(args.ViewType, c.View) var junitXMLView *views.TestJUnitXMLFile if args.JUnitXMLFile != "" { - // JUnit XML output is currently experimental, so that we can gather - // feedback on exactly how we should map the test results to this - // JUnit-oriented format before anyone starts depending on it for real. - if !c.AllowExperimentalFeatures { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "JUnit XML output is not available", - "The -junit-xml option is currently experimental and therefore available only in alpha releases of Terraform CLI.", - )) - view.Diagnostics(nil, nil, diags) - return 1 - } - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Warning, - "JUnit XML output is experimental", - "The -junit-xml option is currently experimental and therefore subject to breaking changes or removal, even in patch releases.", - )) junitXMLView = views.NewTestJUnitXMLFile(args.JUnitXMLFile) view = views.TestMulti{ view, From 4a21385cf137a228ef11684e904b472862f9648d Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 6 Jan 2025 16:00:00 +0000 Subject: [PATCH 2/5] Update `test` docs to include the `-junit-xml` flag --- website/docs/cli/commands/test.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/docs/cli/commands/test.mdx b/website/docs/cli/commands/test.mdx index 6c6898ba6221..5b9a10a88908 100644 --- a/website/docs/cli/commands/test.mdx +++ b/website/docs/cli/commands/test.mdx @@ -30,6 +30,8 @@ The following options apply to the Terraform `test` command: * `-json` - Displays machine-readable JSON output for your testing results. +* `-junit-xml=` - Saves testing results output in JUnit XML format to the specified file. The file path must be relative or absolute. + * `-test-directory=` - Overrides the directory that Terraform looks into for test files. Note that Terraform always loads testing files within the main configuration directory. The default testing directory is `tests`. * `-verbose` - Prints out the plan or state for each `run` block within a test file, based on the `command` attribute of each `run` block. From 60b01d1d0ccc20d04eb52b36b4fdfc73b410a530 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 6 Jan 2025 16:00:45 +0000 Subject: [PATCH 3/5] Add output format section to the `test` docs --- website/docs/cli/commands/test.mdx | 134 +++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/website/docs/cli/commands/test.mdx b/website/docs/cli/commands/test.mdx index 5b9a10a88908..84221e43d728 100644 --- a/website/docs/cli/commands/test.mdx +++ b/website/docs/cli/commands/test.mdx @@ -134,3 +134,137 @@ The testing directory must be beneath the main configuration directory, but it c > Note: Test files within the root configuration directory are always loaded, regardless of the `-test-directory` value. We do not recommend changing the default test directory. The option for customization is included for configuration authors who may have included a `tests` submodule in their configuration before the `terraform test` command was released. In general, the default test directory of `tests` should always be used. + +## Example: Test Output Format Options + +Below is a contrived example of Terraform testing that makes assertions about the values of local variables `true` and `false`. +There are two test files: one contains a passing test, and one contains a failing test. + +```hcl +# main.tf + +locals { + true = "true" + false = "true" # incorrect, should be "false"! +} +``` + +The assertion that local.true == "true" in example_1.tftest.hcl will pass: + +```hcl +# example_1.tftest.hcl + +run "true_is_true" { + assert { + condition = local.true == "true" + error_message = "local.true did not match expected value" + } +} +``` + +The assertion that local.false == "false" in example_2.tftest.hcl will fail: + +```hcl +# example_2.tftest.hcl + +run "false_is_false" { + assert { + condition = local.false == "false" + error_message = "local.false did not match expected value" + } +} +``` + +### General features of test output + +The `test` command will print output to the terminal: +* to show progress running tests across all files +* to display any error messages resulting from failing test assertions +* finally, to summarize the results across all files + +Below is the output of `terraform test` when testing the example files above. When no additional flags are supplied, the command's output defaults to a human-readable format: + +```shell +example_1.tftest.hcl... in progress + run "true_is_true"... pass +example_1.tftest.hcl... tearing down +example_1.tftest.hcl... pass +example_2.tftest.hcl... in progress + run "false_is_false"... fail +╷ +│ Error: Test assertion failed +│ +│ on example_2.tftest.hcl line 21, in run "false_is_false": +│ 21: condition = local.false == "false" +│ ├──────────────── +│ │ local.false is "true" +│ +│ local.false did not match expected value +╵ +example_2.tftest.hcl... tearing down +example_2.tftest.hcl... fail + +Failure! 1 passed, 1 failed. +``` + +### Test output in JSON format + +Below is the output of the `terraform test -json` command using the same example files. The -json flag changes terminal output to be in a machine-readable format. The content of the output is otherwise unchanged: + +```shell +{"@level":"info","@message":"Terraform 1.11.0-dev","@module":"terraform.ui","@timestamp":"2025-01-06T12:19:01.903603Z","terraform":"1.11.0-dev","type":"version","ui":"1.2"} +{"@level":"info","@message":"Found 2 files and 2 run blocks","@module":"terraform.ui","@timestamp":"2025-01-06T12:19:02.229367Z","test_abstract":{"example_1.tftest.hcl":["true_is_true"],"example_2.tftest.hcl":["false_is_false"]},"type":"test_abstract"} +{"@level":"info","@message":"example_1.tftest.hcl... in progress","@module":"terraform.ui","@testfile":"example_1.tftest.hcl","@timestamp":"2025-01-06T12:19:02.229432Z","test_file":{"path":"example_1.tftest.hcl","progress":"starting"},"type":"test_file"} +{"@level":"info","@message":" \"true_is_true\"... in progress","@module":"terraform.ui","@testfile":"example_1.tftest.hcl","@testrun":"true_is_true","@timestamp":"2025-01-06T12:19:02.229458Z","test_run":{"path":"example_1.tftest.hcl","run":"true_is_true","progress":"starting","elapsed":0},"type":"test_run"} +{"@level":"info","@message":" \"true_is_true\"... pass","@module":"terraform.ui","@testfile":"example_1.tftest.hcl","@testrun":"true_is_true","@timestamp":"2025-01-06T12:19:02.230577Z","test_run":{"path":"example_1.tftest.hcl","run":"true_is_true","progress":"complete","status":"pass"},"type":"test_run"} +{"@level":"info","@message":"example_1.tftest.hcl... tearing down","@module":"terraform.ui","@testfile":"example_1.tftest.hcl","@timestamp":"2025-01-06T12:19:02.230590Z","test_file":{"path":"example_1.tftest.hcl","progress":"teardown"},"type":"test_file"} +{"@level":"info","@message":"example_1.tftest.hcl... pass","@module":"terraform.ui","@testfile":"example_1.tftest.hcl","@timestamp":"2025-01-06T12:19:02.230598Z","test_file":{"path":"example_1.tftest.hcl","progress":"complete","status":"pass"},"type":"test_file"} +{"@level":"info","@message":"example_2.tftest.hcl... in progress","@module":"terraform.ui","@testfile":"example_2.tftest.hcl","@timestamp":"2025-01-06T12:19:02.230605Z","test_file":{"path":"example_2.tftest.hcl","progress":"starting"},"type":"test_file"} +{"@level":"info","@message":" \"false_is_false\"... in progress","@module":"terraform.ui","@testfile":"example_2.tftest.hcl","@testrun":"false_is_false","@timestamp":"2025-01-06T12:19:02.230642Z","test_run":{"path":"example_2.tftest.hcl","run":"false_is_false","progress":"starting","elapsed":0},"type":"test_run"} +{"@level":"info","@message":" \"false_is_false\"... fail","@module":"terraform.ui","@testfile":"example_2.tftest.hcl","@testrun":"false_is_false","@timestamp":"2025-01-06T12:19:02.231331Z","test_run":{"path":"example_2.tftest.hcl","run":"false_is_false","progress":"complete","status":"fail"},"type":"test_run"} +{"@level":"error","@message":"Error: Test assertion failed","@module":"terraform.ui","@testfile":"example_2.tftest.hcl","@testrun":"false_is_false","@timestamp":"2025-01-06T12:19:02.231485Z","diagnostic":{"severity":"error","summary":"Test assertion failed","detail":"local.false did not match expected value","range":{"filename":"example_2.tftest.hcl","start":{"line":21,"column":21,"byte":334},"end":{"line":21,"column":43,"byte":356}},"snippet":{"context":"run \"false_is_false\"","code":" condition = local.false == \"false\"","start_line":21,"highlight_start_offset":20,"highlight_end_offset":42,"values":[{"traversal":"local.false","statement":"is \"true\""}]}},"type":"diagnostic"} +{"@level":"info","@message":"example_2.tftest.hcl... tearing down","@module":"terraform.ui","@testfile":"example_2.tftest.hcl","@timestamp":"2025-01-06T12:19:02.231552Z","test_file":{"path":"example_2.tftest.hcl","progress":"teardown"},"type":"test_file"} +{"@level":"info","@message":"example_2.tftest.hcl... fail","@module":"terraform.ui","@testfile":"example_2.tftest.hcl","@timestamp":"2025-01-06T12:19:02.231557Z","test_file":{"path":"example_2.tftest.hcl","progress":"complete","status":"fail"},"type":"test_file"} +{"@level":"info","@message":"Failure! 1 passed, 1 failed.","@module":"terraform.ui","@timestamp":"2025-01-06T12:19:02.231581Z","test_summary":{"status":"fail","passed":1,"failed":1,"errored":0,"skipped":0},"type":"test_summary"} +``` + +### Test output in JUnit XML format, saved to file + +Below is the output of the `terraform test -junit-xml=./output.xml` command using the same example files. +The test output is: + +* printed to the terminal in the default, human-readable format, as already described above +* also saved in JUnit XML format in the specified file + +Below is the contents of the file `output.xml`: + +```xml + + + + + + + + + + + +``` + +The file maps Terraform test command concepts to JUnit XML format according to the table below: + +| Terraform test concept | Element in JUnit XML output | Notes | +| -----------------------| ----------------------------|--------------| +| Test directory | `` | A single `` element wraps all other elements in the XML output, as `terraform test` only processes a single test directory. | +| Test file | `` | There will be as many `` elements as there are test files. The tests in a `` are summarized in its attributes: `name` is the file name; `tests` is total count of tests; `skipped` is a count of skipped tests; `failures` is a count of failed tests, `errors` is a count of errored tests. | +| Run block | `` | Any `` elements will be nested inside a parental `` element corresponding to the file that contains the run block. | +| Test failure error | `` | If a `` fails then it will contain a `` element that contains a message about a test failure. | +| ??? | `` | ??? | From 269fe751643e0de63975d4d097579fa8b8feb974 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 6 Jan 2025 16:55:49 +0000 Subject: [PATCH 4/5] Refactor how data about XML elements is presented, small formatting changes --- website/docs/cli/commands/test.mdx | 41 ++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/website/docs/cli/commands/test.mdx b/website/docs/cli/commands/test.mdx index 84221e43d728..6e02a12f27ca 100644 --- a/website/docs/cli/commands/test.mdx +++ b/website/docs/cli/commands/test.mdx @@ -178,9 +178,9 @@ run "false_is_false" { ### General features of test output The `test` command will print output to the terminal: -* to show progress running tests across all files -* to display any error messages resulting from failing test assertions -* finally, to summarize the results across all files +* To show progress running tests across all files. +* To display any error messages resulting from failing test assertions. +* Finally, to summarize the results across all files. Below is the output of `terraform test` when testing the example files above. When no additional flags are supplied, the command's output defaults to a human-readable format: @@ -233,10 +233,10 @@ Below is the output of the `terraform test -json` command using the same example Below is the output of the `terraform test -junit-xml=./output.xml` command using the same example files. The test output is: -* printed to the terminal in the default, human-readable format, as already described above -* also saved in JUnit XML format in the specified file +* Printed to the terminal in the default, human-readable format, as described above. +* Also saved in JUnit XML format in the specified file. -Below is the contents of the file `output.xml`: +Below is the contents of the resulting `output.xml` file: ```xml @@ -261,10 +261,25 @@ local.false did not match expected value The file maps Terraform test command concepts to JUnit XML format according to the table below: -| Terraform test concept | Element in JUnit XML output | Notes | -| -----------------------| ----------------------------|--------------| -| Test directory | `` | A single `` element wraps all other elements in the XML output, as `terraform test` only processes a single test directory. | -| Test file | `` | There will be as many `` elements as there are test files. The tests in a `` are summarized in its attributes: `name` is the file name; `tests` is total count of tests; `skipped` is a count of skipped tests; `failures` is a count of failed tests, `errors` is a count of errored tests. | -| Run block | `` | Any `` elements will be nested inside a parental `` element corresponding to the file that contains the run block. | -| Test failure error | `` | If a `` fails then it will contain a `` element that contains a message about a test failure. | -| ??? | `` | ??? | +| Terraform test concept | Element in JUnit XML output | +| -----------------------| ----------------------------| +| Test directory | `` | +| Run block | `` | +| Test failure error | `` | +| ??? | `` | + +A JUnit format XML file will contain a single `` element that wraps all other elements in the XML output, as `terraform test` only processes a single test directory. Inside `` there will be as many `` elements as there are test files. Test files can contain many run blocks, so `` elements will contain a `` element per run block within that test file. If a run block's assertions fail then that `` element will contain a `` element describing the test failure. Any other issues that arise while executing that run block will be reported in a similar `` element. + +Some elements have attributes that provide addition information about the testing output data. These are summarized below: + +| JUnit XML Element | Attribute | Meaning | +| -----------------------| ----------------------------|-------------------------------------------------------| +| `` | `name` | The test file name | +| `` | `tests` | Count of total tests (run blocks) in the file | +| `` | `skipped` | Count of skipped tests (run blocs) count in the file | +| `` | `failures` | Count of failed tests (run blocks) count in the file | +| `` | `errored` | Count of errored tests (run blocks) count in the file | +| `` | `name` | The run block's name | +| `` | `classname` | The name of the file containing the run block | +| `` | `time` | The duration of executing the run block | +| `` | `message` | A message describing the test failure | \ No newline at end of file From a7bbdecd61aaff9c8f63d904cc80f4f69c62a48b Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 6 Jan 2025 17:50:01 +0000 Subject: [PATCH 5/5] Add details about `` --- website/docs/cli/commands/test.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/cli/commands/test.mdx b/website/docs/cli/commands/test.mdx index 6e02a12f27ca..37ac993a7023 100644 --- a/website/docs/cli/commands/test.mdx +++ b/website/docs/cli/commands/test.mdx @@ -268,7 +268,7 @@ The file maps Terraform test command concepts to JUnit XML format according to t | Test failure error | `` | | ??? | `` | -A JUnit format XML file will contain a single `` element that wraps all other elements in the XML output, as `terraform test` only processes a single test directory. Inside `` there will be as many `` elements as there are test files. Test files can contain many run blocks, so `` elements will contain a `` element per run block within that test file. If a run block's assertions fail then that `` element will contain a `` element describing the test failure. Any other issues that arise while executing that run block will be reported in a similar `` element. +A JUnit format XML file will contain a single `` element that wraps all other elements in the XML output, as `terraform test` only processes a single test directory. Inside `` there will be as many `` elements as there are test files. Test files can contain many run blocks, so `` elements will contain a `` element per run block within that test file. If a run block's assertions fail then that `` element will contain a `` element describing the test failure. Similarly, a test may contain a nested `` element either if the test is skipped as a result of previous errors encountered in a given test file or if an interrupt is received by Terraform while tests are still running. Any other issues that arise while executing that run block will be reported in a nested `` element. Some elements have attributes that provide addition information about the testing output data. These are summarized below: