Merge pull request #191 from yandex/dev

v0.5.26

.changes/v0.5.26.md

@@ -0,0 +1,6 @@
+## v0.5.26 - 2024-05-21
+### Added
+* scenario config local block in yaml
+* scenario config local block documentation
+### Changed
+* reflect metadata simpler format and docs

.mapping.json

@@ -23,6 +23,7 @@
   ".changes/v0.5.23.md":"load/projects/pandora/.changes/v0.5.23.md",
   ".changes/v0.5.24.md":"load/projects/pandora/.changes/v0.5.24.md",
   ".changes/v0.5.25.md":"load/projects/pandora/.changes/v0.5.25.md",
+  ".changes/v0.5.26.md":"load/projects/pandora/.changes/v0.5.26.md",
   ".changie.yaml":"load/projects/pandora/.changie.yaml",
   ".github/workflows/release.yml":"load/projects/pandora/.github/workflows/release.yml",
   ".github/workflows/test.yml":"load/projects/pandora/.github/workflows/test.yml",
@@ -273,6 +274,7 @@
   "docs/eng/scenario-grpc-generator.md":"load/projects/pandora/docs/eng/scenario-grpc-generator.md",
   "docs/eng/scenario-http-generator.md":"load/projects/pandora/docs/eng/scenario-http-generator.md",
   "docs/eng/scenario/functions.md":"load/projects/pandora/docs/eng/scenario/functions.md",
+  "docs/eng/scenario/locals.md":"load/projects/pandora/docs/eng/scenario/locals.md",
   "docs/eng/scenario/variable_source.md":"load/projects/pandora/docs/eng/scenario/variable_source.md",
   "docs/eng/startup.md":"load/projects/pandora/docs/eng/startup.md",
   "docs/eng/tutorial.md":"load/projects/pandora/docs/eng/tutorial.md",
@@ -304,6 +306,7 @@
   "docs/rus/scenario-grpc-generator.md":"load/projects/pandora/docs/rus/scenario-grpc-generator.md",
   "docs/rus/scenario-http-generator.md":"load/projects/pandora/docs/rus/scenario-http-generator.md",
   "docs/rus/scenario/functions.md":"load/projects/pandora/docs/rus/scenario/functions.md",
+  "docs/rus/scenario/locals.md":"load/projects/pandora/docs/rus/scenario/locals.md",
   "docs/rus/scenario/variable_source.md":"load/projects/pandora/docs/rus/scenario/variable_source.md",
   "docs/rus/startup.md":"load/projects/pandora/docs/rus/startup.md",
   "docs/rus/tutorial.md":"load/projects/pandora/docs/rus/tutorial.md",

CHANGELOG.md

@@ -6,6 +6,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.5.26 - 2024-05-21
+### Added
+* scenario config local block in yaml
+* scenario config local block documentation
+### Changed
+* reflect metadata simpler format and docs
+
 ## v0.5.25 - 2024-05-16
 ### Fixed
 * randInt function without args in preprocessor

README.md

@@ -12,6 +12,9 @@
 
 Pandora is a high-performance load generator in Go language. It has built-in HTTP(S) and HTTP/2 support and you can write your own load scenarios in Go, compiling them just before your test.
 
+## Documentation
+[Documentation](https://yandex.github.io/pandora/)
+
 ## How to start
 
 ### Binary releases
@@ -71,9 +74,3 @@ Create changelog release file - `changie batch v0.5.21`
 Same for next version - `changie batch $(changie next patch)`
 
 Merge to main CHANGELOG.md file - `changie merge`
-
-## Documentation
-[Documentation](https://yandex.github.io/pandora/)
-
-## Old Documentation
-[ReadTheDocs](https://yandexpandora.readthedocs.io/)

cli/cli.go

@@ -25,7 +25,7 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-const Version = "0.5.25"
+const Version = "0.5.26"
 const defaultConfigFile = "load"
 const stdinConfigSelector = "-"
 

components/guns/grpc/core.go

@@ -43,13 +43,13 @@ type GrpcDialOptions struct {
 }
 
 type GunConfig struct {
-	Target          string          `validate:"required"`
-	ReflectPort     int64           `config:"reflect_port"`
-	ReflectMetadata metadata.MD     `config:"reflect_metadata"`
-	Timeout         time.Duration   `config:"timeout"` // grpc request timeout
-	TLS             bool            `config:"tls"`
-	DialOptions     GrpcDialOptions `config:"dial_options"`
-	AnswLog         AnswLogConfig   `config:"answlog"`
+	Target          string            `validate:"required"`
+	ReflectPort     int64             `config:"reflect_port"`
+	ReflectMetadata map[string]string `config:"reflect_metadata"`
+	Timeout         time.Duration     `config:"timeout"` // grpc request timeout
+	TLS             bool              `config:"tls"`
+	DialOptions     GrpcDialOptions   `config:"dial_options"`
+	AnswLog         AnswLogConfig     `config:"answlog"`
 	SharedClient    struct {
 		ClientNumber int  `config:"client-number,omitempty"`
 		Enabled      bool `config:"enabled"`
@@ -111,7 +111,7 @@ func (g *Gun) prepareMethodList(opts *warmup.Options) (map[string]desc.MethodDes
 	}
 	defer conn.Close()
 
-	refCtx := metadata.NewOutgoingContext(context.Background(), g.Conf.ReflectMetadata)
+	refCtx := metadata.NewOutgoingContext(context.Background(), metadata.New(g.Conf.ReflectMetadata))
 	refClient := grpcreflect.NewClientAuto(refCtx, conn)
 	listServices, err := refClient.ListServices()
 	if err != nil {

components/guns/grpc/scenario/core.go

@@ -21,13 +21,13 @@ import (
 const defaultTimeout = time.Second * 15
 
 type GunConfig struct {
-	Target          string          `validate:"required"`
-	ReflectPort     int64           `config:"reflect_port"`
-	ReflectMetadata metadata.MD     `config:"reflect_metadata"`
-	Timeout         time.Duration   `config:"timeout"` // grpc request timeout
-	TLS             bool            `config:"tls"`
-	DialOptions     GrpcDialOptions `config:"dial_options"`
-	AnswLog         AnswLogConfig   `config:"answlog"`
+	Target          string            `validate:"required"`
+	ReflectPort     int64             `config:"reflect_port"`
+	ReflectMetadata map[string]string `config:"reflect_metadata"`
+	Timeout         time.Duration     `config:"timeout"` // grpc request timeout
+	TLS             bool              `config:"tls"`
+	DialOptions     GrpcDialOptions   `config:"dial_options"`
+	AnswLog         AnswLogConfig     `config:"answlog"`
 }
 
 type GrpcDialOptions struct {

components/providers/scenario/config/config.go

@@ -13,6 +13,7 @@ import (
 
 // AmmoConfig is a config for dynamic converting from map[string]interface{}
 type AmmoConfig struct {
+	Locals          map[string]any
 	VariableSources []vs.VariableSource `config:"variable_sources"`
 	Requests        []RequestConfig
 	Calls           []CallConfig

components/providers/scenario/grpc/provider.go

@@ -15,7 +15,7 @@ var _ core.Provider = (*scenario.Provider[*gun.Scenario])(nil)
 const defaultSinkSize = 100
 
 func NewProvider(fs afero.Fs, conf scenario.ProviderConfig) (core.Provider, error) {
-	const op = "config.NewProvider"
+	const op = "scenario_grpc.NewProvider"
 	ammoCfg, err := config.ReadAmmoConfig(fs, conf.File)
 	if err != nil {
 		return nil, fmt.Errorf("%s ReadAmmoConfig %w", op, err)

components/providers/scenario/http/provider.go

@@ -15,7 +15,7 @@ var _ core.Provider = (*scenario.Provider[*gun.Scenario])(nil)
 const defaultSinkSize = 100
 
 func NewProvider(fs afero.Fs, conf scenario.ProviderConfig) (core.Provider, error) {
-	const op = "scenario.NewProvider"
+	const op = "scenario_http.NewProvider"
 	ammoCfg, err := config.ReadAmmoConfig(fs, conf.File)
 	if err != nil {
 		return nil, fmt.Errorf("%s ReadAmmoConfig %w", op, err)

components/providers/scenario/test/decode_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/yandex/pandora/components/providers/scenario/config"
 	_import "github.com/yandex/pandora/components/providers/scenario/import"
 	"github.com/yandex/pandora/core/plugin/pluginconfig"
@@ -22,12 +23,14 @@ func Test_ReadConfig_YamlAndHclSameResult(t *testing.T) {
 
 	t.Run("http", func(t *testing.T) {
 		fromHCL, err := config.ReadAmmoConfig(testFS, "../testdata/http_payload.hcl")
-		assert.NoError(t, err)
+		require.NoError(t, err)
+		fromHCL.Locals = nil
 
 		fromYaml, err := config.ReadAmmoConfig(testFS, "../testdata/http_payload.yaml")
-		assert.NoError(t, err)
+		require.NoError(t, err)
+		fromYaml.Locals = nil
 
-		assert.Equal(t, fromHCL, fromYaml)
+		require.Equal(t, fromHCL, fromYaml)
 	})
 }
 

components/providers/scenario/testdata/http_payload.yaml

@@ -1,3 +1,7 @@
+locals:
+  global-headers: &global-headers
+    Content-Type: application/json
+    Useragent: Yandex
 variable_sources:
   - name: users
     type: file/csv
@@ -20,9 +24,8 @@ requests:
   - name: auth_req
     method: POST
     uri: /auth
-    headers: &global-headers
-      Content-Type: application/json
-      Useragent: Yandex
+    headers:
+      <<: *global-headers
     tag: auth
     body: |
       {"user_id":  {{.request.auth_req.preprocessor.user_id}}}

docs/eng/grpc-generator.md

@@ -12,6 +12,9 @@ gun:
   target: '[hostname]:443'
   timeout: 15s              # Grpc request timeout. Default: 15s
   tls: false                # If true, Pandora accepts any certificate presented by the server and any host name in that certificate. Default: false
+  reflect_port: 8000        # If your reflection service is located on a port other than the main server
+  reflect_metadata:         # Separate metadata data for reflection service
+    auth: Token
   shared-client:
     enabled: false          # If TRUE, the generator will use a common transport client for all instances
     client-number: 1        # The number of shared clients can be increased. The default is 1
@@ -53,6 +56,7 @@ But to unify reports it converts them into HTTP codes.
 
 # References
 
+- [Scenario generator / gRPC](scenario-grpc-generator.md)
 - Best practices
   - [RPS per instance](best_practices/rps-per-instance.md)
   - [Shared client](best_practices/shared-client.md)

docs/eng/scenario-grpc-generator.md

@@ -11,6 +11,7 @@
     - [General principle](#general-principle)
     - [HCL example](#hcl-example)
     - [YAML example](#yaml-example)
+    - [Locals](#locals)
 - [Features](#features)
     - [Calls](#calls)
         - [Templater](#templater)
@@ -87,6 +88,18 @@ The Call is a gRPC call. It has standard gRPC call fields plus additional ones.
 ### HCL example
 
 ```terraform
+locals {
+  common_meta = {
+    "metadata" = "server.proto"
+  }
+  next = "next"
+}
+locals {
+  auth_meta = merge(local.common_meta, {
+    authorization = "{{.request.auth_req.postprocessor.token}}"
+  })
+  next = "next"
+}
 variable_source "users" "file/csv" {
   file              = "users.csv"
   fields            = ["user_id", "login", "pass"]
@@ -106,9 +119,7 @@ variable_source "variables" "variables" {
 call "auth_req" {
   call     = "target.TargetService.Auth"
   tag      = "auth"
-  metadata = {
-    "metadata" = "server.proto"
-  }
+  metadata = local.auth_meta
   preprocessor "prepare" {
     mapping = {
       user = "source.users[next]"
@@ -121,7 +132,6 @@ EOF
     payload     = ["token"]
     status_code = 200
   }
-}}
 }
 
 scenario "scenario_name" {
@@ -139,6 +149,9 @@ You can also see an example in the tests https://github.com/yandex/pandora/blob/
 ### YAML example
 
 ```yaml
+locals:
+  my-meta: &global-meta
+    metadata: "server.proto"
 variable_sources:
   - type: "file/csv"
     name: "users"
@@ -155,7 +168,7 @@ calls:
     tag: auth
     method: POST
     metadata:
-      metadata: "server.proto"
+      <<: *global-meta
     preprocessors:
       - type: prepare
         mapping:
@@ -175,6 +188,10 @@ scenarios:
     ]
 ```
 
+### Locals
+
+Про блок locals смотрите в отдельной [статье Locals](scenario/locals.md)
+
 ## Features
 
 ### Calls

docs/eng/scenario-http-generator.md

@@ -11,6 +11,7 @@
     - [General principle](#general-principle)
     - [HCL example](#hcl-example)
     - [YAML example](#yaml-example)
+    - [Locals](#locals)
 - [Features](#features)
     - [Requests](#requests)
         - [Templater](#templater)
@@ -97,6 +98,19 @@ Request - HTTP request. Has the standard HTTP request fields plus additional fie
 ### HCL example
 
 ```terraform
+locals {
+  common_headers = {
+    Content-Type  = "application/json"
+    Useragent     = "Yandex"
+  }
+  next = "next"
+}
+locals {
+  auth_headers = merge(local.common_headers, {
+    Authorization = "Bearer {{.request.auth_req.postprocessor.token}}"
+  })
+  next = "next"
+}
 variable_source "source_name" "file/csv" {
   file              = "file.csv"
   fields            = ["id", "name"]
@@ -107,9 +121,9 @@ variable_source "source_name" "file/csv" {
 request "request_name" {
   method  = "POST"
   uri     = "/uri"
-  headers = {
-    HeaderName = "header value"
-  }
+  headers = merge(local.common_headers, {
+    Authorization = "Bearer {{.request.auth_req.postprocessor.token}}"
+  })
   tag       = "tag"
   body      = <<EOF
 <body/>
@@ -144,6 +158,10 @@ scenario "scenario_name" {
 ### YAML example
 
 ```yaml
+locals:
+  my-headers: &global-headers
+    Content-Type: application/json
+    Useragent: Yandex
 variable_sources:
   - type: "file/csv"
     name: "source_name"
@@ -157,7 +175,7 @@ requests:
     uri: '/uri'
     method: POST
     headers:
-      Header-Name: "header value"
+      <<: *global-headers
     tag: tag
     body: '<body/>'
     preprocessor:
@@ -179,6 +197,10 @@ scenarios:
     ]
 ```
 
+### Locals
+
+See [Locals article](scenario/locals.md)
+
 ## Features
 
 ### Requests

docs/eng/scenario/functions.md

@@ -135,6 +135,28 @@ preprocessor {
 }
 ```
 
+# HCL functions
+
+You can use follow function 
+
+- [coalesce](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/coalesce)
+- [coalescelist](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/coalescelist)
+- [compact](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/compact)
+- [concat](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/concat)
+- [distinct](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/distinct)
+- [element](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/element)
+- [flatten](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/flatten)
+- [index](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/index-fn)
+- [keys](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/keys)
+- [lookup](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/lookup)
+- [merge](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/merge)
+- [reverse](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/reverse)
+- [slice](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/slice)
+- [sort](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/sort)
+- [split](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/string/split)
+- [values](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/values)
+- [zipmap](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/functions/collection/zipmap)
+
 
 ---