Skip to content

Commit

Permalink
[jaeger-v2] Refactor ElasticSearch/OpenSearch Storage Configurations (j…
Browse files Browse the repository at this point in the history
…aegertracing#6060)

## Which problem is this PR solving?
- Part of jaegertracing#6059

## Description of the changes
- Continuation of jaegertracing#5790
- Refactors the configurations for ElasticSearch and OpenSearch to
simplify the configuration by having more logical groupings

## How was this change tested?
- Unit Tests and E2E Integration Tests

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [x] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `yarn lint` and `yarn test`

---------

Signed-off-by: Jared Tan <[email protected]>
Signed-off-by: Mahad Zaryab <[email protected]>
Signed-off-by: Yuri Shkuro <[email protected]>
Co-authored-by: Jared Tan <[email protected]>
Co-authored-by: Yuri Shkuro <[email protected]>
  • Loading branch information
3 people authored and chahatsagarmain committed Oct 23, 2024
1 parent a02e128 commit 293819e
Show file tree
Hide file tree
Showing 27 changed files with 815 additions and 614 deletions.
23 changes: 10 additions & 13 deletions cmd/es-rollover/app/init/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/jaegertracing/jaeger/cmd/es-rollover/app"
"github.com/jaegertracing/jaeger/pkg/es"
"github.com/jaegertracing/jaeger/pkg/es/client"
"github.com/jaegertracing/jaeger/pkg/es/config"
"github.com/jaegertracing/jaeger/pkg/es/filter"
"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
)
Expand All @@ -28,19 +29,15 @@ type Action struct {
}

func (c Action) getMapping(version uint, templateName string) (string, error) {
c.Config.Indices.IndexPrefix = config.IndexPrefix(c.Config.Config.IndexPrefix)
mappingBuilder := mappings.MappingBuilder{
TemplateBuilder: es.TextTemplateBuilder{},
PrioritySpanTemplate: int64(c.Config.PrioritySpanTemplate),
PriorityServiceTemplate: int64(c.Config.PriorityServiceTemplate),
PriorityDependenciesTemplate: int64(c.Config.PriorityDependenciesTemplate),
PrioritySamplingTemplate: int64(c.Config.PrioritySamplingTemplate),
Shards: int64(c.Config.Shards),
Replicas: int64(c.Config.Replicas),
IndexPrefix: c.Config.IndexPrefix,
UseILM: c.Config.UseILM,
ILMPolicyName: c.Config.ILMPolicyName,
EsVersion: version,
TemplateBuilder: es.TextTemplateBuilder{},
Indices: c.Config.Indices,
UseILM: c.Config.UseILM,
ILMPolicyName: c.Config.ILMPolicyName,
EsVersion: version,
}

return mappingBuilder.GetMapping(templateName)
}

Expand All @@ -62,7 +59,7 @@ func (c Action) Do() error {
return fmt.Errorf("ILM policy %s doesn't exist in Elasticsearch. Please create it and re-run init", c.Config.ILMPolicyName)
}
}
rolloverIndices := app.RolloverIndices(c.Config.Archive, c.Config.SkipDependencies, c.Config.AdaptiveSampling, c.Config.IndexPrefix)
rolloverIndices := app.RolloverIndices(c.Config.Archive, c.Config.SkipDependencies, c.Config.AdaptiveSampling, c.Config.Config.IndexPrefix)
for _, indexName := range rolloverIndices {
if err := c.init(version, indexName); err != nil {
return err
Expand Down Expand Up @@ -114,7 +111,7 @@ func (c Action) init(version uint, indexopt app.IndexOption) error {
return err
}

jaegerIndices, err := c.IndicesClient.GetJaegerIndices(c.Config.IndexPrefix)
jaegerIndices, err := c.IndicesClient.GetJaegerIndices(c.Config.Config.IndexPrefix)
if err != nil {
return err
}
Expand Down
29 changes: 17 additions & 12 deletions cmd/es-rollover/app/init/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/spf13/viper"

"github.com/jaegertracing/jaeger/cmd/es-rollover/app"
cfg "github.com/jaegertracing/jaeger/pkg/es/config"
)

const (
Expand All @@ -21,14 +22,10 @@ const (
)

// Config holds configuration for index cleaner binary.
// Config.IndexPrefix supersedes Indices.IndexPrefix
type Config struct {
app.Config
Shards int
Replicas int
PrioritySpanTemplate int
PriorityServiceTemplate int
PriorityDependenciesTemplate int
PrioritySamplingTemplate int
cfg.Indices
}

// AddFlags adds flags for TLS to the FlagSet.
Expand All @@ -43,10 +40,18 @@ func (*Config) AddFlags(flags *flag.FlagSet) {

// InitFromViper initializes config from viper.Viper.
func (c *Config) InitFromViper(v *viper.Viper) {
c.Shards = v.GetInt(shards)
c.Replicas = v.GetInt(replicas)
c.PrioritySpanTemplate = v.GetInt(prioritySpanTemplate)
c.PriorityServiceTemplate = v.GetInt(priorityServiceTemplate)
c.PriorityDependenciesTemplate = v.GetInt(priorityDependenciesTemplate)
c.PrioritySamplingTemplate = v.GetInt(prioritySamplingTemplate)
c.Indices.Spans.Shards = v.GetInt64(shards)
c.Indices.Services.Shards = v.GetInt64(shards)
c.Indices.Dependencies.Shards = v.GetInt64(shards)
c.Indices.Sampling.Shards = v.GetInt64(shards)

c.Indices.Spans.Replicas = v.GetInt64(replicas)
c.Indices.Services.Replicas = v.GetInt64(replicas)
c.Indices.Dependencies.Replicas = v.GetInt64(replicas)
c.Indices.Sampling.Replicas = v.GetInt64(replicas)

c.Indices.Spans.Priority = v.GetInt64(prioritySpanTemplate)
c.Indices.Services.Priority = v.GetInt64(priorityServiceTemplate)
c.Indices.Dependencies.Priority = v.GetInt64(priorityDependenciesTemplate)
c.Indices.Sampling.Priority = v.GetInt64(prioritySamplingTemplate)
}
12 changes: 6 additions & 6 deletions cmd/es-rollover/app/init/flags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ func TestBindFlags(t *testing.T) {
require.NoError(t, err)

c.InitFromViper(v)
assert.Equal(t, 8, c.Shards)
assert.Equal(t, 16, c.Replicas)
assert.Equal(t, 300, c.PrioritySpanTemplate)
assert.Equal(t, 301, c.PriorityServiceTemplate)
assert.Equal(t, 302, c.PriorityDependenciesTemplate)
assert.Equal(t, 303, c.PrioritySamplingTemplate)
assert.EqualValues(t, 8, c.Indices.Spans.Shards)
assert.EqualValues(t, 16, c.Indices.Spans.Replicas)
assert.EqualValues(t, 300, c.Indices.Spans.Priority)
assert.EqualValues(t, 301, c.Indices.Services.Priority)
assert.EqualValues(t, 302, c.Indices.Dependencies.Priority)
assert.EqualValues(t, 303, c.Indices.Sampling.Priority)
}
5 changes: 3 additions & 2 deletions cmd/esmapping-generator/app/flags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ func TestOptionsWithDefaultFlags(t *testing.T) {

assert.Equal(t, "", o.Mapping)
assert.Equal(t, uint(7), o.EsVersion)
assert.Equal(t, int64(5), o.Shards)
assert.Equal(t, int64(1), o.Replicas)
assert.EqualValues(t, 5, o.Shards)
assert.EqualValues(t, 1, o.Replicas)

assert.Equal(t, "", o.IndexPrefix)
assert.Equal(t, "false", o.UseILM)
assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName)
Expand Down
21 changes: 15 additions & 6 deletions cmd/esmapping-generator/app/renderer/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
"github.com/jaegertracing/jaeger/pkg/es"
cfg "github.com/jaegertracing/jaeger/pkg/es/config"
"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
)

Expand All @@ -25,14 +26,22 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e
return "", err
}

indexOpts := cfg.IndexOptions{
Shards: opt.Shards,
Replicas: opt.Replicas,
}
mappingBuilder := mappings.MappingBuilder{
TemplateBuilder: builder,
Shards: opt.Shards,
Replicas: opt.Replicas,
EsVersion: opt.EsVersion,
IndexPrefix: opt.IndexPrefix,
UseILM: enableILM,
ILMPolicyName: opt.ILMPolicyName,
Indices: cfg.Indices{
IndexPrefix: cfg.IndexPrefix(opt.IndexPrefix),
Spans: indexOpts,
Services: indexOpts,
Dependencies: indexOpts,
Sampling: indexOpts,
},
EsVersion: opt.EsVersion,
UseILM: enableILM,
ILMPolicyName: opt.ILMPolicyName,
}
return mappingBuilder.GetMapping(opt.Mapping)
}
Expand Down
26 changes: 24 additions & 2 deletions cmd/jaeger/config-elasticsearch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,32 @@ extensions:
backends:
some_storage:
elasticsearch:
index_prefix: "jaeger-main"
indices:
index_prefix: "jaeger-main"
spans:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
services:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
dependencies:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
sampling:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
another_storage:
elasticsearch:
index_prefix: "jaeger-archive"
indices:
index_prefix: "jaeger-archive"

receivers:
otlp:
Expand Down
27 changes: 24 additions & 3 deletions cmd/jaeger/config-opensearch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,32 @@ extensions:
backends:
some_storage:
opensearch:
index_prefix: "jaeger-main"

indices:
index_prefix: "jaeger-main"
spans:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
services:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
dependencies:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
sampling:
date_layout: "2006-01-02"
rollover_frequency: "day"
shards: 5
replicas: 1
another_storage:
opensearch:
index_prefix: "jaeger-main"
indices:
index_prefix: "jaeger-archive"

receivers:
otlp:
Expand Down
Loading

0 comments on commit 293819e

Please sign in to comment.