From 7872e1af1ce6241330153f78f51a25197a4d74cc Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Tue, 24 Dec 2024 21:49:22 +0900 Subject: [PATCH] Support exemplar federated query Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + pkg/cortex/modules.go | 1 + .../exemplar_merge_queryable.go | 215 ++++++++++ .../exemplar_merge_queryable_test.go | 374 ++++++++++++++++++ 4 files changed, 591 insertions(+) create mode 100644 pkg/querier/tenantfederation/exemplar_merge_queryable.go create mode 100644 pkg/querier/tenantfederation/exemplar_merge_queryable_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cfbb04b6d3..ee7ee80442 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249 * [FEATURE] Distributor: Accept multiple HA Tracker pairs in the same request. #6256 * [FEATURE] Ruler: Add support for per-user external labels #6340 +* [FEATURE] Query Frontend: Support an exemplar federated query when `-tenant-federation.enabled=true`. #6455 * [ENHANCEMENT] Querier: Add a `-tenant-federation.max-concurrent` flags to configure the number of worker processing federated query and add a `cortex_querier_federated_tenants_per_query` histogram to track the number of tenants per query. #6449 * [ENHANCEMENT] Query Frontend: Add a number of series in the query response to the query stat log. #6423 * [ENHANCEMENT] Store Gateway: Add a hedged request to reduce the tail latency. #6388 diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index a771c22116..24e2dc9b87 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -274,6 +274,7 @@ func (t *Cortex) initTenantFederation() (serv services.Service, err error) { // federation. byPassForSingleQuerier := true t.QuerierQueryable = querier.NewSampleAndChunkQueryable(tenantfederation.NewQueryable(t.QuerierQueryable, t.Cfg.TenantFederation.MaxConcurrent, byPassForSingleQuerier, prometheus.DefaultRegisterer)) + t.ExemplarQueryable = tenantfederation.NewExemplarQueryable(t.ExemplarQueryable, t.Cfg.TenantFederation.MaxConcurrent, byPassForSingleQuerier, prometheus.DefaultRegisterer) } return nil, nil } diff --git a/pkg/querier/tenantfederation/exemplar_merge_queryable.go b/pkg/querier/tenantfederation/exemplar_merge_queryable.go new file mode 100644 index 0000000000..45e519af75 --- /dev/null +++ b/pkg/querier/tenantfederation/exemplar_merge_queryable.go @@ -0,0 +1,215 @@ +package tenantfederation + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/weaveworks/common/user" + + "github.com/cortexproject/cortex/pkg/tenant" + "github.com/cortexproject/cortex/pkg/util/concurrency" + "github.com/cortexproject/cortex/pkg/util/spanlogger" +) + +// NewExemplarQueryable returns a exemplarQueryable that iterates through all the +// tenant IDs that are part of the request and aggregates the results from each +// tenant's ExemplarQuerier by sending of subsequent requests. +// By setting byPassWithSingleQuerier to true the mergeExemplarQuerier gets by-passed +// and results for request with a single exemplar querier will not contain the +// "__tenant_id__" label. This allows a smoother transition, when enabling +// tenant federation in a cluster. +// The result contains a label "__tenant_id__" to identify the tenant ID that +// it originally resulted from. +// If the label "__tenant_id__" is already existing, its value is overwritten +// by the tenant ID and the previous value is exposed through a new label +// prefixed with "original_". This behaviour is not implemented recursively. +func NewExemplarQueryable(upstream storage.ExemplarQueryable, maxConcurrent int, byPassWithSingleQuerier bool, reg prometheus.Registerer) storage.ExemplarQueryable { + return NewMergeExemplarQueryable(defaultTenantLabel, maxConcurrent, tenantExemplarQuerierCallback(upstream), byPassWithSingleQuerier, reg) +} + +func tenantExemplarQuerierCallback(exemplarQueryable storage.ExemplarQueryable) MergeExemplarQuerierCallback { + return func(ctx context.Context) ([]string, []storage.ExemplarQuerier, error) { + tenantIDs, err := tenant.TenantIDs(ctx) + if err != nil { + return nil, nil, err + } + + var queriers = make([]storage.ExemplarQuerier, len(tenantIDs)) + for pos, tenantID := range tenantIDs { + q, err := exemplarQueryable.ExemplarQuerier(user.InjectOrgID(ctx, tenantID)) + if err != nil { + return nil, nil, err + } + queriers[pos] = q + } + + return tenantIDs, queriers, nil + } +} + +// MergeExemplarQuerierCallback returns the underlying exemplar queriers and their +// IDs relevant for the query. +type MergeExemplarQuerierCallback func(ctx context.Context) (ids []string, queriers []storage.ExemplarQuerier, err error) + +// NewMergeExemplarQueryable returns a queryable that merges results from multiple +// underlying ExemplarQueryables. +// By setting byPassWithSingleQuerier to true the mergeExemplarQuerier gets by-passed +// and results for request with a single exemplar querier will not contain the +// "__tenant_id__" label. This allows a smoother transition, when enabling +// tenant federation in a cluster. +// Results contain a label `idLabelName` to identify the underlying exemplar queryable +// that it originally resulted from. +// If the label `idLabelName` is already existing, its value is overwritten and +// the previous value is exposed through a new label prefixed with "original_". +// This behaviour is not implemented recursively. +func NewMergeExemplarQueryable(idLabelName string, maxConcurrent int, callback MergeExemplarQuerierCallback, byPassWithSingleQuerier bool, reg prometheus.Registerer) storage.ExemplarQueryable { + return &mergeExemplarQueryable{ + idLabelName: idLabelName, + byPassWithSingleQuerier: byPassWithSingleQuerier, + callback: callback, + maxConcurrent: maxConcurrent, + + tenantsPerExemplarQuery: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "querier_federated_tenants_per_exemplar_query", + Help: "Number of tenants per exemplar query.", + Buckets: []float64{1, 2, 4, 8, 16, 32, 64}, + }), + } +} + +type mergeExemplarQueryable struct { + idLabelName string + maxConcurrent int + byPassWithSingleQuerier bool + callback MergeExemplarQuerierCallback + tenantsPerExemplarQuery prometheus.Histogram +} + +// ExemplarQuerier returns a new mergeExemplarQuerier which aggregates results from +// multiple exemplar queriers into a single result. +func (m *mergeExemplarQueryable) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { + ids, queriers, err := m.callback(ctx) + if err != nil { + return nil, err + } + + m.tenantsPerExemplarQuery.Observe(float64(len(ids))) + + if m.byPassWithSingleQuerier && len(queriers) == 1 { + return queriers[0], nil + } + + return &mergeExemplarQuerier{ + ctx: ctx, + idLabelName: m.idLabelName, + maxConcurrent: m.maxConcurrent, + tenantIds: ids, + queriers: queriers, + byPassWithSingleQuerier: m.byPassWithSingleQuerier, + }, nil +} + +// mergeExemplarQuerier aggregates the results from underlying exemplar queriers +// and adds a label `idLabelName` to identify the exemplar queryable that +// `seriesLabels` resulted from. +// If the label `idLabelName` is already existing, its value is overwritten and +// the previous value is exposed through a new label prefixed with "original_". +// This behaviour is not implemented recursively. +type mergeExemplarQuerier struct { + ctx context.Context + idLabelName string + maxConcurrent int + tenantIds []string + queriers []storage.ExemplarQuerier + byPassWithSingleQuerier bool +} + +type exemplarSelectJob struct { + pos int + querier storage.ExemplarQuerier + id string +} + +// Select returns aggregated exemplars within given time range for multiple tenants. +func (m mergeExemplarQuerier) Select(start, end int64, matchers ...[]*labels.Matcher) ([]exemplar.QueryResult, error) { + log, ctx := spanlogger.New(m.ctx, "mergeExemplarQuerier.Select") + defer log.Span.Finish() + + // filter out tenants to query and unrelated matchers + allMatchedTenantIds, allUnrelatedMatchers := filterAllTenantsAndMatchers(m.idLabelName, m.tenantIds, matchers) + jobs := make([]interface{}, len(allMatchedTenantIds)) + results := make([][]exemplar.QueryResult, len(allMatchedTenantIds)) + + var jobPos int + for idx, tenantId := range m.tenantIds { + if _, ok := allMatchedTenantIds[tenantId]; !ok { + // skip tenantIds that should not be queried + continue + } + + jobs[jobPos] = &exemplarSelectJob{ + pos: jobPos, + querier: m.queriers[idx], + id: tenantId, + } + jobPos++ + } + + run := func(ctx context.Context, jobIntf interface{}) error { + job, ok := jobIntf.(*exemplarSelectJob) + if !ok { + return fmt.Errorf("unexpected type %T", jobIntf) + } + + res, err := job.querier.Select(start, end, allUnrelatedMatchers...) + if err != nil { + return errors.Wrapf(err, "error exemplars querying %s %s", rewriteLabelName(m.idLabelName), job.id) + } + + // append __tenant__ label to `seriesLabels` to identify each tenants + for i, e := range res { + e.SeriesLabels = setLabelsRetainExisting(e.SeriesLabels, labels.Label{ + Name: m.idLabelName, + Value: job.id, + }) + res[i] = e + } + + results[job.pos] = res + return nil + } + + err := concurrency.ForEach(ctx, jobs, m.maxConcurrent, run) + if err != nil { + return nil, err + } + + var ret []exemplar.QueryResult + for _, exemplars := range results { + ret = append(ret, exemplars...) + } + + return ret, nil +} + +func filterAllTenantsAndMatchers(idLabelName string, tenantIds []string, allMatchers [][]*labels.Matcher) (map[string]struct{}, [][]*labels.Matcher) { + allMatchedTenantIds := make(map[string]struct{}) + allUnrelatedMatchers := make([][]*labels.Matcher, len(allMatchers)) + + for idx, matchers := range allMatchers { + matchedTenantIds, unrelatedMatchers := filterValuesByMatchers(idLabelName, tenantIds, matchers...) + for tenantId := range matchedTenantIds { + allMatchedTenantIds[tenantId] = struct{}{} + } + allUnrelatedMatchers[idx] = unrelatedMatchers + } + + return allMatchedTenantIds, allUnrelatedMatchers +} diff --git a/pkg/querier/tenantfederation/exemplar_merge_queryable_test.go b/pkg/querier/tenantfederation/exemplar_merge_queryable_test.go new file mode 100644 index 0000000000..309f2dea53 --- /dev/null +++ b/pkg/querier/tenantfederation/exemplar_merge_queryable_test.go @@ -0,0 +1,374 @@ +package tenantfederation + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/stretchr/testify/require" + "github.com/weaveworks/common/user" + + "github.com/cortexproject/cortex/pkg/tenant" +) + +var ( + expectedSingleTenantsExemplarMetrics = ` +# HELP cortex_querier_federated_tenants_per_exemplar_query Number of tenants per exemplar query. +# TYPE cortex_querier_federated_tenants_per_exemplar_query histogram +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="1"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="2"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="4"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="8"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="16"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="32"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="64"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="+Inf"} 1 +cortex_querier_federated_tenants_per_exemplar_query_sum 1 +cortex_querier_federated_tenants_per_exemplar_query_count 1 +` + + expectedTwoTenantsExemplarMetrics = ` +# HELP cortex_querier_federated_tenants_per_exemplar_query Number of tenants per exemplar query. +# TYPE cortex_querier_federated_tenants_per_exemplar_query histogram +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="1"} 0 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="2"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="4"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="8"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="16"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="32"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="64"} 1 +cortex_querier_federated_tenants_per_exemplar_query_bucket{le="+Inf"} 1 +cortex_querier_federated_tenants_per_exemplar_query_sum 2 +cortex_querier_federated_tenants_per_exemplar_query_count 1 +` +) + +type mockExemplarQueryable struct { + exemplarQueriers map[string]storage.ExemplarQuerier +} + +func (m *mockExemplarQueryable) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { + // Due to lint check for `ensure the query path is supporting multiple tenants` + ids, err := tenant.TenantIDs(ctx) + if err != nil { + return nil, err + } + + id := ids[0] + if _, ok := m.exemplarQueriers[id]; ok { + return m.exemplarQueriers[id], nil + } else { + return nil, errors.New("failed to get exemplar querier") + } +} + +type mockExemplarQuerier struct { + res []exemplar.QueryResult + err error +} + +func (m *mockExemplarQuerier) Select(_, _ int64, _ ...[]*labels.Matcher) ([]exemplar.QueryResult, error) { + if m.err != nil { + return nil, m.err + } + + return m.res, nil +} + +// getFixtureExemplarResult1 returns fixture examplar1 +func getFixtureExemplarResult1() []exemplar.QueryResult { + res := []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "123"), + Value: 123, + Ts: 1734942337900, + }, + }, + }, + } + return res +} + +// getFixtureExemplarResult2 returns fixture examplar +func getFixtureExemplarResult2() []exemplar.QueryResult { + res := []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "456"), + Value: 456, + Ts: 1734942338000, + }, + }, + }, + } + return res +} + +func Test_MergeExemplarQuerier_Select(t *testing.T) { + // set a multi tenant resolver + tenant.WithDefaultResolver(tenant.NewMultiResolver()) + + tests := []struct { + name string + upstream mockExemplarQueryable + matcher [][]*labels.Matcher + orgId string + expectedResult []exemplar.QueryResult + expectedErr error + expectedMetrics string + }{ + { + name: "should be treated as single tenant", + upstream: mockExemplarQueryable{exemplarQueriers: map[string]storage.ExemplarQuerier{ + "user-1": &mockExemplarQuerier{res: getFixtureExemplarResult1()}, + "user-2": &mockExemplarQuerier{res: getFixtureExemplarResult2()}, + }}, + matcher: [][]*labels.Matcher{{ + labels.MustNewMatcher(labels.MatchEqual, "__name__", "exemplar_series"), + }}, + orgId: "user-1", + expectedResult: []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "123"), + Value: 123, + Ts: 1734942337900, + }, + }, + }, + }, + expectedMetrics: expectedSingleTenantsExemplarMetrics, + }, + { + name: "two tenants results should be aggregated", + upstream: mockExemplarQueryable{exemplarQueriers: map[string]storage.ExemplarQuerier{ + "user-1": &mockExemplarQuerier{res: getFixtureExemplarResult1()}, + "user-2": &mockExemplarQuerier{res: getFixtureExemplarResult2()}, + }}, + matcher: [][]*labels.Matcher{{ + labels.MustNewMatcher(labels.MatchEqual, "__name__", "exemplar_series"), + }}, + orgId: "user-1|user-2", + expectedResult: []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", "__tenant_id__", "user-1"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "123"), + Value: 123, + Ts: 1734942337900, + }, + }, + }, + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", "__tenant_id__", "user-2"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "456"), + Value: 456, + Ts: 1734942338000, + }, + }, + }, + }, + expectedMetrics: expectedTwoTenantsExemplarMetrics, + }, + { + name: "should return the matched tenant query results", + upstream: mockExemplarQueryable{exemplarQueriers: map[string]storage.ExemplarQuerier{ + "user-1": &mockExemplarQuerier{res: getFixtureExemplarResult1()}, + "user-2": &mockExemplarQuerier{res: getFixtureExemplarResult2()}, + }}, + matcher: [][]*labels.Matcher{{ + labels.MustNewMatcher(labels.MatchEqual, "__tenant_id__", "user-1"), + }}, + orgId: "user-1|user-2", + expectedResult: []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", "__tenant_id__", "user-1"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "123"), + Value: 123, + Ts: 1734942337900, + }, + }, + }, + }, + expectedMetrics: expectedTwoTenantsExemplarMetrics, + }, + { + name: "when the '__tenant_id__' label exist, should be converted to the 'original___tenant_id__'", + upstream: mockExemplarQueryable{exemplarQueriers: map[string]storage.ExemplarQuerier{ + "user-1": &mockExemplarQuerier{res: []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", defaultTenantLabel, "tenant"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "123"), + Value: 123, + Ts: 1734942337900, + }, + }, + }, + }}, + "user-2": &mockExemplarQuerier{res: getFixtureExemplarResult2()}, + }}, + matcher: [][]*labels.Matcher{{ + labels.MustNewMatcher(labels.MatchEqual, "__name__", "exemplar_series"), + }}, + orgId: "user-1|user-2", + expectedResult: []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", "__tenant_id__", "user-1", "original___tenant_id__", "tenant"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "123"), + Value: 123, + Ts: 1734942337900, + }, + }, + }, + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", "__tenant_id__", "user-2"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "456"), + Value: 456, + Ts: 1734942338000, + }, + }, + }, + }, + expectedMetrics: expectedTwoTenantsExemplarMetrics, + }, + { + name: "get error from one querier, should get error", + upstream: mockExemplarQueryable{exemplarQueriers: map[string]storage.ExemplarQuerier{ + "user-1": &mockExemplarQuerier{res: getFixtureExemplarResult1()}, + "user-2": &mockExemplarQuerier{err: errors.New("some error")}, + }}, + matcher: [][]*labels.Matcher{{ + labels.MustNewMatcher(labels.MatchEqual, "__name__", "exemplar_series"), + }}, + orgId: "user-1|user-2", + expectedResult: []exemplar.QueryResult{ + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", "__tenant_id__", "user-1"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "123"), + Value: 123, + Ts: 1734942337900, + }, + }, + }, + { + SeriesLabels: labels.FromStrings("__name__", "exemplar_series", "__tenant_id__", "user-2"), + Exemplars: []exemplar.Exemplar{ + { + Labels: labels.FromStrings("traceID", "456"), + Value: 456, + Ts: 1734942338000, + }, + }, + }, + }, + expectedErr: errors.New("some error"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reg := prometheus.NewPedanticRegistry() + exemplarQueryable := NewExemplarQueryable(&test.upstream, defaultMaxConcurrency, true, reg) + ctx := user.InjectOrgID(context.Background(), test.orgId) + q, err := exemplarQueryable.ExemplarQuerier(ctx) + require.NoError(t, err) + + result, err := q.Select(mint, maxt, test.matcher...) + if test.expectedErr != nil { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(test.expectedMetrics), "cortex_querier_federated_tenants_per_exemplar_query")) + require.Equal(t, test.expectedResult, result) + } + }) + } +} + +func Test_filterAllTenantsAndMatchers(t *testing.T) { + idLabelName := defaultTenantLabel + + tests := []struct { + name string + tenantIds []string + allMatchers [][]*labels.Matcher + expectedLenAllMatchedTenantIds int + expectedUnrelatedMatchersCnt int + }{ + { + name: "Should match all tenants", + tenantIds: []string{"user-1", "user-2"}, + allMatchers: [][]*labels.Matcher{ + { + labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + expectedLenAllMatchedTenantIds: 2, + expectedUnrelatedMatchersCnt: 1, + }, + { + name: "Should match target tenant with the `idLabelName` matcher", + tenantIds: []string{"user-1", "user-2"}, + allMatchers: [][]*labels.Matcher{ + { + labels.MustNewMatcher(labels.MatchEqual, defaultTenantLabel, "user-1"), + }, + }, + expectedLenAllMatchedTenantIds: 1, + expectedUnrelatedMatchersCnt: 0, + }, + { + name: "Should match all tenants with the retained label name matcher", + tenantIds: []string{"user-1", "user-2"}, + allMatchers: [][]*labels.Matcher{ + { + labels.MustNewMatcher(labels.MatchEqual, retainExistingPrefix+defaultTenantLabel, "user-1"), + }, + }, + expectedLenAllMatchedTenantIds: 2, + expectedUnrelatedMatchersCnt: 1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + allMatchedTenantIds, allUnrelatedMatchers := filterAllTenantsAndMatchers(idLabelName, test.tenantIds, test.allMatchers) + matcherCnt := 0 + for _, unrelatedMatchers := range allUnrelatedMatchers { + for _, matcher := range unrelatedMatchers { + if matcher.Name != "" { + matcherCnt++ + } + } + } + require.Equal(t, test.expectedLenAllMatchedTenantIds, len(allMatchedTenantIds)) + require.Equal(t, test.expectedUnrelatedMatchersCnt, matcherCnt) + }) + } +}