From 7a632e7f368bb7f3bbd8996f88bdd22cfe927bfc Mon Sep 17 00:00:00 2001 From: Dina Nimrodi Date: Mon, 12 Oct 2020 10:10:09 +0300 Subject: [PATCH] Development (#70) * IG 15394 - use new tsdb metric cache (#67) * new tsdb addFast * new AddFast Co-authored-by: Dina Nimrodi * new tsdb 0.11.2 (#68) Co-authored-by: Dina Nimrodi * fix step parsing in subQueries + fix inferring whether v3io needs to run aggregations (#66) * validate development unstable * Fix go.mod. go mod vendor. (#72) Co-authored-by: Gal Topper Co-authored-by: Dina Nimrodi Co-authored-by: Tal Neiman <33829179+talIguaz@users.noreply.github.com> Co-authored-by: ork Co-authored-by: Gal Topper Co-authored-by: Gal Topper --- README.md | 1 + go.mod | 6 +- go.sum | 8 +- promql/engine.go | 98 +++-- storage/tsdb/promtsdb.go | 35 +- vendor/github.com/golang/groupcache/LICENSE | 191 +++++++++ .../github.com/golang/groupcache/lru/lru.go | 133 +++++++ .../gophercloud/gophercloud/CHANGELOG.md | 138 +++---- .../klauspost/cpuid/CONTRIBUTING.txt | 70 ++-- vendor/github.com/v3io/frames/README.md | 2 +- vendor/github.com/v3io/frames/go.mod | 2 +- vendor/github.com/v3io/frames/go.sum | 2 + .../v3io/v3io-tsdb/pkg/aggregate/aggregate.go | 2 +- .../v3io/v3io-tsdb/pkg/aggregate/iterator.go | 6 +- .../v3io/v3io-tsdb/pkg/appender/appender.go | 59 +-- .../v3io/v3io-tsdb/pkg/appender/ingest.go | 8 +- .../v3io/v3io-tsdb/pkg/appender/store.go | 1 - .../v3io/v3io-tsdb/pkg/config/config.go | 18 +- .../v3io/v3io-tsdb/pkg/partmgr/partmgr.go | 97 +---- .../v3io/v3io-tsdb/pkg/pquerier/collector.go | 2 +- .../v3io/v3io-tsdb/pkg/pquerier/querier.go | 3 +- .../v3io/v3io-tsdb/pkg/pquerier/select.go | 8 +- .../v3io/v3io-tsdb/pkg/pquerier/sql_parser.go | 2 +- .../v3io/v3io-tsdb/pkg/querier/querier.go | 3 +- .../v3io/v3io-tsdb/pkg/tsdb/v3iotsdb.go | 17 +- .../v3io/v3io-tsdb/pkg/utils/labels.go | 19 + .../v3io/v3io-tsdb/pkg/utils/validators.go | 4 +- .../golang.org/x/net/http/httpproxy/proxy.go | 370 ++++++++++++++++++ vendor/modules.txt | 7 +- 29 files changed, 982 insertions(+), 330 deletions(-) create mode 100644 vendor/github.com/golang/groupcache/LICENSE create mode 100644 vendor/github.com/golang/groupcache/lru/lru.go create mode 100644 vendor/golang.org/x/net/http/httpproxy/proxy.go diff --git a/README.md b/README.md index 66b98de7895..1a86c3d054e 100644 --- a/README.md +++ b/README.md @@ -121,3 +121,4 @@ Apache License 2.0, see [LICENSE](https://github.com/prometheus/prometheus/blob/ [hub]: https://hub.docker.com/r/prom/prometheus/ [circleci]: https://circleci.com/gh/prometheus/prometheus [quay]: https://quay.io/repository/prometheus/prometheus + diff --git a/go.mod b/go.mod index 3056a57f7d4..dad6eb02890 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd github.com/soheilhy/cmux v0.1.4 github.com/stretchr/testify v1.4.0 - github.com/v3io/v3io-tsdb v0.10.12 + github.com/v3io/v3io-tsdb v0.11.2 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a @@ -61,8 +61,8 @@ require ( replace ( github.com/golang/lint => golang.org/x/lint v0.0.0-20190409202823-959b441ac422 - github.com/v3io/frames => github.com/v3io/frames v0.7.36 - github.com/v3io/v3io-tsdb => github.com/v3io/v3io-tsdb v0.10.12 + github.com/v3io/frames => github.com/v3io/frames v0.8.1 + github.com/v3io/v3io-tsdb => github.com/v3io/v3io-tsdb v0.11.2 google.golang.org/grpc => google.golang.org/grpc v1.19.1 k8s.io/klog => github.com/simonpasquier/klog-gokit v0.1.0 ) diff --git a/go.sum b/go.sum index 7c834ff7860..c5416ab61ef 100644 --- a/go.sum +++ b/go.sum @@ -388,12 +388,12 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y= github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/v3io/frames v0.7.36 h1:vWLJWbjvKMgWbCNrMj79wfKStcmvuAkh2UyfNGPIRfs= -github.com/v3io/frames v0.7.36/go.mod h1:qUW5rLv+DK4pC0t52685E9HxqDBg3VFgourbbYfRmaI= +github.com/v3io/frames v0.8.1 h1:wUWCX6RUEQ05SG3P9VtMBYh7gWrYNOiA81HcWXRXGZw= +github.com/v3io/frames v0.8.1/go.mod h1:qQnXBGO3uLKwN9HVcQ2ggHlIs8xT+200uJaWUgDocpk= github.com/v3io/v3io-go v0.1.9 h1:etkrrRmrI++i8sxGfK/+13f41TxIMohYeZHwVUM62vw= github.com/v3io/v3io-go v0.1.9/go.mod h1:5poBlcjZG5TiexRTYI44PE6tHzZz5Z60w+iS899pWtc= -github.com/v3io/v3io-tsdb v0.10.12 h1:+AfPHFJVxzpKtng0Hiz256ef9abzIwKwPuzB+0jG7w8= -github.com/v3io/v3io-tsdb v0.10.12/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= +github.com/v3io/v3io-tsdb v0.11.2 h1:knT+IwB+Bkd1aRPTIKltBnYrue1z+2EXmESJLkCexe8= +github.com/v3io/v3io-tsdb v0.11.2/go.mod h1:l+WxedjLmjy/TThj2vhSW/OnpRw8C7dOntIXHmM/a7I= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0 h1:dzZJf2IuMiclVjdw0kkT+f9u4YdrapbNyGAN47E/qnk= diff --git a/promql/engine.go b/promql/engine.go index d3f57d85c6c..a1c72308ddc 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -464,14 +464,14 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) ( if s.Start == s.End && s.Interval == 0 { start := timeMilliseconds(s.Start) evaluator := &evaluator{ - startTimestamp: start, - endTimestamp: start, - interval: 1, - ctx: ctxInnerEval, - maxSamples: ng.maxSamplesPerQuery, - defaultEvalInterval: GetDefaultEvaluationInterval(), - logger: ng.logger, - useV3ioAggregations: querier.(*tsdb.V3ioPromQuerier).UseV3ioAggregations(), + startTimestamp: start, + endTimestamp: start, + interval: 1, + ctx: ctxInnerEval, + maxSamples: ng.maxSamplesPerQuery, + defaultEvalInterval: GetDefaultEvaluationInterval(), + logger: ng.logger, + isAlreadyV3IOAggregated: querier.(*tsdb.V3ioPromQuerier).IsAlreadyAggregated, } val, err := evaluator.Eval(s.Expr) if err != nil { @@ -507,14 +507,14 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) ( // Range evaluation. evaluator := &evaluator{ - startTimestamp: timeMilliseconds(s.Start), - endTimestamp: timeMilliseconds(s.End), - interval: durationMilliseconds(s.Interval), - ctx: ctxInnerEval, - maxSamples: ng.maxSamplesPerQuery, - defaultEvalInterval: GetDefaultEvaluationInterval(), - logger: ng.logger, - useV3ioAggregations: querier.(*tsdb.V3ioPromQuerier).UseV3ioAggregations(), + startTimestamp: timeMilliseconds(s.Start), + endTimestamp: timeMilliseconds(s.End), + interval: durationMilliseconds(s.Interval), + ctx: ctxInnerEval, + maxSamples: ng.maxSamplesPerQuery, + defaultEvalInterval: GetDefaultEvaluationInterval(), + logger: ng.logger, + isAlreadyV3IOAggregated: querier.(*tsdb.V3ioPromQuerier).IsAlreadyAggregated, } val, err := evaluator.Eval(s.Expr) if err != nil { @@ -556,6 +556,7 @@ func (ng *Engine) cumulativeSubqueryOffset(path []Node) time.Duration { func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *EvalStmt) (storage.Querier, storage.Warnings, error) { var maxOffset time.Duration var aggregationWindow int64 + var subQueryStep int64 Inspect(s.Expr, func(node Node, path []Node) error { subqOffset := ng.cumulativeSubqueryOffset(path) @@ -568,13 +569,17 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev maxOffset = n.Offset + LookbackDelta + subqOffset } case *MatrixSelector: - aggregationWindow = n.Range.Nanoseconds() / 1000000 + aggregationWindow = durationToInt64Millis(n.Range) if maxOffset < n.Range+subqOffset { maxOffset = n.Range + subqOffset } if n.Offset+n.Range+subqOffset > maxOffset { maxOffset = n.Offset + n.Range + subqOffset } + case *SubqueryExpr: + // Save the step if it is provided in a subquery rather than as an interval + // Example query: `sum(metric)[1h:10m]` -> step=10m + subQueryStep = durationToInt64Millis(n.Step) } return nil }) @@ -591,10 +596,16 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev Inspect(s.Expr, func(node Node, path []Node) error { var set storage.SeriesSet var wrn storage.Warnings + + // Get the Step from the sub query, in case the interval is not set. + step := durationToInt64Millis(s.Interval) + if step == 0 { + step = subQueryStep + } params := &storage.SelectParams{ Start: timestamp.FromTime(s.Start), End: timestamp.FromTime(s.End), - Step: durationToInt64Millis(s.Interval), + Step: step, } // We need to make sure we select the timerange selected by the subquery. @@ -607,6 +618,9 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev switch n := node.(type) { case *VectorSelector: + // Validate if the current query can be aggregated via v3io + querier.(*tsdb.V3ioPromQuerier).UseAggregates = isV3ioEligibleQueryExpr(path) + params.Start = params.Start - durationMilliseconds(LookbackDelta) params.Func = extractFuncFromPath(path) params.By, params.Grouping = extractGroupsFromPath(path) @@ -629,6 +643,9 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev n.unexpandedSeriesSet = set case *MatrixSelector: + // Validate if the current query can be aggregated via v3io + querier.(*tsdb.V3ioPromQuerier).UseAggregates = isV3ioEligibleQueryExpr(path) + params.Func = extractFuncFromPath(path) params.Range = durationMilliseconds(n.Range) // For all matrix queries we want to ensure that we have (end-start) + range selected @@ -738,7 +755,7 @@ type evaluator struct { defaultEvalInterval int64 logger log.Logger - useV3ioAggregations bool // Indicates whether v3io tsdb already queried and aggregated the data, or just returned raw data + isAlreadyV3IOAggregated func(op string) bool // Indicates whether v3io tsdb already queried and aggregated the data, or just returned raw data } // errorf causes a panic with the input formatted into an error. @@ -980,7 +997,7 @@ func (ev *evaluator) eval(expr Expr) Value { switch e := expr.(type) { case *AggregateExpr: - if ev.useV3ioAggregations { + if ev.isAlreadyV3IOAggregated(e.Op.String()) { return ev.emptyAggregation(e.Expr) } if s, ok := e.Param.(*StringLiteral); ok { @@ -1038,9 +1055,10 @@ func (ev *evaluator) eval(expr Expr) Value { // Evaluate any non-matrix arguments. otherArgs := make([]Matrix, len(e.Args)) otherInArgs := make([]Vector, len(e.Args)) + function := e.Func.Name for i, e := range e.Args { if i != matrixArgIndex { - if ev.useV3ioAggregations { + if ev.isAlreadyV3IOAggregated(function) { return ev.emptyAggregation(e) } @@ -1226,14 +1244,14 @@ func (ev *evaluator) eval(expr Expr) Value { offsetMillis := durationToInt64Millis(e.Offset) rangeMillis := durationToInt64Millis(e.Range) newEv := &evaluator{ - endTimestamp: ev.endTimestamp - offsetMillis, - interval: ev.defaultEvalInterval, - ctx: ev.ctx, - currentSamples: ev.currentSamples, - maxSamples: ev.maxSamples, - defaultEvalInterval: ev.defaultEvalInterval, - logger: ev.logger, - useV3ioAggregations: ev.useV3ioAggregations, + endTimestamp: ev.endTimestamp - offsetMillis, + interval: ev.defaultEvalInterval, + ctx: ev.ctx, + currentSamples: ev.currentSamples, + maxSamples: ev.maxSamples, + defaultEvalInterval: ev.defaultEvalInterval, + logger: ev.logger, + isAlreadyV3IOAggregated: ev.isAlreadyV3IOAggregated, } if e.Step != 0 { @@ -1996,28 +2014,34 @@ func isV3ioEligibleFunction(function string) bool { return supportedV3ioFunctions[function] } -func isV3ioEligibleQueryExpr(e Expr) bool { - switch expr := e.(type) { +func isV3ioEligibleQueryExpr(p []Node) bool { + if len(p) == 0 { + return true + } + switch n := p[len(p)-1].(type) { case *AggregateExpr: - if !isV3ioEligibleAggregation(expr.Op) { + if !isV3ioEligibleAggregation(n.Op) { return false } - if expr.Without { + if n.Without { return false } // Currently only supports non-nested functions. // Not supported - avg(max_over_time(cpu[10m])), Supported - avg(cpu) - if e, ok := expr.Expr.(*Call); ok { + if e, ok := n.Expr.(*Call); ok { if e.Func != nil { return false } } return true case *Call: - return isV3ioEligibleFunction(expr.Func.Name) + return isV3ioEligibleFunction(n.Func.Name) + case *BinaryExpr: + // If we hit a binary expression we terminate since we only care about functions + // or aggregations over a single metric. + return false } - - return false + return isV3ioEligibleQueryExpr(p[:len(p)-1]) } // btos returns 1 if b is true, 0 otherwise. diff --git a/storage/tsdb/promtsdb.go b/storage/tsdb/promtsdb.go index eb20c24e6bf..16db4389b54 100644 --- a/storage/tsdb/promtsdb.go +++ b/storage/tsdb/promtsdb.go @@ -75,14 +75,22 @@ type V3ioPromQuerier struct { logger logger.Logger mint, maxt int64 - UseAggregatesConfig bool // Indicate whether or not to use v3io aggregations by default (passed from prometheus.yml) - UseAggregates bool // Indicate whether the current query is eligible for using v3io aggregations (should be set after creating a Querier instance) + UseAggregatesConfig bool // Indicate whether or not to use v3io aggregations by default (passed from prometheus.yml) + UseAggregates bool // Indicate whether the current query is eligible for using v3io aggregations (should be set after creating a Querier instance) + LastTSDBAggregatedAggr string } func (promQuery *V3ioPromQuerier) UseV3ioAggregations() bool { return promQuery.UseAggregates && promQuery.UseAggregatesConfig } +func (promQuery *V3ioPromQuerier) IsAlreadyAggregated(op string) bool { + if promQuery.UseV3ioAggregations() && promQuery.LastTSDBAggregatedAggr == op { + return true + } + return false +} + // Select returns a set of series that matches the given label matchers. func (promQuery *V3ioPromQuerier) Select(params *storage.SelectParams, oms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { name, filter, function := match2filter(oms, promQuery.logger) @@ -101,15 +109,23 @@ func (promQuery *V3ioPromQuerier) Select(params *storage.SelectParams, oms ...*l promQuery.logger.Debug("SelectParams: %+v", params) overTimeSuffix := "_over_time" + // Currently we do aggregations only for: + // 1. All Cross-series aggregation + // 2. Over-time aggregations where no Step or aggregationWindow was specified + // 3. Over-time aggregation for v3io-tsdb compatible aggregates + // 4. Down sampling - when only a step is provided + // Note: in addition to the above cases, we also take into consider the `UseAggregatesConfig` configuration and of + // course whether or not the requested aggregation is a valid v3io-tsdb aggregation if params.Func != "" { // only pass xx_over_time functions (just the xx part) // TODO: support count/stdxx, require changes in Prometheus: promql/functions.go, not calc aggregate twice if strings.HasSuffix(params.Func, overTimeSuffix) { - if promQuery.UseAggregates && promQuery.UseAggregatesConfig { + if promQuery.UseV3ioAggregations() { function = strings.TrimSuffix(params.Func, overTimeSuffix) } else { f := params.Func[0:3] - if params.Step == 0 && (f == "min" || f == "max" || f == "sum" || f == "avg") { + if params.Step == 0 && params.AggregationWindow == 0 && + (f == "min" || f == "max" || f == "sum" || f == "avg") { function = f } else { noAggr = true @@ -120,6 +136,10 @@ func (promQuery *V3ioPromQuerier) Select(params *storage.SelectParams, oms ...*l } } + if function != "" && !noAggr { + promQuery.LastTSDBAggregatedAggr = params.Func + } + selectParams := &pquerier.SelectParams{Name: name, Functions: function, Step: params.Step, @@ -128,6 +148,8 @@ func (promQuery *V3ioPromQuerier) Select(params *storage.SelectParams, oms ...*l To: promQuery.maxt, AggregationWindow: params.AggregationWindow} + promQuery.logger.DebugWith("Going to query tsdb", "params", selectParams, + "UseAggregates", promQuery.UseAggregates, "UseAggregatesConfig", promQuery.UseAggregatesConfig) set, err := promQuery.v3ioQuerier.SelectProm(selectParams, noAggr) return &V3ioPromSeriesSet{s: set}, nil, err } @@ -276,6 +298,11 @@ func (ls Labels) GetKey() (string, string, uint64) { } + +func (ls Labels) HashWithName() uint64 { + return ls.lbls.Hash() +} + // create update expression func (ls Labels) GetExpr() string { var lblExprBuilder strings.Builder diff --git a/vendor/github.com/golang/groupcache/LICENSE b/vendor/github.com/golang/groupcache/LICENSE new file mode 100644 index 00000000000..37ec93a14fd --- /dev/null +++ b/vendor/github.com/golang/groupcache/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/golang/groupcache/lru/lru.go b/vendor/github.com/golang/groupcache/lru/lru.go new file mode 100644 index 00000000000..eac1c7664f9 --- /dev/null +++ b/vendor/github.com/golang/groupcache/lru/lru.go @@ -0,0 +1,133 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package lru implements an LRU cache. +package lru + +import "container/list" + +// Cache is an LRU cache. It is not safe for concurrent access. +type Cache struct { + // MaxEntries is the maximum number of cache entries before + // an item is evicted. Zero means no limit. + MaxEntries int + + // OnEvicted optionally specifies a callback function to be + // executed when an entry is purged from the cache. + OnEvicted func(key Key, value interface{}) + + ll *list.List + cache map[interface{}]*list.Element +} + +// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators +type Key interface{} + +type entry struct { + key Key + value interface{} +} + +// New creates a new Cache. +// If maxEntries is zero, the cache has no limit and it's assumed +// that eviction is done by the caller. +func New(maxEntries int) *Cache { + return &Cache{ + MaxEntries: maxEntries, + ll: list.New(), + cache: make(map[interface{}]*list.Element), + } +} + +// Add adds a value to the cache. +func (c *Cache) Add(key Key, value interface{}) { + if c.cache == nil { + c.cache = make(map[interface{}]*list.Element) + c.ll = list.New() + } + if ee, ok := c.cache[key]; ok { + c.ll.MoveToFront(ee) + ee.Value.(*entry).value = value + return + } + ele := c.ll.PushFront(&entry{key, value}) + c.cache[key] = ele + if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { + c.RemoveOldest() + } +} + +// Get looks up a key's value from the cache. +func (c *Cache) Get(key Key) (value interface{}, ok bool) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.ll.MoveToFront(ele) + return ele.Value.(*entry).value, true + } + return +} + +// Remove removes the provided key from the cache. +func (c *Cache) Remove(key Key) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.removeElement(ele) + } +} + +// RemoveOldest removes the oldest item from the cache. +func (c *Cache) RemoveOldest() { + if c.cache == nil { + return + } + ele := c.ll.Back() + if ele != nil { + c.removeElement(ele) + } +} + +func (c *Cache) removeElement(e *list.Element) { + c.ll.Remove(e) + kv := e.Value.(*entry) + delete(c.cache, kv.key) + if c.OnEvicted != nil { + c.OnEvicted(kv.key, kv.value) + } +} + +// Len returns the number of items in the cache. +func (c *Cache) Len() int { + if c.cache == nil { + return 0 + } + return c.ll.Len() +} + +// Clear purges all stored items from the cache. +func (c *Cache) Clear() { + if c.OnEvicted != nil { + for _, e := range c.cache { + kv := e.Value.(*entry) + c.OnEvicted(kv.key, kv.value) + } + } + c.ll = nil + c.cache = nil +} diff --git a/vendor/github.com/gophercloud/gophercloud/CHANGELOG.md b/vendor/github.com/gophercloud/gophercloud/CHANGELOG.md index 8bc0c88cd43..d0b120de1b4 100644 --- a/vendor/github.com/gophercloud/gophercloud/CHANGELOG.md +++ b/vendor/github.com/gophercloud/gophercloud/CHANGELOG.md @@ -1,69 +1,69 @@ -## 0.4.0 (Unreleased) - -## 0.3.0 (July 31, 2019) - -IMPROVEMENTS - -* Added `baremetal/apiversions.List` [GH-1577](https://github.com/gophercloud/gophercloud/pull/1577) -* Added `baremetal/apiversions.Get` [GH-1577](https://github.com/gophercloud/gophercloud/pull/1577) -* Added `compute/v2/extensions/servergroups.CreateOpts.Policy` [GH-1636](https://github.com/gophercloud/gophercloud/pull/1636) -* Added `identity/v3/extensions/trusts.Create` [GH-1644](https://github.com/gophercloud/gophercloud/pull/1644) -* Added `identity/v3/extensions/trusts.Delete` [GH-1644](https://github.com/gophercloud/gophercloud/pull/1644) -* Added `CreatedAt` and `UpdatedAt` to `networking/v2/extensions/layer3/floatingips.FloatingIP` [GH-1647](https://github.com/gophercloud/gophercloud/issues/1646) -* Added `CreatedAt` and `UpdatedAt` to `networking/v2/extensions/security/groups.SecGroup` [GH-1654](https://github.com/gophercloud/gophercloud/issues/1654) -* Added `CreatedAt` and `UpdatedAt` to `networking/v2/networks.Network` [GH-1657](https://github.com/gophercloud/gophercloud/issues/1657) -* Added `keymanager/v1/containers.CreateSecretRef` [GH-1659](https://github.com/gophercloud/gophercloud/issues/1659) -* Added `keymanager/v1/containers.DeleteSecretRef` [GH-1659](https://github.com/gophercloud/gophercloud/issues/1659) -* Added `sharedfilesystems/v2/shares.GetMetadata` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) -* Added `sharedfilesystems/v2/shares.GetMetadatum` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) -* Added `sharedfilesystems/v2/shares.SetMetadata` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) -* Added `sharedfilesystems/v2/shares.UpdateMetadata` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) -* Added `sharedfilesystems/v2/shares.DeleteMetadatum` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) -* Added `sharedfilesystems/v2/sharetypes.IDFromName` [GH-1662](https://github.com/gophercloud/gophercloud/issues/1662) - - - -BUG FIXES - -* Changed `baremetal/v1/nodes.CleanStep.Args` from `map[string]string` to `map[string]interface{}` [GH-1638](https://github.com/gophercloud/gophercloud/pull/1638) -* Removed `URLPath` and `ExpectedCodes` from `loadbalancer/v2/monitors.ToMonitorCreateMap` since Octavia now provides default values when these fields are not specified [GH-1640](https://github.com/gophercloud/gophercloud/pull/1540) - - -## 0.2.0 (June 17, 2019) - -IMPROVEMENTS - -* Added `networking/v2/extensions/qos/rules.ListBandwidthLimitRules` [GH-1584](https://github.com/gophercloud/gophercloud/pull/1584) -* Added `networking/v2/extensions/qos/rules.GetBandwidthLimitRule` [GH-1584](https://github.com/gophercloud/gophercloud/pull/1584) -* Added `networking/v2/extensions/qos/rules.CreateBandwidthLimitRule` [GH-1584](https://github.com/gophercloud/gophercloud/pull/1584) -* Added `networking/v2/extensions/qos/rules.UpdateBandwidthLimitRule` [GH-1589](https://github.com/gophercloud/gophercloud/pull/1589) -* Added `networking/v2/extensions/qos/rules.DeleteBandwidthLimitRule` [GH-1590](https://github.com/gophercloud/gophercloud/pull/1590) -* Added `networking/v2/extensions/qos/policies.List` [GH-1591](https://github.com/gophercloud/gophercloud/pull/1591) -* Added `networking/v2/extensions/qos/policies.Get` [GH-1593](https://github.com/gophercloud/gophercloud/pull/1593) -* Added `networking/v2/extensions/qos/rules.ListDSCPMarkingRules` [GH-1594](https://github.com/gophercloud/gophercloud/pull/1594) -* Added `networking/v2/extensions/qos/policies.Create` [GH-1595](https://github.com/gophercloud/gophercloud/pull/1595) -* Added `compute/v2/extensions/diagnostics.Get` [GH-1592](https://github.com/gophercloud/gophercloud/pull/1592) -* Added `networking/v2/extensions/qos/policies.Update` [GH-1603](https://github.com/gophercloud/gophercloud/pull/1603) -* Added `networking/v2/extensions/qos/policies.Delete` [GH-1603](https://github.com/gophercloud/gophercloud/pull/1603) -* Added `networking/v2/extensions/qos/rules.CreateDSCPMarkingRule` [GH-1605](https://github.com/gophercloud/gophercloud/pull/1605) -* Added `networking/v2/extensions/qos/rules.UpdateDSCPMarkingRule` [GH-1605](https://github.com/gophercloud/gophercloud/pull/1605) -* Added `networking/v2/extensions/qos/rules.GetDSCPMarkingRule` [GH-1609](https://github.com/gophercloud/gophercloud/pull/1609) -* Added `networking/v2/extensions/qos/rules.DeleteDSCPMarkingRule` [GH-1609](https://github.com/gophercloud/gophercloud/pull/1609) -* Added `networking/v2/extensions/qos/rules.ListMinimumBandwidthRules` [GH-1615](https://github.com/gophercloud/gophercloud/pull/1615) -* Added `networking/v2/extensions/qos/rules.GetMinimumBandwidthRule` [GH-1615](https://github.com/gophercloud/gophercloud/pull/1615) -* Added `networking/v2/extensions/qos/rules.CreateMinimumBandwidthRule` [GH-1615](https://github.com/gophercloud/gophercloud/pull/1615) -* Added `Hostname` to `baremetalintrospection/v1/introspection.Data` [GH-1627](https://github.com/gophercloud/gophercloud/pull/1627) -* Added `networking/v2/extensions/qos/rules.UpdateMinimumBandwidthRule` [GH-1624](https://github.com/gophercloud/gophercloud/pull/1624) -* Added `networking/v2/extensions/qos/rules.DeleteMinimumBandwidthRule` [GH-1624](https://github.com/gophercloud/gophercloud/pull/1624) -* Added `networking/v2/extensions/qos/ruletypes.GetRuleType` [GH-1625](https://github.com/gophercloud/gophercloud/pull/1625) -* Added `Extra` to `baremetalintrospection/v1/introspection.Data` [GH-1611](https://github.com/gophercloud/gophercloud/pull/1611) -* Added `blockstorage/extensions/volumeactions.SetImageMetadata` [GH-1621](https://github.com/gophercloud/gophercloud/pull/1621) - -BUG FIXES - -* Updated `networking/v2/extensions/qos/rules.UpdateBandwidthLimitRule` to use return code 200 [GH-1606](https://github.com/gophercloud/gophercloud/pull/1606) -* Fixed bug in `compute/v2/extensions/schedulerhints.SchedulerHints.Query` where contents will now be marshalled to a string [GH-1620](https://github.com/gophercloud/gophercloud/pull/1620) - -## 0.1.0 (May 27, 2019) - -Initial tagged release. +## 0.4.0 (Unreleased) + +## 0.3.0 (July 31, 2019) + +IMPROVEMENTS + +* Added `baremetal/apiversions.List` [GH-1577](https://github.com/gophercloud/gophercloud/pull/1577) +* Added `baremetal/apiversions.Get` [GH-1577](https://github.com/gophercloud/gophercloud/pull/1577) +* Added `compute/v2/extensions/servergroups.CreateOpts.Policy` [GH-1636](https://github.com/gophercloud/gophercloud/pull/1636) +* Added `identity/v3/extensions/trusts.Create` [GH-1644](https://github.com/gophercloud/gophercloud/pull/1644) +* Added `identity/v3/extensions/trusts.Delete` [GH-1644](https://github.com/gophercloud/gophercloud/pull/1644) +* Added `CreatedAt` and `UpdatedAt` to `networking/v2/extensions/layer3/floatingips.FloatingIP` [GH-1647](https://github.com/gophercloud/gophercloud/issues/1646) +* Added `CreatedAt` and `UpdatedAt` to `networking/v2/extensions/security/groups.SecGroup` [GH-1654](https://github.com/gophercloud/gophercloud/issues/1654) +* Added `CreatedAt` and `UpdatedAt` to `networking/v2/networks.Network` [GH-1657](https://github.com/gophercloud/gophercloud/issues/1657) +* Added `keymanager/v1/containers.CreateSecretRef` [GH-1659](https://github.com/gophercloud/gophercloud/issues/1659) +* Added `keymanager/v1/containers.DeleteSecretRef` [GH-1659](https://github.com/gophercloud/gophercloud/issues/1659) +* Added `sharedfilesystems/v2/shares.GetMetadata` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) +* Added `sharedfilesystems/v2/shares.GetMetadatum` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) +* Added `sharedfilesystems/v2/shares.SetMetadata` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) +* Added `sharedfilesystems/v2/shares.UpdateMetadata` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) +* Added `sharedfilesystems/v2/shares.DeleteMetadatum` [GH-1656](https://github.com/gophercloud/gophercloud/issues/1656) +* Added `sharedfilesystems/v2/sharetypes.IDFromName` [GH-1662](https://github.com/gophercloud/gophercloud/issues/1662) + + + +BUG FIXES + +* Changed `baremetal/v1/nodes.CleanStep.Args` from `map[string]string` to `map[string]interface{}` [GH-1638](https://github.com/gophercloud/gophercloud/pull/1638) +* Removed `URLPath` and `ExpectedCodes` from `loadbalancer/v2/monitors.ToMonitorCreateMap` since Octavia now provides default values when these fields are not specified [GH-1640](https://github.com/gophercloud/gophercloud/pull/1540) + + +## 0.2.0 (June 17, 2019) + +IMPROVEMENTS + +* Added `networking/v2/extensions/qos/rules.ListBandwidthLimitRules` [GH-1584](https://github.com/gophercloud/gophercloud/pull/1584) +* Added `networking/v2/extensions/qos/rules.GetBandwidthLimitRule` [GH-1584](https://github.com/gophercloud/gophercloud/pull/1584) +* Added `networking/v2/extensions/qos/rules.CreateBandwidthLimitRule` [GH-1584](https://github.com/gophercloud/gophercloud/pull/1584) +* Added `networking/v2/extensions/qos/rules.UpdateBandwidthLimitRule` [GH-1589](https://github.com/gophercloud/gophercloud/pull/1589) +* Added `networking/v2/extensions/qos/rules.DeleteBandwidthLimitRule` [GH-1590](https://github.com/gophercloud/gophercloud/pull/1590) +* Added `networking/v2/extensions/qos/policies.List` [GH-1591](https://github.com/gophercloud/gophercloud/pull/1591) +* Added `networking/v2/extensions/qos/policies.Get` [GH-1593](https://github.com/gophercloud/gophercloud/pull/1593) +* Added `networking/v2/extensions/qos/rules.ListDSCPMarkingRules` [GH-1594](https://github.com/gophercloud/gophercloud/pull/1594) +* Added `networking/v2/extensions/qos/policies.Create` [GH-1595](https://github.com/gophercloud/gophercloud/pull/1595) +* Added `compute/v2/extensions/diagnostics.Get` [GH-1592](https://github.com/gophercloud/gophercloud/pull/1592) +* Added `networking/v2/extensions/qos/policies.Update` [GH-1603](https://github.com/gophercloud/gophercloud/pull/1603) +* Added `networking/v2/extensions/qos/policies.Delete` [GH-1603](https://github.com/gophercloud/gophercloud/pull/1603) +* Added `networking/v2/extensions/qos/rules.CreateDSCPMarkingRule` [GH-1605](https://github.com/gophercloud/gophercloud/pull/1605) +* Added `networking/v2/extensions/qos/rules.UpdateDSCPMarkingRule` [GH-1605](https://github.com/gophercloud/gophercloud/pull/1605) +* Added `networking/v2/extensions/qos/rules.GetDSCPMarkingRule` [GH-1609](https://github.com/gophercloud/gophercloud/pull/1609) +* Added `networking/v2/extensions/qos/rules.DeleteDSCPMarkingRule` [GH-1609](https://github.com/gophercloud/gophercloud/pull/1609) +* Added `networking/v2/extensions/qos/rules.ListMinimumBandwidthRules` [GH-1615](https://github.com/gophercloud/gophercloud/pull/1615) +* Added `networking/v2/extensions/qos/rules.GetMinimumBandwidthRule` [GH-1615](https://github.com/gophercloud/gophercloud/pull/1615) +* Added `networking/v2/extensions/qos/rules.CreateMinimumBandwidthRule` [GH-1615](https://github.com/gophercloud/gophercloud/pull/1615) +* Added `Hostname` to `baremetalintrospection/v1/introspection.Data` [GH-1627](https://github.com/gophercloud/gophercloud/pull/1627) +* Added `networking/v2/extensions/qos/rules.UpdateMinimumBandwidthRule` [GH-1624](https://github.com/gophercloud/gophercloud/pull/1624) +* Added `networking/v2/extensions/qos/rules.DeleteMinimumBandwidthRule` [GH-1624](https://github.com/gophercloud/gophercloud/pull/1624) +* Added `networking/v2/extensions/qos/ruletypes.GetRuleType` [GH-1625](https://github.com/gophercloud/gophercloud/pull/1625) +* Added `Extra` to `baremetalintrospection/v1/introspection.Data` [GH-1611](https://github.com/gophercloud/gophercloud/pull/1611) +* Added `blockstorage/extensions/volumeactions.SetImageMetadata` [GH-1621](https://github.com/gophercloud/gophercloud/pull/1621) + +BUG FIXES + +* Updated `networking/v2/extensions/qos/rules.UpdateBandwidthLimitRule` to use return code 200 [GH-1606](https://github.com/gophercloud/gophercloud/pull/1606) +* Fixed bug in `compute/v2/extensions/schedulerhints.SchedulerHints.Query` where contents will now be marshalled to a string [GH-1620](https://github.com/gophercloud/gophercloud/pull/1620) + +## 0.1.0 (May 27, 2019) + +Initial tagged release. diff --git a/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt b/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt index 452d28eda8e..2ef4714f716 100644 --- a/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt +++ b/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt @@ -1,35 +1,35 @@ -Developer Certificate of Origin -Version 1.1 - -Copyright (C) 2015- Klaus Post & Contributors. -Email: klauspost@gmail.com - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - - -Developer's Certificate of Origin 1.1 - -By making a contribution to this project, I certify that: - -(a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - -(b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - -(c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - -(d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2015- Klaus Post & Contributors. +Email: klauspost@gmail.com + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. diff --git a/vendor/github.com/v3io/frames/README.md b/vendor/github.com/v3io/frames/README.md index 1241e59d86f..976b6e069ba 100644 --- a/vendor/github.com/v3io/frames/README.md +++ b/vendor/github.com/v3io/frames/README.md @@ -477,7 +477,7 @@ Reads data from a data collection to a DataFrame, according to the specified bac #### Syntax ```python -read(backend='', table='', query='', columns=None, filter='', group_by='', +read(backend, table='', query='', columns=None, filter='', group_by='', limit=0, data_format='', row_layout=False, max_rows_in_msg=0, marker='', iterator=False, get_raw=False, **kw) ``` diff --git a/vendor/github.com/v3io/frames/go.mod b/vendor/github.com/v3io/frames/go.mod index ea54634d9ad..05bc48580ec 100644 --- a/vendor/github.com/v3io/frames/go.mod +++ b/vendor/github.com/v3io/frames/go.mod @@ -12,7 +12,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/stretchr/testify v1.4.0 github.com/v3io/v3io-go v0.1.9 - github.com/v3io/v3io-tsdb v0.10.12 + github.com/v3io/v3io-tsdb v0.11.0 github.com/valyala/fasthttp v1.2.0 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 golang.org/x/net v0.0.0-20190311183353-d8887717615a diff --git a/vendor/github.com/v3io/frames/go.sum b/vendor/github.com/v3io/frames/go.sum index ca962057290..0db0b76fb05 100644 --- a/vendor/github.com/v3io/frames/go.sum +++ b/vendor/github.com/v3io/frames/go.sum @@ -70,6 +70,8 @@ github.com/v3io/v3io-go v0.1.9 h1:etkrrRmrI++i8sxGfK/+13f41TxIMohYeZHwVUM62vw= github.com/v3io/v3io-go v0.1.9/go.mod h1:5poBlcjZG5TiexRTYI44PE6tHzZz5Z60w+iS899pWtc= github.com/v3io/v3io-tsdb v0.10.12 h1:+AfPHFJVxzpKtng0Hiz256ef9abzIwKwPuzB+0jG7w8= github.com/v3io/v3io-tsdb v0.10.12/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= +github.com/v3io/v3io-tsdb v0.11.0 h1:jN8A1EVCAzsJxozI/23Yl/aNoHJmXz+TKVQlx3ddAuk= +github.com/v3io/v3io-tsdb v0.11.0/go.mod h1:kp586KxTfROIGwb/nzNxwDbX2Wterxro+HbiZHmK548= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0 h1:dzZJf2IuMiclVjdw0kkT+f9u4YdrapbNyGAN47E/qnk= diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/aggregate.go b/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/aggregate.go index 64d85d1693c..f6588eb3a9a 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/aggregate.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/aggregate.go @@ -310,7 +310,7 @@ func MaskToString(mask AggrType) string { return output.String() } -func ToAttrName(aggr AggrType) string { +func ToAttrName(aggr fmt.Stringer) string { return config.AggregateAttrPrefix + aggr.String() } diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/iterator.go b/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/iterator.go index f8699148bff..cef61bf611c 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/iterator.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/aggregate/iterator.go @@ -134,7 +134,7 @@ func (as *Series) NewSetFromAttrs( } aggrArrays[aggr] = utils.AsInt64Array(attrBlob.([]byte)) - dataArrays[aggr] = make([]float64, length, length) + dataArrays[aggr] = make([]float64, length) copy(dataArrays[aggr], getOrCreateInitDataArray(aggr, length)) } } @@ -189,7 +189,7 @@ func (as *Series) NewSetFromChunks(length int) *Set { for _, aggr := range rawAggregates { if aggr&as.aggrMask != 0 { - dataArrays[aggr] = make([]float64, length, length) // TODO: len/capacity & reuse (pool) + dataArrays[aggr] = make([]float64, length) // TODO: len/capacity & reuse (pool) initArray := getOrCreateInitDataArray(aggr, length) copy(dataArrays[aggr], initArray) } @@ -373,7 +373,7 @@ func getOrCreateInitDataArray(aggrType AggrType, length int) []float64 { func createInitDataArray(aggrType AggrType, length int) []float64 { // Prepare "clean" array for fastest reset of "uninitialized" data arrays - resultArray := make([]float64, length, length) + resultArray := make([]float64, length) var initWith float64 switch aggrType { diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go b/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go index 24273713d42..3ecec3f4450 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/appender/appender.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/golang/groupcache/lru" "github.com/nuclio/logger" "github.com/pkg/errors" "github.com/v3io/v3io-go/pkg/dataplane" @@ -50,7 +51,6 @@ type MetricState struct { key string name string hash uint64 - refID uint64 aggrs []*MetricState @@ -103,11 +103,6 @@ func (m *MetricState) error() error { return m.err } -type cacheKey struct { - name string - hash uint64 -} - // store the state and metadata for all the metrics type MetricsCache struct { cfg *config.V3ioConfig @@ -129,11 +124,7 @@ type MetricsCache struct { outstandingUpdates int64 requestsInFlight int64 - lastMetric uint64 - - // TODO: consider switching to synch.Map (https://golang.org/pkg/sync/#Map) - cacheMetricMap map[cacheKey]*MetricState // TODO: maybe use hash as key & combine w ref - cacheRefMap map[uint64]*MetricState // TODO: maybe turn to list + free list, periodically delete old matrics + cacheMetricMap *lru.Cache NameLabelMap map[string]bool // temp store all lable names @@ -147,8 +138,7 @@ func NewMetricsCache(container v3io.Container, logger logger.Logger, cfg *config partMngr *partmgr.PartitionManager) *MetricsCache { newCache := MetricsCache{container: container, logger: logger, cfg: cfg, partitionMngr: partMngr} - newCache.cacheMetricMap = map[cacheKey]*MetricState{} - newCache.cacheRefMap = map[uint64]*MetricState{} + newCache.cacheMetricMap = lru.New(cfg.MetricCacheSize) newCache.responseChan = make(chan *v3io.Response, channelSize) newCache.nameUpdateChan = make(chan *v3io.Response, channelSize) @@ -182,12 +172,15 @@ func (mc *MetricsCache) Start() error { } // return metric struct by key -func (mc *MetricsCache) getMetric(name string, hash uint64) (*MetricState, bool) { +func (mc *MetricsCache) getMetric(hash uint64) (*MetricState, bool) { mc.mtx.RLock() defer mc.mtx.RUnlock() - metric, ok := mc.cacheMetricMap[cacheKey{name, hash}] - return metric, ok + metric, ok := mc.cacheMetricMap.Get(hash) + if ok { + return metric.(*MetricState), ok + } + return nil, ok } // create a new metric and save in the map @@ -195,25 +188,13 @@ func (mc *MetricsCache) addMetric(hash uint64, name string, metric *MetricState) mc.mtx.Lock() defer mc.mtx.Unlock() - mc.lastMetric++ - metric.refID = mc.lastMetric - mc.cacheRefMap[mc.lastMetric] = metric - mc.cacheMetricMap[cacheKey{name, hash}] = metric + mc.cacheMetricMap.Add(hash, metric) if _, ok := mc.NameLabelMap[name]; !ok { metric.newName = true mc.NameLabelMap[name] = true } } -// return metric struct by refID -func (mc *MetricsCache) getMetricByRef(ref uint64) (*MetricState, bool) { - mc.mtx.RLock() - defer mc.mtx.RUnlock() - - metric, ok := mc.cacheRefMap[ref] - return metric, ok -} - // Push append to async channel func (mc *MetricsCache) appendTV(metric *MetricState, t int64, v interface{}) { mc.asyncAppendChan <- &asyncAppend{metric: metric, t: t, v: v} @@ -236,21 +217,23 @@ func (mc *MetricsCache) Add(lset utils.LabelsIfc, t int64, v interface{}) (uint6 isValueVariantType = true } - name, key, hash := lset.GetKey() + name, key, _ := lset.GetKey() + hash := lset.HashWithName() err = utils.IsValidMetricName(name) if err != nil { return 0, err } - metric, ok := mc.getMetric(name, hash) + metric, ok := mc.getMetric(hash) var aggrMetrics []*MetricState if !ok { for _, preAggr := range mc.partitionMngr.GetConfig().TableSchemaInfo.PreAggregates { subLset := lset.Filter(preAggr.Labels) - name, key, hash := subLset.GetKey() - aggrMetric, ok := mc.getMetric(name, hash) + name, key, _ := subLset.GetKey() + hash := subLset.HashWithName() + _, ok := mc.getMetric(hash) if !ok { - aggrMetric = &MetricState{Lset: subLset, key: key, name: name, hash: hash} + aggrMetric := &MetricState{Lset: subLset, key: key, name: name, hash: hash} aggrMetric.store = newChunkStore(mc.logger, subLset.LabelNames(), true) mc.addMetric(hash, name, aggrMetric) aggrMetrics = append(aggrMetrics, aggrMetric) @@ -285,7 +268,7 @@ func (mc *MetricsCache) Add(lset utils.LabelsIfc, t int64, v interface{}) (uint6 mc.appendTV(aggrMetric, t, v) } - return metric.refID, err + return hash, err } // fast Add to metric (by refID) @@ -295,11 +278,9 @@ func (mc *MetricsCache) AddFast(ref uint64, t int64, v interface{}) error { if err != nil { return err } - - metric, ok := mc.getMetricByRef(ref) + metric, ok := mc.getMetric(ref) if !ok { - mc.logger.ErrorWith("Ref not found", "ref", ref) - return fmt.Errorf("ref not found") + return fmt.Errorf(fmt.Sprintf("metric not found. ref=%v", ref)) } err = metric.error() diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go b/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go index 32d05592211..9b2c91a1467 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/appender/ingest.go @@ -53,7 +53,7 @@ func (mc *MetricsCache) metricFeed(index int) { for { select { - case _ = <-mc.stopChan: + case <-mc.stopChan: return case app := <-mc.asyncAppendChan: newMetrics := 0 @@ -150,9 +150,9 @@ func (mc *MetricsCache) metricsUpdateLoop(index int) { counter := 0 for { select { - case _ = <-mc.stopChan: + case <-mc.stopChan: return - case _ = <-mc.newUpdates: + case <-mc.newUpdates: // Handle new metric notifications (from metricFeed) for mc.updatesInFlight < mc.cfg.Workers*2 { freeSlots := mc.cfg.Workers*2 - mc.updatesInFlight @@ -390,7 +390,7 @@ func (mc *MetricsCache) nameUpdateRespLoop() { go func() { for { select { - case _ = <-mc.stopChan: + case <-mc.stopChan: return case resp := <-mc.nameUpdateChan: // Handle V3IO PutItem in names table diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/appender/store.go b/vendor/github.com/v3io/v3io-tsdb/pkg/appender/store.go index cce59bb1eab..cf6aa991ebd 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/appender/store.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/appender/store.go @@ -475,7 +475,6 @@ func (cs *chunkStore) writeChunks(mc *MetricsCache, metric *MetricState) (hasPen hasPendingUpdates = true cs.performanceReporter.UpdateHistogram("WriteChunksSizeHistogram", int64(pendingSamplesCount)) - return }) return diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/config/config.go b/vendor/github.com/v3io/v3io-tsdb/pkg/config/config.go index 494f4e5857f..f85bab31b5f 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/config/config.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/config/config.go @@ -76,6 +76,8 @@ const ( PrometheusMetricNameAttribute = "__name__" NamesDirectory = "names" + + MetricCacheSize = 131072 ) type BuildInfo struct { @@ -171,9 +173,9 @@ type V3ioConfig struct { UsePreciseAggregations bool `json:"usePreciseAggregations,omitempty"` // Coefficient to decide whether or not to use server aggregates optimization // use server aggregations if ` / > UseServerAggregateCoefficient` - UseServerAggregateCoefficient int `json:"useServerAggregateCoefficient,omitempty"` - LoadPartitionsFromSchemaAttr bool `json:"loadPartitionsFromSchemaAttr,omitempty"` - RequestChanLength int `json:"RequestChanLength,omitempty"` + UseServerAggregateCoefficient int `json:"useServerAggregateCoefficient,omitempty"` + RequestChanLength int `json:"RequestChanLength,omitempty"` + MetricCacheSize int `json:"MetricCacheSize,omitempty"` } type MetricsReporterConfig struct { @@ -225,8 +227,7 @@ type PartitionSchema struct { } type Partition struct { - StartTime int64 `json:"startTime"` - SchemaInfo PartitionSchema `json:"schemaInfo"` + StartTime int64 `json:"startTime"` } type SchemaField struct { @@ -260,7 +261,6 @@ func GetOrDefaultConfig() (*V3ioConfig, error) { func GetOrLoadFromFile(path string) (*V3ioConfig, error) { once.Do(func() { instance, failure = loadConfig(path) - return }) return instance, failure @@ -269,7 +269,6 @@ func GetOrLoadFromFile(path string) (*V3ioConfig, error) { func GetOrLoadFromData(data []byte) (*V3ioConfig, error) { once.Do(func() { instance, failure = loadFromData(data) - return }) return instance, failure @@ -280,7 +279,6 @@ func GetOrLoadFromStruct(cfg *V3ioConfig) (*V3ioConfig, error) { once.Do(func() { initDefaults(cfg) instance = cfg - return }) return instance, nil @@ -455,4 +453,8 @@ func initDefaults(cfg *V3ioConfig) { if cfg.Password == "" { cfg.Password = os.Getenv("V3IO_PASSWORD") } + + if cfg.MetricCacheSize == 0 { + cfg.MetricCacheSize = MetricCacheSize + } } diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/partmgr/partmgr.go b/vendor/github.com/v3io/v3io-tsdb/pkg/partmgr/partmgr.go index aaf716cc072..4a746e93590 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/partmgr/partmgr.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/partmgr/partmgr.go @@ -25,7 +25,6 @@ import ( "fmt" "math" "path" - "sort" "strconv" "strings" "sync" @@ -50,7 +49,7 @@ func NewPartitionMngr(schemaConfig *config.Schema, cont v3io.Container, v3ioConf return nil, err } newMngr := &PartitionManager{schemaConfig: schemaConfig, cyclic: false, container: cont, currentPartitionInterval: currentPartitionInterval, v3ioConfig: v3ioConfig} - err = newMngr.updatePartitionsFromSchema(schemaConfig, nil) + err = newMngr.updatePartitionsFromSchema(schemaConfig) if err != nil { return nil, err } @@ -141,7 +140,7 @@ func (p *PartitionManager) createAndUpdatePartition(t int64) (*DBPartition, erro } p.currentPartitionInterval = partition.partitionInterval - schemaPartition := &config.Partition{StartTime: partition.startTime, SchemaInfo: p.schemaConfig.PartitionSchemaInfo} + schemaPartition := &config.Partition{StartTime: partition.startTime} if p.headPartition == nil || time > p.headPartition.startTime { p.headPartition = partition p.partitions = append(p.partitions, partition) @@ -182,24 +181,6 @@ func (p *PartitionManager) updateSchema() error { err = p.container.PutObjectSync(&v3io.PutObjectInput{Path: schemaFilePath, Body: data}) if err != nil { outerError = err - return - } - attributes := make(map[string]interface{}, len(p.partitions)) - for _, part := range p.partitions { - marshalledPartition, err := json.Marshal(part.ToMap()) - if err != nil { - outerError = err - return - } - attributes[part.GetPartitionAttributeName()] = marshalledPartition - } - - input := &v3io.PutItemInput{Path: schemaFilePath, Attributes: attributes} - _, err := p.container.PutItemSync(input) - - if err != nil { - outerError = errors.Wrap(err, "failed to update partitions table.") - return } } }) @@ -256,7 +237,7 @@ func (p *PartitionManager) ReadAndUpdateSchema() (err error) { err = errors.Wrap(err, "Failed to create timer ReadAndUpdateSchemaTimer.") return } - schemaInfoResp, err := p.container.GetItemSync(&v3io.GetItemInput{Path: schemaFilePath, AttributeNames: []string{"**"}}) + schemaInfoResp, err := p.container.GetItemSync(&v3io.GetItemInput{Path: schemaFilePath, AttributeNames: []string{"__mtime_secs", "__mtime_nsecs"}}) if err != nil { err = errors.Wrapf(err, "Failed to read schema at path '%s'.", schemaFilePath) return @@ -280,29 +261,13 @@ func (p *PartitionManager) ReadAndUpdateSchema() (err error) { p.schemaMtimeNanosecs = mtimeNsecs metricReporter.WithTimer("ReadAndUpdateSchemaTimer", func() { - err = p.updatePartitionsFromSchema(nil, schemaGetItemResponse) - return + err = p.updatePartitionsFromSchema(nil) }) } return } -func (p *PartitionManager) updatePartitionsFromSchema(schemaConfig *config.Schema, schemaGetItemResponse *v3io.GetItemOutput) error { - var currentSchemaVersion int - if schemaConfig == nil { - currentSchemaVersion = p.schemaConfig.TableSchemaInfo.Version - } else { - currentSchemaVersion = schemaConfig.TableSchemaInfo.Version - } - - if currentSchemaVersion == 4 && p.v3ioConfig.LoadPartitionsFromSchemaAttr { - return p.newLoadPartitions(schemaGetItemResponse) - } - - return p.oldLoadPartitions(schemaConfig) -} - -func (p *PartitionManager) oldLoadPartitions(schema *config.Schema) error { +func (p *PartitionManager) updatePartitionsFromSchema(schema *config.Schema) error { if schema == nil { schemaFilePath := p.GetSchemaFilePath() resp, innerError := p.container.GetObjectSync(&v3io.GetObjectInput{Path: schemaFilePath}) @@ -334,58 +299,6 @@ func (p *PartitionManager) oldLoadPartitions(schema *config.Schema) error { return nil } -func (p *PartitionManager) newLoadPartitions(schemaAttributesResponse *v3io.GetItemOutput) error { - if p.container == nil { // Tests use case only - return nil - } - - if schemaAttributesResponse == nil { - schemaFilePath := p.GetSchemaFilePath() - schemaInfoResp, err := p.container.GetItemSync(&v3io.GetItemInput{Path: schemaFilePath, AttributeNames: []string{"*"}}) - if err != nil { - return errors.Wrapf(err, "Failed to read schema at path '%s'.", schemaFilePath) - } - - schemaAttributesResponse = schemaInfoResp.Output.(*v3io.GetItemOutput) - } - - p.partitions = []*DBPartition{} - for partitionStartTime, partitionAttrBlob := range schemaAttributesResponse.Item { - // Only process "partition" attributes - if !strings.HasPrefix(partitionStartTime, partitionAttributePrefix) { - continue - } - intStartTime, err := strconv.ParseInt(partitionStartTime[1:], 10, 64) - if err != nil { - return errors.Wrapf(err, "invalid partition name '%v'", partitionStartTime) - } - - partPath := path.Join(p.Path(), strconv.FormatInt(intStartTime/1000, 10)) + "/" - - partitionAttr := make(map[string]interface{}, 5) - err = json.Unmarshal(partitionAttrBlob.([]byte), &partitionAttr) - if err != nil { - return err - } - newPart, err := NewDBPartitionFromMap(p, intStartTime, partPath, partitionAttr) - if err != nil { - return err - } - p.partitions = append(p.partitions, newPart) - if p.headPartition == nil { - p.headPartition = newPart - } else if p.headPartition.startTime < newPart.startTime { - p.headPartition = newPart - } - } - - sort.SliceStable(p.partitions, func(i, j int) bool { - return p.partitions[i].startTime < p.partitions[j].startTime - }) - - return nil -} - //if inclusive is true than partial partitions (not fully in range) will be retireved as well func (p *PartitionManager) PartsForRange(mint, maxt int64, inclusive bool) []*DBPartition { var parts []*DBPartition diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/collector.go b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/collector.go index 820cbd87396..7bd2aa63eed 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/collector.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/collector.go @@ -66,7 +66,7 @@ func mainCollector(ctx *selectQueryContext, responseChannel chan *qryResults) { for { select { - case _ = <-ctx.stopChan: + case <-ctx.stopChan: return case res, ok := <-responseChannel: if !ok { diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/querier.go b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/querier.go index 573af6a0cc2..338286439f7 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/querier.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/querier.go @@ -187,7 +187,6 @@ func (q *V3ioQuerier) baseSelectQry(params *SelectParams, showAggregateLabel boo } iter, err = selectContext.start(parts, params) - return }) return @@ -227,7 +226,7 @@ func (q *V3ioQuerier) getMetricNames() ([]string, error) { metricNames = append(metricNames, iter.GetField(config.ObjectNameAttrName).(string)) } - sort.Sort(sort.StringSlice(metricNames)) + sort.Strings(metricNames) if iter.Err() != nil { return nil, fmt.Errorf("failed to read metric names; err = %v", iter.Err().Error()) diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/select.go b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/select.go index 053610d797f..01d14ab482c 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/select.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/select.go @@ -289,7 +289,6 @@ func (queryCtx *selectQueryContext) startCollectors() error { } close(queryCtx.finalErrorChan) - return }() return nil @@ -497,7 +496,7 @@ func (queryCtx *selectQueryContext) generateTimeColumn() Column { for t := queryCtx.queryParams.From; t <= queryCtx.queryParams.To; t += queryCtx.queryParams.Step { err := timeColumn.SetDataAt(i, time.Unix(t/1000, (t%1000)*1e6)) if err != nil { - queryCtx.logger.ErrorWith(errors.Wrap(err, fmt.Sprintf("could not set data"))) + queryCtx.logger.ErrorWith(errors.Wrap(err, "could not set data")) } else { i++ } @@ -588,10 +587,7 @@ func (query *partQuery) getItems(ctx *selectQueryContext) error { } func (query *partQuery) Next() bool { - var res bool - - res = query.iter.Next() - return res + return query.iter.Next() } func (query *partQuery) GetField(name string) interface{} { diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/sql_parser.go b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/sql_parser.go index ccb5aefd2e2..b1450d114c5 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/sql_parser.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/pquerier/sql_parser.go @@ -117,7 +117,7 @@ func parseFuncExpr(expr *sqlparser.FuncExpr, destCol *RequestedColumn) error { case *sqlparser.FuncExpr: err := parseFuncExpr(innerExpr, destCol) if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not parse expr")) + return errors.Wrap(err, "could not parse expr") } } } diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/querier/querier.go b/vendor/github.com/v3io/v3io-tsdb/pkg/querier/querier.go index 9487978916a..131b9df3960 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/querier/querier.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/querier/querier.go @@ -169,7 +169,6 @@ func (q *V3ioQuerier) selectQry(params *SelectParams) (set utils.SeriesSet, err } set, err = newIterSortMerger(sets) - return }) return @@ -284,7 +283,7 @@ func (q *V3ioQuerier) getMetricNames() ([]string, error) { metricNames = append(metricNames, iter.GetField("__name").(string)) } - sort.Sort(sort.StringSlice(metricNames)) + sort.Strings(metricNames) if iter.Err() != nil { q.logger.InfoWith("Failed to read metric names; returning an empty list.", "err", iter.Err().Error()) diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/tsdb/v3iotsdb.go b/vendor/github.com/v3io/v3io-tsdb/pkg/tsdb/v3iotsdb.go index 761c854c1c1..fb7c39cf254 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/tsdb/v3iotsdb.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/tsdb/v3iotsdb.go @@ -44,7 +44,6 @@ import ( "github.com/v3io/v3io-tsdb/pkg/partmgr" "github.com/v3io/v3io-tsdb/pkg/pquerier" "github.com/v3io/v3io-tsdb/pkg/querier" - "github.com/v3io/v3io-tsdb/pkg/tsdb/schema" "github.com/v3io/v3io-tsdb/pkg/utils" ) @@ -202,12 +201,6 @@ func (a *V3ioAdapter) connect() error { return errors.Wrapf(err, "Failed to unmarshal the TSDB schema at '%s', got: %v .", fullpath, string(resp.Body())) } - // in order to support backward compatibility we do not fail on version mismatch and only logging warning - if a.cfg.LoadPartitionsFromSchemaAttr && tableSchema.TableSchemaInfo.Version != schema.Version { - a.logger.Warn("Table Schema version mismatch - existing table schema version is %d while the tsdb library version is %d! Make sure to create the table with same library version", - tableSchema.TableSchemaInfo.Version, schema.Version) - } - a.partitionMngr, err = partmgr.NewPartitionMngr(&tableSchema, a.container, a.cfg) if err != nil { return errors.Wrapf(err, "Failed to create a TSDB partition manager at '%s'.", fullpath) @@ -491,7 +484,7 @@ func getItemsWorker(logger logger.Logger, container v3io.Container, input *v3io. filesToDeleteChan chan<- v3io.Item, terminationChan chan<- error, onErrorTerminationChannel <-chan struct{}) { for { select { - case _ = <-onErrorTerminationChannel: + case <-onErrorTerminationChannel: terminationChan <- nil return default: @@ -511,7 +504,7 @@ func getItemsWorker(logger logger.Logger, container v3io.Container, input *v3io. // In case we got error on delete while iterating getItems response select { - case _ = <-onErrorTerminationChannel: + case <-onErrorTerminationChannel: terminationChan <- nil return default: @@ -532,7 +525,7 @@ func deleteObjectWorker(container v3io.Container, deleteParams *DeleteParams, lo aggrMask aggregate.AggrType) { for { select { - case _ = <-onErrorTerminationChannel: + case <-onErrorTerminationChannel: return case itemToDelete, ok := <-filesToDeleteChannel: if !ok { @@ -809,7 +802,7 @@ func (a v3ioAppender) Add(lset utils.Labels, t int64, v interface{}) (uint64, er } // Faster Add using refID obtained from Add (avoid some hash/lookup overhead) -func (a v3ioAppender) AddFast(lset utils.Labels, ref uint64, t int64, v interface{}) error { +func (a v3ioAppender) AddFast(ref uint64, t int64, v interface{}) error { return a.metricsCache.AddFast(ref, t, v) } @@ -829,7 +822,7 @@ func (a v3ioAppender) Rollback() error { return nil } // The Appender interface provides batched appends against a storage. type Appender interface { Add(l utils.Labels, t int64, v interface{}) (uint64, error) - AddFast(l utils.Labels, ref uint64, t int64, v interface{}) error + AddFast(ref uint64, t int64, v interface{}) error WaitForCompletion(timeout time.Duration) (int, error) Commit() error Rollback() error diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/utils/labels.go b/vendor/github.com/v3io/v3io-tsdb/pkg/utils/labels.go index a29e234c1ad..088c77ef8ce 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/utils/labels.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/utils/labels.go @@ -49,6 +49,7 @@ type LabelsIfc interface { GetExpr() string Filter([]string) LabelsIfc LabelNames() []string + HashWithName() uint64 } func (ls Labels) Filter(keep []string) LabelsIfc { @@ -195,6 +196,24 @@ func (ls Labels) Hash() uint64 { return hash.Sum64() } +func (ls Labels) HashWithName() uint64 { + b := make([]byte, 0, 1024) + + for _, v := range ls { + b = append(b, v.Name...) + b = append(b, sep) + b = append(b, v.Value...) + b = append(b, sep) + } + + hash := xxhash.New() + _, err := hash.Write(b) + if err != nil { + return 0 + } + return hash.Sum64() +} + // Copy returns a copy of the labels. func (ls Labels) Copy() Labels { res := make(Labels, len(ls)) diff --git a/vendor/github.com/v3io/v3io-tsdb/pkg/utils/validators.go b/vendor/github.com/v3io/v3io-tsdb/pkg/utils/validators.go index 05abce58862..300d1956ffc 100644 --- a/vendor/github.com/v3io/v3io-tsdb/pkg/utils/validators.go +++ b/vendor/github.com/v3io/v3io-tsdb/pkg/utils/validators.go @@ -22,7 +22,7 @@ func IsValidMetricName(name string) error { } if !metricNameValidationRegex.Match([]byte(trimmed)) { - return fmt.Errorf("metric name containes illegal characters. Name should conform to '%s'", + return fmt.Errorf("metric name contains illegal characters. Name should conform to '%s'", metricNameValidationRegexStr) } @@ -36,7 +36,7 @@ func IsValidLabelName(labelName string) error { } if !labelValidationRegex.Match([]byte(trimmed)) { - return fmt.Errorf("label name containes illegal characters. Label name should conform to '%s'", + return fmt.Errorf("label name contains illegal characters. Label name should conform to '%s'", labelValidationRegexStr) } diff --git a/vendor/golang.org/x/net/http/httpproxy/proxy.go b/vendor/golang.org/x/net/http/httpproxy/proxy.go new file mode 100644 index 00000000000..163645b86f3 --- /dev/null +++ b/vendor/golang.org/x/net/http/httpproxy/proxy.go @@ -0,0 +1,370 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpproxy provides support for HTTP proxy determination +// based on environment variables, as provided by net/http's +// ProxyFromEnvironment function. +// +// The API is not subject to the Go 1 compatibility promise and may change at +// any time. +package httpproxy + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "unicode/utf8" + + "golang.org/x/net/idna" +) + +// Config holds configuration for HTTP proxy settings. See +// FromEnvironment for details. +type Config struct { + // HTTPProxy represents the value of the HTTP_PROXY or + // http_proxy environment variable. It will be used as the proxy + // URL for HTTP requests and HTTPS requests unless overridden by + // HTTPSProxy or NoProxy. + HTTPProxy string + + // HTTPSProxy represents the HTTPS_PROXY or https_proxy + // environment variable. It will be used as the proxy URL for + // HTTPS requests unless overridden by NoProxy. + HTTPSProxy string + + // NoProxy represents the NO_PROXY or no_proxy environment + // variable. It specifies a string that contains comma-separated values + // specifying hosts that should be excluded from proxying. Each value is + // represented by an IP address prefix (1.2.3.4), an IP address prefix in + // CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*). + // An IP address prefix and domain name can also include a literal port + // number (1.2.3.4:80). + // A domain name matches that name and all subdomains. A domain name with + // a leading "." matches subdomains only. For example "foo.com" matches + // "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com". + // A single asterisk (*) indicates that no proxying should be done. + // A best effort is made to parse the string and errors are + // ignored. + NoProxy string + + // CGI holds whether the current process is running + // as a CGI handler (FromEnvironment infers this from the + // presence of a REQUEST_METHOD environment variable). + // When this is set, ProxyForURL will return an error + // when HTTPProxy applies, because a client could be + // setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy. + CGI bool +} + +// config holds the parsed configuration for HTTP proxy settings. +type config struct { + // Config represents the original configuration as defined above. + Config + + // httpsProxy is the parsed URL of the HTTPSProxy if defined. + httpsProxy *url.URL + + // httpProxy is the parsed URL of the HTTPProxy if defined. + httpProxy *url.URL + + // ipMatchers represent all values in the NoProxy that are IP address + // prefixes or an IP address in CIDR notation. + ipMatchers []matcher + + // domainMatchers represent all values in the NoProxy that are a domain + // name or hostname & domain name + domainMatchers []matcher +} + +// FromEnvironment returns a Config instance populated from the +// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the +// lowercase versions thereof). HTTPS_PROXY takes precedence over +// HTTP_PROXY for https requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. An error +// is returned if the value is a different form. +func FromEnvironment() *Config { + return &Config{ + HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"), + HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"), + NoProxy: getEnvAny("NO_PROXY", "no_proxy"), + CGI: os.Getenv("REQUEST_METHOD") != "", + } +} + +func getEnvAny(names ...string) string { + for _, n := range names { + if val := os.Getenv(n); val != "" { + return val + } + } + return "" +} + +// ProxyFunc returns a function that determines the proxy URL to use for +// a given request URL. Changing the contents of cfg will not affect +// proxy functions created earlier. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, as +// defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without a +// port number), then a nil URL and nil error will be returned. +func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) { + // Preprocess the Config settings for more efficient evaluation. + cfg1 := &config{ + Config: *cfg, + } + cfg1.init() + return cfg1.proxyForURL +} + +func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) { + var proxy *url.URL + if reqURL.Scheme == "https" { + proxy = cfg.httpsProxy + } + if proxy == nil { + proxy = cfg.httpProxy + if proxy != nil && cfg.CGI { + return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } + } + if proxy == nil { + return nil, nil + } + if !cfg.useProxy(canonicalAddr(reqURL)) { + return nil, nil + } + + return proxy, nil +} + +func parseProxy(proxy string) (*url.URL, error) { + if proxy == "" { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func (cfg *config) useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + ip := net.ParseIP(host) + if ip != nil { + if ip.IsLoopback() { + return false + } + } + + addr = strings.ToLower(strings.TrimSpace(host)) + + if ip != nil { + for _, m := range cfg.ipMatchers { + if m.match(addr, port, ip) { + return false + } + } + } + for _, m := range cfg.domainMatchers { + if m.match(addr, port, ip) { + return false + } + } + return true +} + +func (c *config) init() { + if parsed, err := parseProxy(c.HTTPProxy); err == nil { + c.httpProxy = parsed + } + if parsed, err := parseProxy(c.HTTPSProxy); err == nil { + c.httpsProxy = parsed + } + + for _, p := range strings.Split(c.NoProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + + if p == "*" { + c.ipMatchers = []matcher{allMatch{}} + c.domainMatchers = []matcher{allMatch{}} + return + } + + // IPv4/CIDR, IPv6/CIDR + if _, pnet, err := net.ParseCIDR(p); err == nil { + c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet}) + continue + } + + // IPv4:port, [IPv6]:port + phost, pport, err := net.SplitHostPort(p) + if err == nil { + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + if phost[0] == '[' && phost[len(phost)-1] == ']' { + phost = phost[1 : len(phost)-1] + } + } else { + phost = p + } + // IPv4, IPv6 + if pip := net.ParseIP(phost); pip != nil { + c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport}) + continue + } + + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + + // domain.com or domain.com:80 + // foo.com matches bar.foo.com + // .domain.com or .domain.com:port + // *.domain.com or *.domain.com:port + if strings.HasPrefix(phost, "*.") { + phost = phost[1:] + } + matchHost := false + if phost[0] != '.' { + matchHost = true + phost = "." + phost + } + c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost}) + } +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if isASCII(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// matcher represents the matching rule for a given value in the NO_PROXY list +type matcher interface { + // match returns true if the host and optional port or ip and optional port + // are allowed + match(host, port string, ip net.IP) bool +} + +// allMatch matches on all possible inputs +type allMatch struct{} + +func (a allMatch) match(host, port string, ip net.IP) bool { + return true +} + +type cidrMatch struct { + cidr *net.IPNet +} + +func (m cidrMatch) match(host, port string, ip net.IP) bool { + return m.cidr.Contains(ip) +} + +type ipMatch struct { + ip net.IP + port string +} + +func (m ipMatch) match(host, port string, ip net.IP) bool { + if m.ip.Equal(ip) { + return m.port == "" || m.port == port + } + return false +} + +type domainMatch struct { + host string + port string + + matchHost bool +} + +func (m domainMatch) match(host, port string, ip net.IP) bool { + if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) { + return m.port == "" || m.port == port + } + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index fd96fb3bb9b..905bfd9ef92 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -147,6 +147,8 @@ github.com/gogo/protobuf/vanity github.com/gogo/protobuf/vanity/command # github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/glog +# github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 +github.com/golang/groupcache/lru # github.com/golang/protobuf v1.3.2 github.com/golang/protobuf/descriptor github.com/golang/protobuf/jsonpb @@ -333,7 +335,7 @@ github.com/soheilhy/cmux github.com/stretchr/testify/assert github.com/stretchr/testify/require github.com/stretchr/testify/suite -# github.com/v3io/frames v0.7.33 => github.com/v3io/frames v0.7.36 +# github.com/v3io/frames v0.7.33 => github.com/v3io/frames v0.8.1 github.com/v3io/frames github.com/v3io/frames/pb # github.com/v3io/v3io-go v0.1.9 @@ -341,7 +343,7 @@ github.com/v3io/v3io-go/pkg/dataplane github.com/v3io/v3io-go/pkg/dataplane/http github.com/v3io/v3io-go/pkg/dataplane/schemas/node/common github.com/v3io/v3io-go/pkg/errors -# github.com/v3io/v3io-tsdb v0.10.12 => github.com/v3io/v3io-tsdb v0.10.12 +# github.com/v3io/v3io-tsdb v0.11.2 => github.com/v3io/v3io-tsdb v0.11.2 github.com/v3io/v3io-tsdb/internal/pkg/performance github.com/v3io/v3io-tsdb/pkg/aggregate github.com/v3io/v3io-tsdb/pkg/appender @@ -404,6 +406,7 @@ golang.org/x/net/bpf golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/http/httpguts +golang.org/x/net/http/httpproxy golang.org/x/net/http2 golang.org/x/net/http2/hpack golang.org/x/net/idna