-
Notifications
You must be signed in to change notification settings - Fork 104
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
spike mode for dynamic series adjustment #70
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -198,8 +198,32 @@ func handleGradualChangeMode(metricCount, metricLength, metricCycle, seriesCycle | |||||||||||||||||||||
} | ||||||||||||||||||||||
} | ||||||||||||||||||||||
|
||||||||||||||||||||||
func handleSpikeMode(metricCount, metricLength, metricCycle, seriesCycle int, labelKeys, labelValues []string, currentSeriesCount *int, spikeMultiplier float64, changeSeriesChan <-chan time.Time, updateNotify chan struct{}) { | ||||||||||||||||||||||
initialSeriesCount := *currentSeriesCount | ||||||||||||||||||||||
for tick := range changeSeriesChan { | ||||||||||||||||||||||
metricsMux.Lock() | ||||||||||||||||||||||
unregisterMetrics() | ||||||||||||||||||||||
registerMetrics(metricCount, metricLength, metricCycle, labelKeys) | ||||||||||||||||||||||
cycleValues(labelKeys, labelValues, *currentSeriesCount, seriesCycle) | ||||||||||||||||||||||
metricsMux.Unlock() | ||||||||||||||||||||||
|
||||||||||||||||||||||
if *currentSeriesCount > initialSeriesCount { | ||||||||||||||||||||||
*currentSeriesCount = initialSeriesCount | ||||||||||||||||||||||
} else { | ||||||||||||||||||||||
*currentSeriesCount = int(float64(initialSeriesCount) * spikeMultiplier) | ||||||||||||||||||||||
} | ||||||||||||||||||||||
|
||||||||||||||||||||||
fmt.Printf("%v: Adjusting series count. New count: %d\n", tick, *currentSeriesCount) | ||||||||||||||||||||||
|
||||||||||||||||||||||
select { | ||||||||||||||||||||||
case updateNotify <- struct{}{}: | ||||||||||||||||||||||
default: | ||||||||||||||||||||||
Comment on lines
+218
to
+220
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not sure what's happening here, and looking at the other There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the updateNotify channel are used here to: Lines 140 to 149 in 4b732e1
Each time a notification is received, the metrics are updated and logged. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You're right! We learned something. It looks like |
||||||||||||||||||||||
} | ||||||||||||||||||||||
} | ||||||||||||||||||||||
} | ||||||||||||||||||||||
|
||||||||||||||||||||||
// RunMetrics creates a set of Prometheus test series that update over time | ||||||||||||||||||||||
func RunMetrics(metricCount, labelCount, seriesCount, seriesChangeRate, maxSeriesCount, minSeriesCount, metricLength, labelLength, valueInterval, seriesInterval, metricInterval, seriesChangeInterval int, seriesOperationMode string, constLabels []string, stop chan struct{}) (chan struct{}, error) { | ||||||||||||||||||||||
func RunMetrics(metricCount, labelCount, seriesCount, seriesChangeRate, maxSeriesCount, minSeriesCount, metricLength, labelLength, valueInterval, seriesInterval, metricInterval, seriesChangeInterval int, spikeMultiplier float64, seriesOperationMode string, constLabels []string, stop chan struct{}) (chan struct{}, error) { | ||||||||||||||||||||||
labelKeys := make([]string, labelCount) | ||||||||||||||||||||||
for idx := 0; idx < labelCount; idx++ { | ||||||||||||||||||||||
labelKeys[idx] = fmt.Sprintf("label_key_%s_%v", strings.Repeat("k", labelLength), idx) | ||||||||||||||||||||||
|
@@ -249,6 +273,17 @@ func RunMetrics(metricCount, labelCount, seriesCount, seriesChangeRate, maxSerie | |||||||||||||||||||||
go handleValueTicks(&labelKeys, &labelValues, ¤tSeriesCount, &seriesCycle, updateNotify, valueTick) | ||||||||||||||||||||||
go handleSeriesTicks(&labelKeys, &labelValues, ¤tSeriesCount, &seriesCycle, updateNotify, seriesTick) | ||||||||||||||||||||||
|
||||||||||||||||||||||
case "spike": | ||||||||||||||||||||||
if spikeMultiplier < 1 { | ||||||||||||||||||||||
return nil, fmt.Errorf("error: spikeMultiplier must be greater than or equal to 1, got %f", spikeMultiplier) | ||||||||||||||||||||||
} | ||||||||||||||||||||||
registerMetrics(metricCount, metricLength, metricCycle, labelKeys) | ||||||||||||||||||||||
cycleValues(labelKeys, labelValues, currentSeriesCount, seriesCycle) | ||||||||||||||||||||||
fmt.Printf("Starting spike mode; initial series: %d, spike multiplier: %f, spike interval: %v\n", currentSeriesCount, spikeMultiplier, seriesChangeInterval) | ||||||||||||||||||||||
go handleSpikeMode(metricCount, metricLength, metricCycle, seriesCycle, labelKeys, labelValues, ¤tSeriesCount, spikeMultiplier, changeSeriesTick.C, updateNotify) | ||||||||||||||||||||||
go handleValueTicks(&labelKeys, &labelValues, ¤tSeriesCount, &seriesCycle, updateNotify, valueTick) | ||||||||||||||||||||||
go handleSeriesTicks(&labelKeys, &labelValues, ¤tSeriesCount, &seriesCycle, updateNotify, seriesTick) | ||||||||||||||||||||||
|
||||||||||||||||||||||
default: | ||||||||||||||||||||||
registerMetrics(metricCount, metricLength, metricCycle, labelKeys) | ||||||||||||||||||||||
cycleValues(labelKeys, labelValues, currentSeriesCount, seriesCycle) | ||||||||||||||||||||||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We don't need to do it now, but we should look at breaking
RunMetrics
into separate functions for each of the run modes. The current functions such as this newhandleSpikeMode
could create the other go routines that we create withinRunMetrics
currently.This would mean we don't have to pass so many parameters around across a few functions.