Skip to content

Commit

Permalink
Changed aggregateByLevels to return tree structure
Browse files Browse the repository at this point in the history 
Signed-off-by: Peter Alfonsi <[email protected]>
  • Loading branch information
Peter Alfonsi committed Mar 27, 2024
1 parent 5ecdcff commit fb3baaa
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 102 deletions.
94 changes: 49 additions & 45 deletions server/src/main/java/org/opensearch/common/cache/stats/MultiDimensionCacheStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
Expand Down Expand Up @@ -89,61 +88,66 @@ public long getTotalEntries() {
return getTotalStats().getEntries();
}

static class DimensionNode {
private final String dimensionValue;
// Storing dimensionValue is useful for producing XContent
final TreeMap<String, DimensionNode> children; // Map from dimensionValue to the DimensionNode for that dimension value
private CounterSnapshot snapshot;

DimensionNode(String dimensionValue) {
this.dimensionValue = dimensionValue;
this.children = new TreeMap<>();
this.snapshot = null;
// Only leaf nodes have non-null snapshots. Might make it be sum-of-children in future.
}

/**
* Increments the snapshot in this node.
*/
void addSnapshot(CounterSnapshot newSnapshot) {
if (snapshot == null) {
snapshot = newSnapshot;
} else {
snapshot = CounterSnapshot.addSnapshots(snapshot, newSnapshot);
}
}

/**
* Returns the node found by following these dimension values down from the current node.
* If such a node does not exist, creates it.
*/
DimensionNode getNode(List<String> dimensionValues) {
DimensionNode current = this;
for (String dimensionValue : dimensionValues) {
current.children.putIfAbsent(dimensionValue, new DimensionNode(dimensionValue));
current = current.children.get(dimensionValue);
}
return current;
}

CounterSnapshot getSnapshot() {
return snapshot;
}
}

/**
* Return a TreeMap containing stats values aggregated by the levels passed in. Results are ordered so that
* values are grouped by their dimension values, which matches the order they should be outputted in an API response.
* Example: if the dimension names are "indices", "shards", and "tier", and levels are "indices" and "shards", it
* groups the stats by indices and shard values and returns them in order.
* Pkg-private for testing.
* @param levels The levels to aggregate by
* @return The resulting stats
* Returns a tree containing the stats aggregated by the levels passed in. The root node is a dummy node,
* whose name and value are null.
*/
TreeMap<StatsHolder.Key, CounterSnapshot> aggregateByLevels(List<String> levels) {
DimensionNode aggregateByLevels(List<String> levels) {
int[] levelPositions = getLevelsInSortedOrder(levels); // Check validity of levels and get their indices in dimensionNames
TreeMap<StatsHolder.Key, CounterSnapshot> result = new TreeMap<>(new KeyComparator());

DimensionNode root = new DimensionNode(null);
for (Map.Entry<StatsHolder.Key, CounterSnapshot> entry : snapshot.entrySet()) {
List<String> levelValues = new ArrayList<>(); // This key's relevant dimension values, which match the levels
List<String> keyDimensionValues = entry.getKey().dimensionValues;
for (int levelPosition : levelPositions) {
levelValues.add(keyDimensionValues.get(levelPosition));
}
// The new keys, for the aggregated stats, contain only the dimensions specified in levels
StatsHolder.Key levelsKey = new StatsHolder.Key(levelValues);
CounterSnapshot originalCounter = entry.getValue();
// Increment existing key in aggregation with this value, or create a new one if it's not present.
result.compute(
levelsKey,
(k, v) -> (v == null) ? originalCounter : CounterSnapshot.addSnapshots(result.get(levelsKey), originalCounter)
);
}
return result;
}

public TreeMap<StatsHolder.Key, CounterSnapshot> getSortedMap() {
TreeMap<StatsHolder.Key, CounterSnapshot> result = new TreeMap<>(new KeyComparator());
result.putAll(snapshot);
return result;
}

// First compare outermost dimension, then second outermost, etc.
// Pkg-private for testing
static class KeyComparator implements Comparator<StatsHolder.Key> {
@Override
public int compare(StatsHolder.Key k1, StatsHolder.Key k2) {
assert k1.dimensionValues.size() == k2.dimensionValues.size();
for (int i = 0; i < k1.dimensionValues.size(); i++) {
String value1 = k1.dimensionValues.get(i);
String value2 = k2.dimensionValues.get(i);
int compareValue = value1.compareTo(value2);
if (compareValue != 0) {
// If the values aren't equal for this dimension, return
return compareValue;
}
}
// If all dimension values have been equal, the keys overall are equal
return 0;
DimensionNode leafNode = root.getNode(levelValues);
leafNode.addSnapshot(entry.getValue());
}
return root;
}

private int[] getLevelsInSortedOrder(List<String> levels) {
Expand Down
100 changes: 49 additions & 51 deletions server/src/test/java/org/opensearch/common/cache/stats/MultiDimensionCacheStatsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,12 @@ public void testAddAndGet() throws Exception {
List<String> dimensionNames = List.of("dim1", "dim2", "dim3", "dim4");
StatsHolder statsHolder = new StatsHolder(dimensionNames);
Map<String, List<String>> usedDimensionValues = getUsedDimensionValues(statsHolder, 10);
Map<Set<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10);
Map<List<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10);
MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats();

// test the value in the map is as expected for each distinct combination of values
for (Set<CacheStatsDimension> dimSet : expected.keySet()) {
CacheStatsCounter expectedCounter = expected.get(dimSet);
List<CacheStatsDimension> dims = new ArrayList<>(dimSet);
for (List<CacheStatsDimension> dims : expected.keySet()) {
CacheStatsCounter expectedCounter = expected.get(dims);
StatsHolder.Key key = new StatsHolder.Key(StatsHolder.getOrderedDimensionValues(dims, dimensionNames));
CounterSnapshot actual = stats.snapshot.get(key);

Expand All @@ -60,8 +59,8 @@ public void testAddAndGet() throws Exception {

// test gets for total
CacheStatsCounter expectedTotal = new CacheStatsCounter();
for (Set<CacheStatsDimension> dimSet : expected.keySet()) {
expectedTotal.add(expected.get(dimSet));
for (List<CacheStatsDimension> dims : expected.keySet()) {
expectedTotal.add(expected.get(dims));
}
assertEquals(expectedTotal.snapshot(), stats.getTotalStats());

Expand All @@ -83,49 +82,29 @@ public void testEmptyDimsList() throws Exception {
assertEquals(stats.getTotalStats(), stats.snapshot.get(new StatsHolder.Key(List.of())));
}

public void testKeyComparator() throws Exception {
MultiDimensionCacheStats.KeyComparator comp = new MultiDimensionCacheStats.KeyComparator();
StatsHolder.Key k1 = new StatsHolder.Key(List.of("a", "b", "c"));
StatsHolder.Key k2 = new StatsHolder.Key(List.of("a", "b", "d"));
StatsHolder.Key k3 = new StatsHolder.Key(List.of("b", "a", "a"));
StatsHolder.Key k4 = new StatsHolder.Key(List.of("a", "a", "e"));
StatsHolder.Key k5 = new StatsHolder.Key(List.of("a", "b", "c"));

// expected order: k4 < k1 = k5 < k2 < k3
assertTrue(comp.compare(k4, k1) < 0);
assertTrue(comp.compare(k1, k5) == 0);
assertTrue(comp.compare(k1, k2) < 0);
assertTrue(comp.compare(k5, k2) < 0);
assertTrue(comp.compare(k2, k3) < 0);
}

public void testAggregateByAllDimensions() throws Exception {
// Aggregating with all dimensions as levels should just give us the same values that were in the original map
List<String> dimensionNames = List.of("dim1", "dim2", "dim3", "dim4");
StatsHolder statsHolder = new StatsHolder(dimensionNames);
Map<String, List<String>> usedDimensionValues = getUsedDimensionValues(statsHolder, 10);
Map<Set<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10);
Map<List<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10);
MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats();

Map<StatsHolder.Key, CounterSnapshot> aggregated = stats.aggregateByLevels(dimensionNames);
for (Map.Entry<StatsHolder.Key, CounterSnapshot> aggregatedEntry : aggregated.entrySet()) {
StatsHolder.Key aggregatedKey = aggregatedEntry.getKey();

Set<CacheStatsDimension> expectedKey = new HashSet<>();
for (int i = 0; i < dimensionNames.size(); i++) {
expectedKey.add(new CacheStatsDimension(dimensionNames.get(i), aggregatedKey.dimensionValues.get(i)));
MultiDimensionCacheStats.DimensionNode aggregated = stats.aggregateByLevels(dimensionNames);
for (Map.Entry<List<CacheStatsDimension>, CacheStatsCounter> expectedEntry : expected.entrySet()) {
List<String> dimensionValues = new ArrayList<>();
for (CacheStatsDimension dim : expectedEntry.getKey()) {
dimensionValues.add(dim.dimensionValue);
}
CacheStatsCounter expectedCounter = expected.get(expectedKey);
assertEquals(expectedCounter.snapshot(), aggregatedEntry.getValue());
assertEquals(expectedEntry.getValue().snapshot(), aggregated.getNode(dimensionValues).getSnapshot());
}
assertEquals(expected.size(), aggregated.size());
}

public void testAggregateBySomeDimensions() throws Exception {
List<String> dimensionNames = List.of("dim1", "dim2", "dim3", "dim4");
StatsHolder statsHolder = new StatsHolder(dimensionNames);
Map<String, List<String>> usedDimensionValues = getUsedDimensionValues(statsHolder, 10);
Map<Set<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10);
Map<List<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10);
MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats();

for (int i = 0; i < (1 << dimensionNames.size()); i++) {
Expand All @@ -139,25 +118,45 @@ public void testAggregateBySomeDimensions() throws Exception {
if (levels.size() == 0) {
assertThrows(IllegalArgumentException.class, () -> stats.aggregateByLevels(levels));
} else {
Map<StatsHolder.Key, CounterSnapshot> aggregated = stats.aggregateByLevels(levels);
for (Map.Entry<StatsHolder.Key, CounterSnapshot> aggregatedEntry : aggregated.entrySet()) {
StatsHolder.Key aggregatedKey = aggregatedEntry.getKey();
MultiDimensionCacheStats.DimensionNode aggregated = stats.aggregateByLevels(levels);
Map<List<String>, MultiDimensionCacheStats.DimensionNode> aggregatedLeafNodes = getAllLeafNodes(aggregated);

for (Map.Entry<List<String>, MultiDimensionCacheStats.DimensionNode> aggEntry : aggregatedLeafNodes.entrySet()) {
CacheStatsCounter expectedCounter = new CacheStatsCounter();
for (Set<CacheStatsDimension> expectedDims : expected.keySet()) {
for (List<CacheStatsDimension> expectedDims : expected.keySet()) {
List<String> orderedDimValues = StatsHolder.getOrderedDimensionValues(
new ArrayList<>(expectedDims),
dimensionNames
);
if (orderedDimValues.containsAll(aggregatedKey.dimensionValues)) {
if (orderedDimValues.containsAll(aggEntry.getKey())) {
expectedCounter.add(expected.get(expectedDims));
}
}
assertEquals(expectedCounter.snapshot(), aggregatedEntry.getValue());
assertEquals(expectedCounter.snapshot(), aggEntry.getValue().getSnapshot());
}
}
}
}

// Get a map from the list of dimension values to the corresponding leaf node.
private Map<List<String>, MultiDimensionCacheStats.DimensionNode> getAllLeafNodes(MultiDimensionCacheStats.DimensionNode root) {
Map<List<String>, MultiDimensionCacheStats.DimensionNode> result = new HashMap<>();
getAllLeafNodesHelper(result, root, new ArrayList<>());
return result;
}

private void getAllLeafNodesHelper(Map<List<String>, MultiDimensionCacheStats.DimensionNode> result, MultiDimensionCacheStats.DimensionNode current, List<String> pathToCurrent) {
if (current.children.isEmpty()) {
result.put(pathToCurrent, current);
} else {
for (Map.Entry<String, MultiDimensionCacheStats.DimensionNode> entry : current.children.entrySet()) {
List<String> newPath = new ArrayList<>(pathToCurrent);
newPath.add(entry.getKey());
getAllLeafNodesHelper(result, entry.getValue(), newPath);
}
}
}

static Map<String, List<String>> getUsedDimensionValues(StatsHolder statsHolder, int numValuesPerDim) {
Map<String, List<String>> usedDimensionValues = new HashMap<>();
for (int i = 0; i < statsHolder.getDimensionNames().size(); i++) {
Expand All @@ -170,20 +169,19 @@ static Map<String, List<String>> getUsedDimensionValues(StatsHolder statsHolder,
return usedDimensionValues;
}

static Map<Set<CacheStatsDimension>, CacheStatsCounter> populateStats(
static Map<List<CacheStatsDimension>, CacheStatsCounter> populateStats(
StatsHolder statsHolder,
Map<String, List<String>> usedDimensionValues,
int numDistinctValuePairs,
int numRepetitionsPerValue
) {
Map<Set<CacheStatsDimension>, CacheStatsCounter> expected = new HashMap<>();
Map<List<CacheStatsDimension>, CacheStatsCounter> expected = new HashMap<>();

Random rand = Randomness.get();
for (int i = 0; i < numDistinctValuePairs; i++) {
List<CacheStatsDimension> dimensions = getRandomDimList(statsHolder.getDimensionNames(), usedDimensionValues, true, rand);
Set<CacheStatsDimension> dimSet = new HashSet<>(dimensions);
if (expected.get(dimSet) == null) {
expected.put(dimSet, new CacheStatsCounter());
if (expected.get(dimensions) == null) {
expected.put(dimensions, new CacheStatsCounter());
}
ICacheKey<String> dummyKey = getDummyKey(dimensions);

Expand All @@ -192,38 +190,38 @@ static Map<Set<CacheStatsDimension>, CacheStatsCounter> populateStats(
int numHitIncrements = rand.nextInt(10);
for (int k = 0; k < numHitIncrements; k++) {
statsHolder.incrementHits(dummyKey);
expected.get(new HashSet<>(dimensions)).hits.inc();
expected.get(dimensions).hits.inc();
}

int numMissIncrements = rand.nextInt(10);
for (int k = 0; k < numMissIncrements; k++) {
statsHolder.incrementMisses(dummyKey);
expected.get(new HashSet<>(dimensions)).misses.inc();
expected.get(dimensions).misses.inc();
}

int numEvictionIncrements = rand.nextInt(10);
for (int k = 0; k < numEvictionIncrements; k++) {
statsHolder.incrementEvictions(dummyKey);
expected.get(new HashSet<>(dimensions)).evictions.inc();
expected.get(dimensions).evictions.inc();
}

int numMemorySizeIncrements = rand.nextInt(10);
for (int k = 0; k < numMemorySizeIncrements; k++) {
long memIncrementAmount = rand.nextInt(5000);
statsHolder.incrementSizeInBytes(dummyKey, memIncrementAmount);
expected.get(new HashSet<>(dimensions)).sizeInBytes.inc(memIncrementAmount);
expected.get(dimensions).sizeInBytes.inc(memIncrementAmount);
}

int numEntryIncrements = rand.nextInt(9) + 1;
for (int k = 0; k < numEntryIncrements; k++) {
statsHolder.incrementEntries(dummyKey);
expected.get(new HashSet<>(dimensions)).entries.inc();
expected.get(dimensions).entries.inc();
}

int numEntryDecrements = rand.nextInt(numEntryIncrements);
for (int k = 0; k < numEntryDecrements; k++) {
statsHolder.decrementEntries(dummyKey);
expected.get(new HashSet<>(dimensions)).entries.dec();
expected.get(dimensions).entries.dec();
}
}
}
Expand Down
12 changes: 6 additions & 6 deletions server/src/test/java/org/opensearch/common/cache/stats/StatsHolderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,23 +41,23 @@ public void testReset() throws Exception {
List<String> dimensionNames = List.of("dim1", "dim2");
StatsHolder statsHolder = new StatsHolder(dimensionNames);
Map<String, List<String>> usedDimensionValues = getUsedDimensionValues(statsHolder, 10);
Map<Set<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 100, 10);
Map<List<CacheStatsDimension>, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 100, 10);

statsHolder.reset();

for (Set<CacheStatsDimension> dimSet : expected.keySet()) {
CacheStatsCounter originalCounter = expected.get(dimSet);
for (List<CacheStatsDimension> dims : expected.keySet()) {
CacheStatsCounter originalCounter = expected.get(dims);
originalCounter.sizeInBytes = new CounterMetric();
originalCounter.entries = new CounterMetric();

StatsHolder.Key key = new StatsHolder.Key(StatsHolder.getOrderedDimensionValues(new ArrayList<>(dimSet), dimensionNames));
StatsHolder.Key key = new StatsHolder.Key(StatsHolder.getOrderedDimensionValues(dims, dimensionNames));
CacheStatsCounter actual = statsHolder.getStatsMap().get(key);
assertEquals(originalCounter, actual);
}

CacheStatsCounter expectedTotal = new CacheStatsCounter();
for (Set<CacheStatsDimension> dimSet : expected.keySet()) {
expectedTotal.add(expected.get(dimSet));
for (List<CacheStatsDimension> dims : expected.keySet()) {
expectedTotal.add(expected.get(dims));
}
expectedTotal.sizeInBytes = new CounterMetric();
expectedTotal.entries = new CounterMetric();
Expand Down

0 comments on commit fb3baaa

Please sign in to comment.