diff --git a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java index 823d869..7605a33 100644 --- a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java +++ b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java @@ -23,6 +23,8 @@ import org.opensearch.ltr.stats.LTRStats; import org.opensearch.ltr.stats.StatName; import org.opensearch.ltr.stats.suppliers.CacheStatsOnNodeSupplier; +import org.opensearch.ltr.stats.suppliers.PluginHealthStatusSupplier; +import org.opensearch.ltr.stats.suppliers.StoreStatsSupplier; import org.opensearch.ltr.stats.suppliers.CounterSupplier; import com.o19s.es.explore.ExplorerQueryBuilder; import com.o19s.es.ltr.action.AddFeaturesToSetAction; @@ -125,7 +127,7 @@ public class LtrQueryParserPlugin extends Plugin implements SearchPlugin, Script public static final String LTR_LEGACY_BASE_URI = "/_opendistro/_ltr"; private final LtrRankerParserFactory parserFactory; private final Caches caches; - private LTRStats ltrStats; + private final LTRStats ltrStats; public LtrQueryParserPlugin(Settings settings) { caches = new Caches(settings); @@ -278,9 +280,23 @@ public Collection createComponents(Client client, final JvmService jvmService = new JvmService(environment.settings()); final LTRCircuitBreakerService ltrCircuitBreakerService = new LTRCircuitBreakerService(jvmService).init(); + addStats(client, clusterService, ltrCircuitBreakerService); return asList(caches, parserFactory, ltrCircuitBreakerService, ltrStats); } + private void addStats( + final Client client, + final ClusterService clusterService, + final LTRCircuitBreakerService ltrCircuitBreakerService + ) { + final StoreStatsSupplier storeStatsSupplier = StoreStatsSupplier.create(client, clusterService); + ltrStats.addStats(StatName.LTR_STORES_STATS.getName(), new LTRStat<>(true, storeStatsSupplier)); + + final PluginHealthStatusSupplier pluginHealthStatusSupplier = PluginHealthStatusSupplier.create( + client, clusterService, ltrCircuitBreakerService); + ltrStats.addStats(StatName.LTR_PLUGIN_STATUS.getName(), new LTRStat<>(true, pluginHealthStatusSupplier)); + } + private LTRStats getInitialStats() { Map> stats = new HashMap<>(); stats.put(StatName.LTR_CACHE_STATS.getName(), @@ -294,7 +310,7 @@ private LTRStats getInitialStats() { protected FeatureStoreLoader getFeatureStoreLoader() { return (storeName, clientSupplier) -> - new CachedFeatureStore(new IndexFeatureStore(storeName, clientSupplier, parserFactory), caches); + new CachedFeatureStore(new IndexFeatureStore(storeName, clientSupplier, parserFactory), caches); } // A simplified version of some token filters needed by the feature stores. diff --git a/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java b/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java index ebf588a..d54de73 100644 --- a/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java +++ b/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java @@ -28,7 +28,10 @@ /** * Supplier for an overall plugin health status. + * @deprecated This class is outdated since 3.0.0-3.0.0 and will be removed in the future. + * Please use the new stats framework in the {@link org.opensearch.ltr.stats} package. */ +@Deprecated(since = "3.0.0-3.0.0", forRemoval = true) public class PluginHealthStatusSupplier implements Supplier { private static final String STATUS_GREEN = "green"; private static final String STATUS_YELLOW = "yellow"; diff --git a/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java b/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java index efac97d..8e86e57 100644 --- a/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java +++ b/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java @@ -48,7 +48,10 @@ * A supplier which provides information on all feature stores. It provides basic * information such as the index health and count of feature sets, features and * models in the store. + * @deprecated This class is outdated since 3.0.0-3.0.0 and will be removed in the future. + * Please use the new stats framework in the {@link org.opensearch.ltr.stats} package. */ +@Deprecated(since = "3.0.0-3.0.0", forRemoval = true) public class StoreStatsSupplier implements Supplier>> { private static final Logger LOG = LogManager.getLogger(StoreStatsSupplier.class); private static final String AGG_FIELD = "type"; diff --git a/src/main/java/org/opensearch/ltr/stats/suppliers/PluginHealthStatusSupplier.java b/src/main/java/org/opensearch/ltr/stats/suppliers/PluginHealthStatusSupplier.java new file mode 100644 index 0000000..733a00d --- /dev/null +++ b/src/main/java/org/opensearch/ltr/stats/suppliers/PluginHealthStatusSupplier.java @@ -0,0 +1,61 @@ +package org.opensearch.ltr.stats.suppliers; + + +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.ltr.breaker.LTRCircuitBreakerService; +import org.opensearch.ltr.stats.suppliers.utils.StoreUtils; + +import java.util.List; +import java.util.function.Supplier; + +/** + * Supplier for an overall plugin health status, which is based on the + * aggregate store health and the circuit breaker state. + */ +public class PluginHealthStatusSupplier implements Supplier { + private static final String STATUS_GREEN = "green"; + private static final String STATUS_YELLOW = "yellow"; + private static final String STATUS_RED = "red"; + + private final StoreUtils storeUtils; + private final LTRCircuitBreakerService ltrCircuitBreakerService; + + protected PluginHealthStatusSupplier(StoreUtils storeUtils, + LTRCircuitBreakerService ltrCircuitBreakerService) { + this.storeUtils = storeUtils; + this.ltrCircuitBreakerService = ltrCircuitBreakerService; + } + + @Override + public String get() { + if (ltrCircuitBreakerService.isOpen()) { + return STATUS_RED; + } + return getAggregateStoresStatus(); + } + + private String getAggregateStoresStatus() { + List storeNames = storeUtils.getAllLtrStoreNames(); + return storeNames.stream() + .map(storeUtils::getLtrStoreHealthStatus) + .reduce(STATUS_GREEN, this::combineStatuses); + } + + private String combineStatuses(String s1, String s2) { + if (s2 == null || STATUS_RED.equals(s1) || STATUS_RED.equals(s2)) { + return STATUS_RED; + } else if (STATUS_YELLOW.equals(s1) || STATUS_YELLOW.equals(s2)) { + return STATUS_YELLOW; + } else { + return STATUS_GREEN; + } + } + + public static PluginHealthStatusSupplier create( + final Client client, + final ClusterService clusterService, + LTRCircuitBreakerService ltrCircuitBreakerService) { + return new PluginHealthStatusSupplier(new StoreUtils(client, clusterService), ltrCircuitBreakerService); + } +} diff --git a/src/main/java/org/opensearch/ltr/stats/suppliers/StoreStatsSupplier.java b/src/main/java/org/opensearch/ltr/stats/suppliers/StoreStatsSupplier.java new file mode 100644 index 0000000..be9d0cf --- /dev/null +++ b/src/main/java/org/opensearch/ltr/stats/suppliers/StoreStatsSupplier.java @@ -0,0 +1,54 @@ +package org.opensearch.ltr.stats.suppliers; + +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.ltr.stats.suppliers.utils.StoreUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +/** + * A supplier to provide stats on the LTR stores. It retrieves basic information + * on the store, such as the health of the underlying index and number of documents + * in the store grouped by their type. + */ +public class StoreStatsSupplier implements Supplier>> { + static final String LTR_STORE_STATUS = "status"; + static final String LTR_STORE_FEATURE_COUNT = "feature_count"; + static final String LTR_STORE_FEATURE_SET_COUNT = "featureset_count"; + static final String LTR_STORE_MODEL_COUNT = "model_count"; + + private final StoreUtils storeUtils; + + protected StoreStatsSupplier(final StoreUtils storeUtils) { + this.storeUtils = storeUtils; + } + + @Override + public Map> get() { + Map> storeStats = new ConcurrentHashMap<>(); + List storeNames = storeUtils.getAllLtrStoreNames(); + storeNames.forEach(s -> storeStats.put(s, getStoreStat(s))); + return storeStats; + } + + private Map getStoreStat(String storeName) { + if (!storeUtils.checkLtrStoreExists(storeName)) { + throw new IllegalArgumentException("LTR Store [" + storeName + "] doesn't exist."); + } + Map storeStat = new HashMap<>(); + storeStat.put(LTR_STORE_STATUS, storeUtils.getLtrStoreHealthStatus(storeName)); + Map featureSets = storeUtils.extractFeatureSetStats(storeName); + storeStat.put(LTR_STORE_FEATURE_COUNT, featureSets.values().stream().reduce(Integer::sum).orElse(0)); + storeStat.put(LTR_STORE_FEATURE_SET_COUNT, featureSets.size()); + storeStat.put(LTR_STORE_MODEL_COUNT, storeUtils.getModelCount(storeName)); + return storeStat; + } + + public static StoreStatsSupplier create(final Client client, final ClusterService clusterService) { + return new StoreStatsSupplier(new StoreUtils(client, clusterService)); + } +} diff --git a/src/main/java/org/opensearch/ltr/stats/suppliers/utils/StoreUtils.java b/src/main/java/org/opensearch/ltr/stats/suppliers/utils/StoreUtils.java new file mode 100644 index 0000000..fe91e33 --- /dev/null +++ b/src/main/java/org/opensearch/ltr/stats/suppliers/utils/StoreUtils.java @@ -0,0 +1,115 @@ +package org.opensearch.ltr.stats.suppliers.utils; + +import com.o19s.es.ltr.feature.store.StoredFeatureSet; +import com.o19s.es.ltr.feature.store.StoredLtrModel; +import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.search.SearchType; +import org.opensearch.client.Client; +import org.opensearch.cluster.health.ClusterIndexHealth; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A utility class to provide details on the LTR stores. It queries the underlying + * indices to get the details. + */ +public class StoreUtils { + + private static final String FEATURE_SET_KEY = "featureset"; + private static final String FEATURE_SET_NAME_KEY = "name"; + private static final String FEATURES_KEY = "features"; + private Client client; + private ClusterService clusterService; + private IndexNameExpressionResolver indexNameExpressionResolver; + + public StoreUtils(Client client, ClusterService clusterService) { + this.client = client; + this.clusterService = clusterService; + this.indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(clusterService.getSettings())); + } + + public boolean checkLtrStoreExists(String storeName) { + return clusterService.state().getRoutingTable().hasIndex(storeName); + } + + public List getAllLtrStoreNames() { + String[] names = indexNameExpressionResolver.concreteIndexNames(clusterService.state(), + new ClusterStateRequest().indices( + IndexFeatureStore.DEFAULT_STORE, IndexFeatureStore.STORE_PREFIX + "*")); + return Arrays.asList(names); + } + + public String getLtrStoreHealthStatus(String storeName) { + if (!checkLtrStoreExists(storeName)) { + throw new IndexNotFoundException(storeName); + } + ClusterIndexHealth indexHealth = new ClusterIndexHealth( + clusterService.state().metadata().index(storeName), + clusterService.state().getRoutingTable().index(storeName) + ); + + return indexHealth.getStatus().name().toLowerCase(); + } + + /** + * Returns a map of feaureset and the number of features in the featureset. + * + * @param storeName the name of the index for the LTR store. + * @return A map of (featureset, features count) + */ + @SuppressWarnings("unchecked") + public Map extractFeatureSetStats(String storeName) { + final Map featureSetStats = new HashMap<>(); + final SearchHits featureSetHits = searchStore(storeName, StoredFeatureSet.TYPE); + + for (final SearchHit featureSetHit : featureSetHits) { + extractFeatureSetFromFeatureSetHit(featureSetHit).ifPresent(featureSet -> { + final List features = (List) featureSet.get(FEATURES_KEY); + featureSetStats.put((String) featureSet.get(FEATURE_SET_NAME_KEY), features.size()); + }); + } + return featureSetStats; + } + + @SuppressWarnings("unchecked") + private Optional> extractFeatureSetFromFeatureSetHit(SearchHit featureSetHit) { + final Map featureSetMap = featureSetHit.getSourceAsMap(); + if (featureSetMap != null && featureSetMap.containsKey(FEATURE_SET_KEY)) { + final Map featureSet = (Map) featureSetMap.get(FEATURE_SET_KEY); + + if (featureSet != null && featureSet.containsKey(FEATURES_KEY) && featureSet.containsKey(FEATURE_SET_NAME_KEY)) { + return Optional.of(featureSet); + } + } + + return Optional.empty(); + } + + + +// private + + public long getModelCount(String storeName) { + return searchStore(storeName, StoredLtrModel.TYPE).getTotalHits().value; + } + + private SearchHits searchStore(String storeName, String docType) { + return client.prepareSearch(storeName) + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) + .setQuery(QueryBuilders.termQuery("type", docType)) + .get() + .getHits(); + } +} diff --git a/src/test/java/org/opensearch/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java b/src/test/java/org/opensearch/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java new file mode 100644 index 0000000..0ab502b --- /dev/null +++ b/src/test/java/org/opensearch/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java @@ -0,0 +1,65 @@ +package org.opensearch.ltr.stats.suppliers; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.opensearch.ltr.breaker.LTRCircuitBreakerService; +import org.opensearch.ltr.stats.suppliers.utils.StoreUtils; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +public class PluginHealthStatusSupplierTests { + private PluginHealthStatusSupplier pluginHealthStatusSupplier; + + @Mock + private LTRCircuitBreakerService ltrCircuitBreakerService; + + @Mock + StoreUtils storeUtils; + + @Before + public void setup() { + MockitoAnnotations.openMocks(this); + pluginHealthStatusSupplier = + new PluginHealthStatusSupplier(storeUtils, ltrCircuitBreakerService); + } + + @Test + public void testStatusGreen() { + when(ltrCircuitBreakerService.isOpen()).thenReturn(false); + when(storeUtils.getAllLtrStoreNames()).thenReturn(Arrays.asList("store1", "store2")); + when(storeUtils.getLtrStoreHealthStatus(Mockito.anyString())).thenReturn("green"); + + assertEquals("green", pluginHealthStatusSupplier.get()); + } + + @Test + public void testStatusYellowStoreStatusYellow() { + when(ltrCircuitBreakerService.isOpen()).thenReturn(false); + when(storeUtils.getAllLtrStoreNames()).thenReturn(Arrays.asList("store1", "store2")); + when(storeUtils.getLtrStoreHealthStatus("store1")).thenReturn("green"); + when(storeUtils.getLtrStoreHealthStatus("store2")).thenReturn("yellow"); + assertEquals("yellow", pluginHealthStatusSupplier.get()); + } + + @Test + public void testStatusRedStoreStatusRed() { + when(ltrCircuitBreakerService.isOpen()).thenReturn(false); + when(storeUtils.getAllLtrStoreNames()).thenReturn(Arrays.asList("store1", "store2")); + when(storeUtils.getLtrStoreHealthStatus("store1")).thenReturn("red"); + when(storeUtils.getLtrStoreHealthStatus("store2")).thenReturn("green"); + + assertEquals("red", pluginHealthStatusSupplier.get()); + } + + @Test + public void testStatusRedCircuitBreakerOpen() { + when(ltrCircuitBreakerService.isOpen()).thenReturn(true); + assertEquals("red", pluginHealthStatusSupplier.get()); + } +} diff --git a/src/test/java/org/opensearch/ltr/stats/suppliers/StoreStatsSupplierTests.java b/src/test/java/org/opensearch/ltr/stats/suppliers/StoreStatsSupplierTests.java new file mode 100644 index 0000000..c592ea6 --- /dev/null +++ b/src/test/java/org/opensearch/ltr/stats/suppliers/StoreStatsSupplierTests.java @@ -0,0 +1,56 @@ +package org.opensearch.ltr.stats.suppliers; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.ltr.stats.suppliers.utils.StoreUtils; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.when; + +public class StoreStatsSupplierTests extends OpenSearchTestCase { + private static final String STORE_NAME = ".ltrstore"; + + @Mock + private StoreUtils storeUtils; + + private StoreStatsSupplier storeStatsSupplier; + + @Before + public void setup() { + MockitoAnnotations.openMocks(this); + storeStatsSupplier = new StoreStatsSupplier(storeUtils); + } + + @Test + public void getStoreStats_NoLtrStore() { + when(storeUtils.getAllLtrStoreNames()).thenReturn(Collections.emptyList()); + Map> stats = storeStatsSupplier.get(); + assertTrue(stats.isEmpty()); + } + + @Test + public void getStoreStats_Success() { + when(storeUtils.getAllLtrStoreNames()).thenReturn(Collections.singletonList(STORE_NAME)); + when(storeUtils.checkLtrStoreExists(STORE_NAME)).thenReturn(true); + when(storeUtils.getLtrStoreHealthStatus(STORE_NAME)).thenReturn("green"); + Map featureSets = new HashMap<>(); + featureSets.put("featureset_1", 10); + when(storeUtils.extractFeatureSetStats(STORE_NAME)).thenReturn(featureSets); + when(storeUtils.getModelCount(STORE_NAME)).thenReturn(5L); + + Map> stats = storeStatsSupplier.get(); + Map ltrStoreStats = stats.get(STORE_NAME); + + assertNotNull(ltrStoreStats); + assertEquals("green", ltrStoreStats.get(StoreStatsSupplier.LTR_STORE_STATUS)); + assertEquals(10, ltrStoreStats.get(StoreStatsSupplier.LTR_STORE_FEATURE_COUNT)); + assertEquals(1, ltrStoreStats.get(StoreStatsSupplier.LTR_STORE_FEATURE_SET_COUNT)); + assertEquals(5L, ltrStoreStats.get(StoreStatsSupplier.LTR_STORE_MODEL_COUNT)); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/ltr/stats/suppliers/utils/StoreUtilsTests.java b/src/test/java/org/opensearch/ltr/stats/suppliers/utils/StoreUtilsTests.java new file mode 100644 index 0000000..358bd36 --- /dev/null +++ b/src/test/java/org/opensearch/ltr/stats/suppliers/utils/StoreUtilsTests.java @@ -0,0 +1,126 @@ +package org.opensearch.ltr.stats.suppliers.utils; + +import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.ltr.stats.suppliers.utils.StoreUtils; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Map; + +public class StoreUtilsTests extends OpenSearchIntegTestCase { + private StoreUtils storeUtils; + + @Before + public void setup() { + storeUtils = new StoreUtils(client(), clusterService()); + } + + @Test + public void checkLtrStoreExists() { + createIndex(IndexFeatureStore.DEFAULT_STORE); + flush(); + assertTrue(storeUtils.checkLtrStoreExists(IndexFeatureStore.DEFAULT_STORE)); + } + + @Test + public void getAllLtrStoreNames_NoLtrStores() { + assertTrue(storeUtils.getAllLtrStoreNames().isEmpty()); + } + + @Test + public void getAllLtrStoreNames() { + createIndex(IndexFeatureStore.DEFAULT_STORE); + flush(); + assertEquals(1, storeUtils.getAllLtrStoreNames().size()); + assertEquals(IndexFeatureStore.DEFAULT_STORE, storeUtils.getAllLtrStoreNames().get(0)); + } + + @Test(expected = IndexNotFoundException.class) + public void getLtrStoreHealthStatus_IndexNotExist() { + storeUtils.getLtrStoreHealthStatus("non-existent"); + } + + @Test + public void getLtrStoreHealthStatus() { + createIndex(IndexFeatureStore.DEFAULT_STORE); + flush(); + String status = storeUtils.getLtrStoreHealthStatus(IndexFeatureStore.DEFAULT_STORE); + assertTrue(status.equals("green") || status.equals("yellow")); + } + + @Test(expected = IndexNotFoundException.class) + public void extractFeatureSetStats_IndexNotExist() { + storeUtils.extractFeatureSetStats("non-existent"); + } + + @Test + public void extractFeatureSetStats() { + createIndex(IndexFeatureStore.DEFAULT_STORE); + flush(); + index(IndexFeatureStore.DEFAULT_STORE, "_doc", "featureset_1", testFeatureSet()); + flushAndRefresh(IndexFeatureStore.DEFAULT_STORE); + Map featureset = storeUtils.extractFeatureSetStats(IndexFeatureStore.DEFAULT_STORE); + + assertEquals(1, featureset.size()); + assertEquals(2, (int) featureset.values().stream().reduce(Integer::sum).get()); + } + + @Test(expected = IndexNotFoundException.class) + public void getModelCount_IndexNotExist() { + storeUtils.getModelCount("non-existent"); + } + + @Test + public void getModelCount() { + createIndex(IndexFeatureStore.DEFAULT_STORE); + flush(); + index(IndexFeatureStore.DEFAULT_STORE, "_doc", "model_1", testModel()); + flushAndRefresh(IndexFeatureStore.DEFAULT_STORE); + assertEquals(1, storeUtils.getModelCount(IndexFeatureStore.DEFAULT_STORE)); + } + + + private String testFeatureSet() { + return "{\n" + + "\"name\": \"movie_features\",\n" + + "\"type\": \"featureset\",\n" + + "\"featureset\": {\n" + + " \"name\": \"movie_features\",\n" + + " \"features\": [\n" + + " {\n" + + " \"name\": \"1\",\n" + + " \"params\": [\n" + + " \"keywords\"\n" + + " ],\n" + + " \"template_language\": \"mustache\",\n" + + " \"template\": {\n" + + " \"match\": {\n" + + " \"title\": \"{{keywords}}\"\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"2\",\n" + + " \"params\": [\n" + + " \"keywords\"\n" + + " ],\n" + + " \"template_language\": \"mustache\",\n" + + " \"template\": {\n" + + " \"match\": {\n" + + " \"overview\": \"{{keywords}}\"\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + "}\n}"; + } + + private String testModel() { + return "{\n" + + "\"name\": \"movie_model\",\n" + + "\"type\": \"model\"" + + "\n}"; + } +}