Inject the backend query type map for pluggable backend selection (#2…

…1121)

* Inject the backend query type map for pluggable backend selection

always also bind backends in the generic map binder, even for versioned backends in non-versioned backends, use the map binder directly so the engine can pick the correct backend based on the BackendQuery instance used

for query normalization, ignore non-elasticsearch based queries for now, we will need to address normalizers later to make them type safe (or just live with the instanceof check)

Simplified version of #21073

* refactor query engine to choose from versioned and unversioned backends

turns out mixing them doesn't work, because the elasticsearch-based ones have the same name, but all bindings are installed :(

* use elasticsearch provider for versioned backend for correct version logic

graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/ViewsESBackendModule.java

@@ -24,8 +24,6 @@
 import com.google.inject.multibindings.MapBinder;
 import org.graylog.plugins.views.ViewsModule;
 import org.graylog.plugins.views.search.SearchType;
-import org.graylog.plugins.views.search.engine.GeneratedQueryContext;
-import org.graylog.plugins.views.search.engine.QueryBackend;
 import org.graylog.plugins.views.search.export.ExportBackend;
 import org.graylog.plugins.views.search.searchtypes.MessageList;
 import org.graylog.plugins.views.search.searchtypes.events.EventList;
@@ -86,8 +84,7 @@ public ViewsESBackendModule(SearchVersion supportedSearchVersion) {
     protected void configure() {
         install(new FactoryModuleBuilder().build(ESGeneratedQueryContext.Factory.class));
 
-        bindForVersion(supportedSearchVersion, new TypeLiteral<QueryBackend<? extends GeneratedQueryContext>>() {})
-                .to(ElasticsearchBackend.class);
+        registerVersionedQueryBackend(supportedSearchVersion, ElasticsearchBackend.class);
 
         registerESSearchTypeHandler(MessageList.NAME, ESMessageList.class);
         registerESSearchTypeHandler(EventList.NAME, ESEventList.class);

graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ViewsOSBackendModule.java

@@ -25,8 +25,6 @@
 import com.google.inject.multibindings.OptionalBinder;
 import org.graylog.plugins.views.ViewsModule;
 import org.graylog.plugins.views.search.SearchType;
-import org.graylog.plugins.views.search.engine.GeneratedQueryContext;
-import org.graylog.plugins.views.search.engine.QueryBackend;
 import org.graylog.plugins.views.search.export.ExportBackend;
 import org.graylog.plugins.views.search.searchtypes.MessageList;
 import org.graylog.plugins.views.search.searchtypes.events.EventList;
@@ -89,8 +87,7 @@ public ViewsOSBackendModule(SearchVersion supportedSearchVersion) {
     protected void configure() {
         install(new FactoryModuleBuilder().build(OSGeneratedQueryContext.Factory.class));
 
-        bindForVersion(supportedSearchVersion, new TypeLiteral<QueryBackend<? extends GeneratedQueryContext>>() {})
-                .to(OpenSearchBackend.class);
+        registerVersionedQueryBackend(supportedSearchVersion, OpenSearchBackend.class);
 
         registerOSSearchTypeHandler(MessageList.NAME, OSMessageList.class);
         registerOSSearchTypeHandler(EventList.NAME, OSEventListDelegate.class);

graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java

@@ -181,6 +181,8 @@ protected void configure() {
         registerJacksonSubtype(StreamCategoryFilter.class);
         registerJacksonSubtype(QueryStringFilter.class);
 
+        // to support unversioned query backends
+        queryBackendBinder();
         // query backends for jackson
         registerJacksonSubtype(ElasticsearchQueryString.class);
 

graylog2-server/src/main/java/org/graylog/plugins/views/ViewsModule.java

@@ -18,7 +18,6 @@
 
 import com.google.inject.TypeLiteral;
 import com.google.inject.binder.LinkedBindingBuilder;
-import com.google.inject.binder.ScopedBindingBuilder;
 import com.google.inject.multibindings.MapBinder;
 import com.google.inject.multibindings.Multibinder;
 import com.google.inject.multibindings.OptionalBinder;
@@ -85,15 +84,15 @@ protected void registerPivotAggregationFunction(String name, String description,
         seriesSpecBinder().addBinding(name).toInstance(SeriesDescription.create(name, description));
     }
 
-    protected MapBinder<String, QueryBackend<? extends GeneratedQueryContext>> queryBackendBinder(SearchVersion version) {
+    protected MapBinder<String, QueryBackend<? extends GeneratedQueryContext>> queryBackendBinder() {
         return MapBinder.newMapBinder(binder(),
                 TypeLiteral.get(String.class),
-                new TypeLiteral<QueryBackend<? extends GeneratedQueryContext>>() {});
+                new TypeLiteral<>() {});
 
     }
 
-    protected ScopedBindingBuilder registerQueryBackend(SearchVersion version, String name, Class<? extends QueryBackend<? extends GeneratedQueryContext>> implementation) {
-        return queryBackendBinder(version).addBinding(name).to(implementation);
+    protected void registerVersionedQueryBackend(SearchVersion version, Class<? extends QueryBackend<? extends GeneratedQueryContext>> implementation) {
+        bindForVersion(version, new TypeLiteral<QueryBackend<? extends GeneratedQueryContext>>() {}).to(implementation);
     }
 
     protected void registerESQueryDecorator(Class<? extends QueryStringDecorator> esQueryDecorator) {
@@ -135,4 +134,6 @@ protected void registerSearchValidator(Class<? extends SearchValidator> validato
     protected Multibinder<SearchValidator> searchValidatorBinder() {
         return Multibinder.newSetBinder(binder(), SearchValidator.class);
     }
+
+
 }

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryEngine.java

@@ -27,9 +27,11 @@
 import org.graylog.plugins.views.search.QueryResult;
 import org.graylog.plugins.views.search.Search;
 import org.graylog.plugins.views.search.SearchJob;
+import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString;
 import org.graylog.plugins.views.search.errors.QueryError;
 import org.graylog.plugins.views.search.errors.SearchError;
 import org.graylog.plugins.views.search.errors.SearchException;
+import org.graylog2.storage.providers.ElasticsearchBackendProvider;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -52,13 +54,16 @@ public class QueryEngine {
 
     // TODO proper thread pool with tunable settings
     private final Executor queryPool = Executors.newFixedThreadPool(4, new ThreadFactoryBuilder().setNameFormat("query-engine-%d").build());
-    private final QueryBackend<? extends GeneratedQueryContext> backend;
+    private final ElasticsearchBackendProvider elasticsearchBackendProvider;
+    private final Map<String, QueryBackend<? extends GeneratedQueryContext>> unversionedBackends;
 
     @Inject
-    public QueryEngine(QueryBackend<? extends GeneratedQueryContext> backend,
+    public QueryEngine(ElasticsearchBackendProvider elasticsearchBackendProvider,
+                       Map<String, QueryBackend<? extends GeneratedQueryContext>> unversionedBackends,
                        Set<QueryMetadataDecorator> queryMetadataDecorators,
                        QueryParser queryParser) {
-        this.backend = backend;
+        this.elasticsearchBackendProvider = elasticsearchBackendProvider;
+        this.unversionedBackends = unversionedBackends;
         this.queryMetadataDecorators = queryMetadataDecorators;
         this.queryParser = queryParser;
     }
@@ -75,6 +80,7 @@ public QueryMetadata parse(Search search, Query query) {
     public ExplainResults explain(SearchJob searchJob, Set<SearchError> validationErrors, DateTimeZone timezone) {
         final Map<String, ExplainResults.QueryExplainResult> queries = searchJob.getSearch().queries().stream()
                 .collect(Collectors.toMap(Query::id, q -> {
+                    var backend = getBackendForQuery(q);
                     final GeneratedQueryContext generatedQueryContext = backend.generate(q, Set.of(), timezone);
 
                     return backend.explain(searchJob, q, generatedQueryContext);
@@ -116,6 +122,7 @@ public SearchJob execute(SearchJob searchJob, Set<SearchError> validationErrors,
     }
 
     private QueryResult prepareAndRun(SearchJob searchJob, Query query, Set<SearchError> validationErrors, DateTimeZone timezone) {
+        final var backend = getBackendForQuery(query);
         LOG.debug("[{}] Using {} to generate query", query.id(), backend);
         // with all the results done, we can execute the current query and eventually complete our own result
         // if any of this throws an exception, the handle in #execute will convert it to an error and return a "failed" result instead
@@ -137,4 +144,15 @@ private boolean isQueryWithError(Collection<SearchError> validationErrors, Query
                 .map(QueryError::queryId)
                 .anyMatch(id -> Objects.equals(id, query.id()));
     }
+
+    private QueryBackend<? extends GeneratedQueryContext> getBackendForQuery(Query query) {
+        var backendQuery = query.query();
+        if (backendQuery.type().equals(ElasticsearchQueryString.NAME)) {
+            return elasticsearchBackendProvider.get();
+        }
+        if (unversionedBackends.containsKey(backendQuery.type())) {
+            return unversionedBackends.get(backendQuery.type());
+        }
+        throw new IllegalArgumentException("Unknown backend type: " + backendQuery.type());
+    }
 }

graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/DecorateQueryStringsNormalizer.java

@@ -38,6 +38,10 @@ public DecorateQueryStringsNormalizer(QueryStringDecorators queryStringDecorator
 
     @Override
     public Query normalizeQuery(final Query query, final ParameterProvider parameterProvider) {
+        // this only makes sense for ElasticsearchQueryString instances, don't touch any other type
+        if (!(query.query() instanceof ElasticsearchQueryString)) {
+            return query;
+        }
         return query.toBuilder()
                 .query(ElasticsearchQueryString.of(this.queryStringDecorators.decorate(query.query().queryString(), parameterProvider, query)))
                 .filter(normalizeFilter(query.filter(), query, parameterProvider))