Merge pull request #2384 from uncharted-distil/prefeaturized-explanat…

…ions

generate explanations for unpooled remote sensing datasets

api/compute/explain.go

@@ -28,6 +28,7 @@ import (
 	"github.com/uncharted-distil/distil-compute/primitive/compute/result"
 	log "github.com/unchartedsoftware/plog"
 
+	"github.com/uncharted-distil/distil/api/env"
 	api "github.com/uncharted-distil/distil/api/model"
 	"github.com/uncharted-distil/distil/api/serialization"
 	"github.com/uncharted-distil/distil/api/util"
@@ -176,10 +177,20 @@ func parseConfidencesWrapper(params []int) func([]string) (*api.SolutionExplainV
 }
 
 func (s *SolutionRequest) createExplainPipeline(desc *pipeline.DescribeSolutionResponse) (*pipeline.PipelineDescription, map[string]*pipelineOutput) {
-	// pre featurized datasets are not explainable
+	// *POOLED* pre featurized datasets are not explainable.  Pooling is currently controlled by
+	// an env var, although it could be added to the metadata for a dataset.
+	pooled := true
+	config, err := env.LoadConfig()
+	if err == nil {
+		log.Warnf("failed to load environment variables")
+	} else {
+		pooled = config.PoolFeatures
+	}
+
+	//
 	// TODO: we may want to look into folding this filtering functionality into
 	// the function that builds the explainable pipeline (explainablePipeline).
-	if s.DatasetMetadata != nil && s.DatasetMetadata.LearningDataset != "" {
+	if s.DatasetMetadata != nil && s.DatasetMetadata.LearningDataset != "" && !pooled {
 		return nil, nil
 	}
 

api/compute/search.go

@@ -81,12 +81,9 @@ func (s *SolutionRequest) dispatchSolutionExplainPipeline(client *compute.Client
 	for _, eo := range explainOutputs {
 		exposedOutputs = append(exposedOutputs, eo.output)
 	}
-
-	exposeType := []string{}
-	if s.useParquet {
-		exposeType = append(exposeType, compute.ParquetURIValueType)
-	}
-	produceSolutionRequest := createProduceSolutionRequest(explainDatasetURI, searchResult.fittedSolutionID, exposedOutputs, exposeType)
+	// create the produce request that will generate the explanations - we force the use of CSV for output since the
+	// go parquet library doesn't handle nested lists well
+	produceSolutionRequest := createProduceSolutionRequest(explainDatasetURI, searchResult.fittedSolutionID, exposedOutputs, []string{compute.CSVURIValueType})
 
 	// generate predictions
 	_, predictionResponses, err := client.GeneratePredictions(context.Background(), produceSolutionRequest)