add alerce database initialization for integration tests

test: add more queries to the csv tests for different tables test: add tests against alerce database add tests for alercehtml, alercetext and alercevotable
alercebroker · Jul 21, 2024 · c17250d · c17250d
1 parent 8bb128f
commit c17250d
Show file tree

Hide file tree

Showing 23 changed files with 2,499 additions and 16 deletions.
diff --git a/tapservicego/dagger/main.go b/tapservicego/dagger/main.go
@@ -39,14 +39,14 @@ func (m *Tapservicego) BuildEnv(ctx context.Context, source *Directory) *Contain
 func (m *Tapservicego) Test(ctx context.Context, source *Directory) (string, error) {
 	db := dag.
 		Container().
-		From("index.docker.io/postgres").
-		WithEnvVariable("POSTGRES_USER", "postgres").
-		WithEnvVariable("POSTGRES_PASSWORD", "postgres").
+		From("docker.io/postgres:16-alpine").
+		WithEnvVariable("POSTGRES_USER", "testuser").
+		WithEnvVariable("POSTGRES_PASSWORD", "testpassword").
 		AsService()
 	return m.BuildEnv(ctx, source).
 		WithEnvVariable("ENV", "CI").
 		WithServiceBinding("db", db).
-		WithExec([]string{"go", "test", "./..."}).
+		WithExec([]string{"go", "test", "-failfast", "./..."}).
 		Stdout(ctx)
 }
 

diff --git a/tapservicego/internal/parsers/csvparser.go b/tapservicego/internal/parsers/csvparser.go
@@ -74,6 +74,9 @@ func ParseTSV(data []map[string]interface{}) (string, error) {
 }
 
 func parseCsvData(data []map[string]interface{}, w *csv.Writer) error {
+	if len(data) == 0 {
+		return nil
+	}
 	headers := getHeaders(data[0])
 	err := w.Write(headers)
 	if err != nil {

diff --git a/tapservicego/internal/parsers/fitsparser.go b/tapservicego/internal/parsers/fitsparser.go
@@ -99,6 +99,8 @@ func createColumns(data []map[string]interface{}) ([]fitsio.Column, error) {
 			format = "D" // 64-bit floating point
 		case string:
 			format = fmt.Sprintf("%dA", maxStringLength+1) // Character string with length
+		case bool:
+			format = "L" // logical
 		default:
 			return nil, fmt.Errorf("unsupported data type for column %s", key)
 		}

diff --git a/tapservicego/internal/parsers/fitsparser_test.go b/tapservicego/internal/parsers/fitsparser_test.go
@@ -11,8 +11,8 @@ import (
 
 func TestParseFits(t *testing.T) {
 	data := []map[string]interface{}{
-		{"name": "Alice", "age": int64(30)},
-		{"name": "Bob", "age": int64(25)},
+		{"name": "Alice", "age": int64(30), "boolvalue": true},
+		{"name": "Bob", "age": int64(25), "boolvalue": false},
 	}
 	result, err := ParseFits(data)
 	assert.Nil(t, err)
@@ -25,7 +25,7 @@ func TestParseFits(t *testing.T) {
 	table := hdu.(*fitsio.Table)
 	assert.True(t, table.Type() == fitsio.BINARY_TBL)
 	assert.Equal(t, "results", table.Name())
-	assert.Equal(t, 2, table.NumCols())
+	assert.Equal(t, 3, table.NumCols())
 	assert.Equal(t, int64(2), table.NumRows())
 	rows, err := table.Read(0, table.NumRows())
 	assert.Nil(t, err)
@@ -38,6 +38,7 @@ func TestParseFits(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Equal(t, data[count]["name"], row["name"])
 		assert.Equal(t, data[count]["age"], row["age"])
+		assert.Equal(t, data[count]["boolvalue"], row["boolvalue"])
 		parsedData = append(parsedData, row)
 		count = count + 1
 	}
@@ -75,3 +76,30 @@ func TestCreateFits(t *testing.T) {
 	assert.Equal(t, "K", result.Col(0).Format)
 	assert.Equal(t, "6A", result.Col(1).Format)
 }
+
+func TestCreateColumnsWithBoolean(t *testing.T) {
+	data := []map[string]interface{}{
+		{"boolvalue": true},
+		{"boolvalue": false},
+	}
+	col, err := createColumns(data)
+	assert.Nil(t, err)
+	assert.NotNil(t, col)
+	assert.Equal(t, 1, len(col))
+	assert.Equal(t, "boolvalue", col[0].Name)
+	assert.Equal(t, "L", col[0].Format)
+}
+
+func TestCreateFitsWithBoolean(t *testing.T) {
+	data := []map[string]interface{}{
+		{"boolvalue": true},
+		{"boolvalue": false},
+	}
+	result, err := CreateFits(data)
+	assert.Nil(t, err)
+	assert.NotNil(t, result)
+	assert.Equal(t, "results", result.Name())
+	assert.Equal(t, 1, result.NumCols())
+	assert.Equal(t, "boolvalue", result.Col(0).Name)
+	assert.Equal(t, "L", result.Col(0).Format)
+}
diff --git a/tapservicego/internal/tapsync/alercecsv_test.go b/tapservicego/internal/tapsync/alercecsv_test.go
@@ -0,0 +1,206 @@
+package tapsync
+
+import (
+	"ataps/internal/testhelpers"
+	"ataps/pkg/alercedb"
+	"encoding/csv"
+	"fmt"
+	"net/http"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCsv(t *testing.T) {
+	test_get_csv_data_from_object := func(t *testing.T, table string, ensureObjectExistIn *string, overrideOid *string) [][]string {
+		testhelpers.ClearALeRCEDB()
+		db := populateAlerceDB()
+		defer db.Close()
+		var oid string
+		if overrideOid == nil {
+			if ensureObjectExistIn == nil {
+				row := db.QueryRow("SELECT oid FROM object LIMIT 1")
+				if row.Err() != nil {
+					t.Fatal(row.Err())
+				}
+				row.Scan(&oid)
+			} else {
+				row := db.QueryRow(fmt.Sprintf("SELECT oid FROM %s LIMIT 1", *ensureObjectExistIn))
+				if row.Err() != nil {
+					t.Fatal(row.Err())
+				}
+				row.Scan(&oid)
+			}
+		} else {
+			oid = *overrideOid
+		}
+		service := NewTapSyncService()
+		response := sendTestQuery(fmt.Sprintf("LANG=PSQL&&FORMAT=csv&&QUERY=SELECT * FROM %s WHERE oid='%s'", table, oid), service)
+		require.Equal(t, http.StatusOK, response.Code)
+		reader := csv.NewReader(response.Body)
+		records, err := reader.ReadAll()
+		if err != nil {
+			t.Fatal(err)
+		}
+		return records
+	}
+
+	t.Run("TestCsv_Detections", func(t *testing.T) {
+		testhelpers.ClearALeRCEDB()
+		db := populateAlerceDB()
+		defer db.Close()
+		service := NewTapSyncService()
+		w := sendTestQuery("LANG=PSQL&&FORMAT=csv&&QUERY=SELECT * FROM detection LIMIT 1", service)
+		require.Equal(t, http.StatusOK, w.Code)
+		require.Equal(t, "text/csv", w.Header().Get("Content-Type"))
+		records, err := csv.NewReader(w.Body).ReadAll()
+		if err != nil {
+			t.Fatal(err)
+		}
+		columnNames := getColumnNames(alercedb.Detection{})
+		sort.Strings(columnNames)
+		require.Equal(t, columnNames, records[0])
+
+	})
+
+	t.Run("TestCsv_DetectionsFromAnObject", func(t *testing.T) {
+		ensureObjectExistIn := "detection"
+		records := test_get_csv_data_from_object(t, "detection", &ensureObjectExistIn, nil)
+		require.Greater(t, len(records), 1)
+	})
+
+	t.Run("TestCsv_DetectionsFromUnknownObject", func(t *testing.T) {
+		var oid = "unknown"
+		records := test_get_csv_data_from_object(t, "detection", nil, &oid)
+		require.Equal(t, 0, len(records))
+	})
+
+	t.Run("TestCsv_NonDetection", func(t *testing.T) {
+		testhelpers.ClearALeRCEDB()
+		db := populateAlerceDB()
+		defer db.Close()
+		service := NewTapSyncService()
+		w := sendTestQuery("LANG=PSQL&&FORMAT=csv&&QUERY=SELECT * FROM non_detection LIMIT 1", service)
+		require.Equal(t, http.StatusOK, w.Code)
+		require.Equal(t, "text/csv", w.Header().Get("Content-Type"))
+		columnNames := getColumnNames(alercedb.NonDetection{})
+		records, err := csv.NewReader(w.Body).ReadAll()
+		if err != nil {
+			t.Fatal(err)
+		}
+		sort.Strings(columnNames)
+		require.Equal(t, columnNames, records[0])
+	})
+
+	t.Run("TestCsv_NonDetectionFromAnObject", func(t *testing.T) {
+		oidFrom := "non_detection"
+		records := test_get_csv_data_from_object(t, "non_detection", &oidFrom, nil)
+		require.Greater(t, len(records), 1)
+	})
+
+	t.Run("TestCsv_NonDetectionFromUnknownObject", func(t *testing.T) {
+		var oid = "unknown"
+		records := test_get_csv_data_from_object(t, "non_detection", nil, &oid)
+		require.Equal(t, 0, len(records))
+	})
+
+	t.Run("TestCsv_ForcedPhotometry", func(t *testing.T) {
+		testhelpers.ClearALeRCEDB()
+		db := populateAlerceDB()
+		defer db.Close()
+		service := NewTapSyncService()
+		w := sendTestQuery("LANG=PSQL&&FORMAT=csv&&QUERY=SELECT * FROM forced_photometry LIMIT 1", service)
+		require.Equal(t, http.StatusOK, w.Code)
+		require.Equal(t, "text/csv", w.Header().Get("Content-Type"))
+		columnNames := getColumnNames(alercedb.ForcedPhotometry{})
+		records, err := csv.NewReader(w.Body).ReadAll()
+		if err != nil {
+			t.Fatal(err)
+		}
+		sort.Strings(columnNames)
+		require.Equal(t, columnNames, records[0])
+	})
+
+	t.Run("TestCsv_ForcedPhotometryFromAnObject", func(t *testing.T) {
+		oidFrom := "forced_photometry"
+		records := test_get_csv_data_from_object(t, "forced_photometry", &oidFrom, nil)
+		require.Greater(t, len(records), 1)
+	})
+
+	t.Run("TestCsv_ForcedPhotometryFromUnknownObject", func(t *testing.T) {
+		var oid = "unknown"
+		records := test_get_csv_data_from_object(t, "forced_photometry", nil, &oid)
+		require.Equal(t, 0, len(records))
+	})
+
+	t.Run("TestCsv_Features", func(t *testing.T) {
+		testhelpers.ClearALeRCEDB()
+		db := populateAlerceDB()
+		defer db.Close()
+		service := NewTapSyncService()
+		w := sendTestQuery("LANG=PSQL&&FORMAT=csv&&QUERY=SELECT * FROM feature LIMIT 1", service)
+		require.Equal(t, http.StatusOK, w.Code)
+		require.Equal(t, "text/csv", w.Header().Get("Content-Type"))
+		columnNames := getColumnNames(alercedb.Feature{})
+		records, err := csv.NewReader(w.Body).ReadAll()
+		if err != nil {
+			t.Fatal(err)
+		}
+		sort.Strings(columnNames)
+		require.Equal(t, columnNames, records[0])
+	})
+
+	t.Run("TestCsv_FeaturesFromAnObject", func(t *testing.T) {
+		oidFrom := "feature"
+		records := test_get_csv_data_from_object(t, "feature", &oidFrom, nil)
+		require.Greater(t, len(records), 1)
+	})
+
+	t.Run("TestCsv_FeaturesFromUnknownObject", func(t *testing.T) {
+		var oid = "unknown"
+		records := test_get_csv_data_from_object(t, "feature", nil, &oid)
+		require.Equal(t, 0, len(records))
+	})
+
+	t.Run("TestCsv_Objects", func(t *testing.T) {
+		testhelpers.ClearALeRCEDB()
+		db := populateAlerceDB()
+		defer db.Close()
+		service := NewTapSyncService()
+		w := sendTestQuery("LANG=PSQL&&FORMAT=csv&&QUERY=SELECT * FROM object LIMIT 2", service)
+		require.Equal(t, http.StatusOK, w.Code)
+		require.Equal(t, "text/csv", w.Header().Get("Content-Type"))
+		columnNames := getColumnNames(alercedb.Object{})
+		records, err := csv.NewReader(w.Body).ReadAll()
+		if err != nil {
+			t.Fatal(err)
+		}
+		sort.Strings(columnNames)
+		require.Equal(t, columnNames, records[0])
+		require.Equal(t, 3, len(records)) // header + 2 objects
+	})
+
+	t.Run("TestCsv_Probabilities", func(t *testing.T) {
+		testhelpers.ClearALeRCEDB()
+		db := populateAlerceDB()
+		defer db.Close()
+		service := NewTapSyncService()
+		w := sendTestQuery("LANG=PSQL&&FORMAT=csv&&QUERY=SELECT * FROM probability LIMIT 1", service)
+		require.Equal(t, http.StatusOK, w.Code)
+		require.Equal(t, "text/csv", w.Header().Get("Content-Type"))
+		columnNames := getColumnNames(alercedb.Probability{})
+		records, err := csv.NewReader(w.Body).ReadAll()
+		if err != nil {
+			t.Fatal(err)
+		}
+		sort.Strings(columnNames)
+		require.Equal(t, columnNames, records[0])
+	})
+
+	t.Run("TestCsv_ProbabilitiesFromAnObject", func(t *testing.T) {
+		oidFrom := "probability"
+		records := test_get_csv_data_from_object(t, "probability", &oidFrom, nil)
+		require.Greater(t, len(records), 1)
+	})
+}