functest fix

repo_utils/pqeq.py

@@ -0,0 +1,29 @@
+"""
+Checks if two parquet files are equal
+"""
+import sys
+import pandas as pd
+
+def compare_parquet_files(file1, file2):
+    df1 = pd.read_parquet(file1)
+    df2 = pd.read_parquet(file2)
+
+    if df1.shape != df2.shape:
+        return False
+
+    if set(df1.columns) != set(df2.columns):
+        return False
+
+    return df1.equals(df2)
+
+# Example usage
+file1 = sys.argv[1]
+file2 = sys.argv[2]
+
+if compare_parquet_files(file1, file2):
+    print("The Parquet files are equal.")
+    sys.exit(0)
+else:
+    print("The Parquet files are not equal.")
+    sys.exit(1)
+

repo_utils/tdb_ssshtests.sh

@@ -153,24 +153,27 @@ fi
 
 run test_q_methyl $tdb query methyl $INDIR/tdb/merge1.tdb -O p -o $OD/methyl.pq
 if [ $test_q_methyl ]; then
+    assert_exit_code 0
     if [ "${STOPCHECK}" != 'true' ]; then
-        assert_equal $(fn_md5 $INDIR/queries/methyl.pq) $(fn_md5 $OD/methyl.pq)
+        python3 repo_utils/pqeq.py $INDIR/queries/methyl.pq $OD/methyl.pq
+        assert_equal $? 0
     fi
-    assert_exit_code 0
 fi
 
 run test_q_comp_poly_score $tdb query comp_poly_score $INDIR/tdb/merge1.tdb -O p -o $OD/comp_poly_score.pq
 if [ $test_q_comp_poly_score ]; then
     if [ "${STOPCHECK}" != 'true' ]; then
-        assert_equal $(fn_md5 $INDIR/queries/comp_poly_score.pq) $(fn_md5 $OD/comp_poly_score.pq)
+        python3 repo_utils/pqeq.py $INDIR/queries/comp_poly_score.pq $OD/comp_poly_score.pq
+        assert_equal $? 0
     fi
     assert_exit_code 0
 fi
 
 run test_q_len_poly_score $tdb query len_poly_score $INDIR/tdb/merge1.tdb -O p -o $OD/len_poly_score.pq
 if [ $test_q_len_poly_score ]; then
     if [ "${STOPCHECK}" != 'true' ]; then
-        assert_equal $(fn_md5 $INDIR/queries/len_poly_score.pq) $(fn_md5 $OD/len_poly_score.pq)
+        python3 repo_utils/pqeq.py $INDIR/queries/len_poly_score.pq $OD/len_poly_score.pq
+        assert_equal $? 0
     fi
     assert_exit_code 0
 fi

tdb/create.py

@@ -77,7 +77,7 @@ def make_parquets(samples, out_dir, compression):
     return ret
 
 
-def sample_extract(locus_id, fmt_fields):
+def sample_extract(locus_id, fmt):
     """
     Given a dict from a vcf record sample, turn them into sample rows
     """