routine maintenance (May 2023)

* bump SQLite & zstd
* fix bitrot in tests & CI

.github/workflows/build.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 env:
   APT_DEPS: libzstd-dev samtools tabix libhts-dev pigz python3-pip libcurl4-openssl-dev
-  BREW_DEPS: [email protected] zstd samtools pigz coreutils sqlite
+  BREW_DEPS: zstd samtools pigz coreutils sqlite
   PIP_DEPS: pytest pytest-xdist
 
 jobs:
@@ -114,11 +114,14 @@ jobs:
         path: libsqlite3.so.0
 
   macOS-x86-64-dylib:
-    runs-on: macOS-10.15
+    runs-on: macOS-11
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.8'
     - name: dependencies
       run: |
         rm -f /usr/local/bin/2to3  # https://github.com/pypa/pipenv/issues/3831
@@ -127,17 +130,17 @@ jobs:
           brew install $dep || brew upgrade $dep
         done
 
-        /usr/local/bin/pip3 install $PIP_DEPS
+        pip install $PIP_DEPS
     - name: rust toolchain
       uses: actions-rs/toolchain@v1
       with:
         toolchain: stable
         default: true
     - name: build environment
       run: |
-        echo "PATH=$(brew --prefix)/opt/python/libexec/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV
-        echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=sandybridge" >> $GITHUB_ENV
-        echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=sandybridge" >> $GITHUB_ENV
+        echo "PATH=$(brew --prefix)/bin:$(brew --prefix)/opt/sqlite/bin:${PATH}" >> $GITHUB_ENV
+        echo "CFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=ivybridge" >> $GITHUB_ENV
+        echo "CXXFLAGS=-I$(brew --prefix)/include -I$(brew --prefix)/opt/sqlite/include -march=ivybridge" >> $GITHUB_ENV
         echo "LDFLAGS=-L$(brew --prefix)/lib -L$(brew --prefix)/opt/sqlite/lib" >> $GITHUB_ENV
         # used by rusqlite:
         echo "SQLITE3_INCLUDE_DIR=$(brew --prefix)/opt/sqlite/include" >> $GITHUB_ENV

.gitignore

@@ -35,3 +35,5 @@ build/
 __pycache__
 .vscode/
 site/
+venv/
+.venv/

Dockerfile

@@ -4,8 +4,8 @@
 FROM centos:7 as builder
 
 ARG CMAKE_VERSION=3.17.3
-ARG SQLITE_VERSION=3410000
-ARG ZSTD_VERSION=1.5.4
+ARG SQLITE_VERSION=3410200
+ARG ZSTD_VERSION=1.5.5
 ARG CPU_ARCH=ivybridge
 ENV CFLAGS="-march=${CPU_ARCH} -O3"
 ENV CXXFLAGS=${CFLAGS}

README.md

@@ -22,7 +22,7 @@ Our **[Colab notebook](https://colab.research.google.com/drive/1OlHPOcRQBhDmEnS1
 
 **Start Here 👉 [full documentation site](https://mlin.github.io/GenomicSQLite/)**
 
-We supply the extension [prepackaged](https://github.com/mlin/GenomicSQLite/releases) for Linux x86-64 and macOS Catalina. An up-to-date version of SQLite itself is also required, as specified in the docs.
+We supply the extension [prepackaged](https://github.com/mlin/GenomicSQLite/releases) for Linux and macOS on x86-64. An up-to-date version of SQLite itself is also required, as specified in the docs.
 
 Programming language support:
 

src/genomicsqlite.cc

@@ -1552,9 +1552,12 @@ const unsigned char dna_complement_table[] = {
     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
 
 extern "C" int dna_revcomp(const char *dna, size_t len, char *out) {
-    for (; len; --len, ++out)
-        if ((*out = dna_complement_table[(unsigned char)dna[len - 1]]) == 0xFF)
+    for (; len; --len, ++out) {
+        unsigned char c = dna_complement_table[(unsigned char)dna[len - 1]];
+        if (c == 0xFF)
             return -1;
+        *out = c;
+    }
     *out = 0;
     return 0;
 }

test/test_gri.py

@@ -37,6 +37,9 @@ def test_gri_lvl():
     )
 
 
+@pytest.mark.skipif(
+    sqlite3.sqlite_version in ("3.40.0", "3.40.1"), reason="SQLite query planning regression"
+)
 def test_indexing():
     con = sqlite3.connect(":memory:")
     _fill_exons(con)
@@ -271,7 +274,7 @@ def test_connect(tmp_path):
     con.executescript(genomicsqlite.put_reference_assembly_sql(con, "GRCh38_no_alt_analysis_set"))
     _fill_exons(con)
     con.commit()
-    del con
+    con.close()
 
     con = genomicsqlite.connect(dbfile, read_only=True)
     query = (

test/test_twobit.py

@@ -110,11 +110,11 @@ def test_dna_revcomp():
     assert next(con.execute("SELECT dna_revcomp('')"))[0] == ""
     assert next(con.execute("SELECT dna_revcomp(NULL)"))[0] is None
 
-    with pytest.raises(sqlite3.OperationalError):
+    with pytest.raises(sqlite3.OperationalError, match="non-DNA input to dna_revcomp()"):
         con.execute("SELECT dna_revcomp('GATTACAb')")
 
-    with pytest.raises(sqlite3.OperationalError):
+    with pytest.raises(sqlite3.OperationalError, match="non-DNA input to dna_revcomp()"):
         con.execute("SELECT dna_revcomp('GATTACA ')")
 
-    with pytest.raises(sqlite3.OperationalError):
+    with pytest.raises(sqlite3.OperationalError, match="argument #1 type mismatch"):
         con.execute("SELECT dna_revcomp(42)")

test/test_vcf.py

@@ -1,5 +1,7 @@
 import os
+import sqlite3
 import subprocess
+import pytest
 import genomicsqlite
 
 HERE = os.path.dirname(__file__)
@@ -18,6 +20,9 @@ def vcf_into_sqlite(infilename, outfilename, *options):
     print(outfilename)
 
 
+@pytest.mark.skipif(
+    sqlite3.sqlite_version in ("3.40.0", "3.40.1"), reason="SQLite query planning regression"
+)
 def test_gnomad_sites_small(tmp_path):
     dbfile = str(tmp_path / "test.gsql")
 

test/test_vcf_lines.py

@@ -1,6 +1,9 @@
 import os
+import sqlite3
 import subprocess
 import genomicsqlite
+import pytest
+
 
 HERE = os.path.dirname(__file__)
 BUILD = os.path.abspath(os.path.join(HERE, "..", "build"))
@@ -13,6 +16,9 @@ def vcf_lines_into_sqlite(infilename, outfilename, *options):
     print(outfilename)
 
 
+@pytest.mark.skipif(
+    sqlite3.sqlite_version in ("3.40.0", "3.40.1"), reason="SQLite query planning regression"
+)
 def test_gnomad_sites_small(tmp_path):
     dbfile = str(tmp_path / "gnomad_lines.gsql")