The centralized nightlies job failed on Friday (2024-08-16) #18

github-actions · 2024-08-17T02:41:12Z

The centralized nightlies job failed on Friday (2024-08-16) in run 10428720345

github-actions · 2024-08-18T02:49:31Z

The centralized nightlies job failed on Saturday (2024-08-17) in run 10437049686

github-actions · 2024-08-19T02:42:54Z

The centralized nightlies job failed on Sunday (2024-08-18) in run 10445960340

jdblischak · 2024-08-19T14:23:20Z

Both TileDB-Go and TileDB-SOMA-Py started failing their tests on Friday night. I had expected these failures because they both also failed earlier on Friday at noon.

jdblischak · 2024-08-19T14:32:43Z

The TileDB-Go test failure has been consistent:

=== RUN   TestDeserializeArraySchemaGC
SIGSEGV: segmentation violation
PC=0x7f7c1bbfc2b0 m=3 sigcode=1
signal arrived during cgo execution

I don't see any open PR/Issue in TileDB-Go related to this. Also, the test file itself, serialize_test.go, hasn't been edited in 3 years. Looking upstream in TileDB, my guess is that this PR TileDB-Inc/TileDB#5225 is the culprit since it is related to ArraySchema

Click here for full traceback

=== RUN   TestDeserializeArraySchemaGC
SIGSEGV: segmentation violation
PC=0x7fe5a2bfc2b0 m=3 sigcode=1
signal arrived during cgo execution

goroutine 161 [syscall]:
runtime.cgocall(0x7623e0, 0xc000045e68)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/cgocall.go:157 +0x4b fp=0xc000045e40 sp=0xc000045e08 pc=0x41258b
github.com/TileDB-Inc/TileDB-Go._Cfunc_tiledb_deserialize_array_schema(0x7fe54d19e9e0, 0x7fe54d1b8b20, 0x1, 0x1, 0xc000371640)
	_cgo_gotypes.go:2780 +0x4b fp=0xc000045e68 sp=0xc000045e40 pc=0x6f70cb
github.com/TileDB-Inc/TileDB-Go.DeserializeArraySchema.func1(0xc000371640, 0x716b06?, 0x70?, 0xc0?)
	/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/serialize.go:49 +0xc5 fp=0xc000045eb8 sp=0xc000045e68 pc=0x743bc5
github.com/TileDB-Inc/TileDB-Go.DeserializeArraySchema(0xc000355aa0, 0x70?, 0x1)
	/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/serialize.go:49 +0x65 fp=0xc000045f10 sp=0xc000045eb8 pc=0x743a45
github.com/TileDB-Inc/TileDB-Go.TestDeserializeArraySchemaGC(0xc000378ea0)
	/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/serialize_test.go:23 +0x11f fp=0xc000045f70 sp=0xc000045f10 pc=0x6e889f
testing.tRunner(0xc000378ea0, 0x82fac0)
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1595 +0xff fp=0xc000045fc0 sp=0xc000045f70 pc=0x526bff
testing.(*T).Run.func1()
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1648 +0x25 fp=0xc000045fe0 sp=0xc000045fc0 pc=0x527b85
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000045fe8 sp=0xc000045fe0 pc=0x47bba1
created by testing.(*T).Run in goroutine 1
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1648 +0x3ad

goroutine 1 [chan receive]:
runtime.gopark(0xc0000d99e8?, 0x41b7c5?, 0x98?, 0x79?, 0x18?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc0000d9980 sp=0xc0000d9960 pc=0x448e8e
runtime.chanrecv(0xc0000ef420, 0xc0000d9a67, 0x1)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/chan.go:583 +0x3cd fp=0xc0000d99f8 sp=0xc0000d9980 pc=0x414bcd
runtime.chanrecv1(0xaf72c0?, 0x78c5a0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/chan.go:442 +0x12 fp=0xc0000d9a20 sp=0xc0000d99f8 pc=0x4147d2
testing.(*T).Run(0xc000083040, {0x8035d3?, 0x5264dc?}, 0x82fac0)
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1649 +0x3c8 fp=0xc0000d9ae0 sp=0xc0000d9a20 pc=0x527a28
testing.runTests.func1(0xaf8200?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2054 +0x3e fp=0xc0000d9b30 sp=0xc0000d9ae0 pc=0x529b3e
testing.tRunner(0xc000083040, 0xc0000fdc48)
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1595 +0xff fp=0xc0000d9b80 sp=0xc0000d9b30 pc=0x526bff
testing.runTests(0xc0000a3220?, {0xac55a0, 0x65, 0x65}, {0x4217bf?, 0xc0000fdd08?, 0xaf7920?})
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2052 +0x445 fp=0xc0000d9c78 sp=0xc0000d9b80 pc=0x529a25
testing.(*M).Run(0xc0000a3220)
	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1925 +0x636 fp=0xc0000d9ec0 sp=0xc0000d9c78 pc=0x528416
main.main()
	_testmain.go:259 +0x19c fp=0xc0000d9f40 sp=0xc0000d9ec0 pc=0x76119c
runtime.main()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:267 +0x2bb fp=0xc0000d9fe0 sp=0xc0000d9f40 pc=0x448a1b
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000d9fe8 sp=0xc0000d9fe0 pc=0x47bba1

goroutine 2 [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc000042fa8 sp=0xc000042f88 pc=0x448e8e
runtime.goparkunlock(...)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:404
runtime.forcegchelper()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:322 +0xb3 fp=0xc000042fe0 sp=0xc000042fa8 pc=0x448cf3
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000042fe8 sp=0xc000042fe0 pc=0x47bba1
created by runtime.init.6 in goroutine 1
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:310 +0x1a

goroutine 3 [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc000043778 sp=0xc000043758 pc=0x448e8e
runtime.goparkunlock(...)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:404
runtime.bgsweep(0x0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgcsweep.go:321 +0xdf fp=0xc0000437c8 sp=0xc000043778 pc=0x4330bf
runtime.gcenable.func1()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:200 +0x25 fp=0xc0000437e0 sp=0xc0000437c8 pc=0x428185
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000437e8 sp=0xc0000437e0 pc=0x47bba1
created by runtime.gcenable in goroutine 1
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:200 +0x66

goroutine 4 [sleep]:
runtime.gopark(0xc000016050?, 0x19e3d46efc?, 0x0?, 0x0?, 0x82fe98?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc000043f30 sp=0xc000043f10 pc=0x448e8e
runtime.goparkunlock(...)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:404
runtime.(*scavengerState).sleep(0xaf79a0, 0x4113de7800000000)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgcscavenge.go:504 +0x113 fp=0xc000043fa0 sp=0xc000043f30 pc=0x430b13
runtime.bgscavenge(0x0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgcscavenge.go:662 +0x74 fp=0xc000043fc8 sp=0xc000043fa0 pc=0x430f14
runtime.gcenable.func2()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:201 +0x25 fp=0xc000043fe0 sp=0xc000043fc8 pc=0x428125
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000043fe8 sp=0xc000043fe0 pc=0x47bba1
created by runtime.gcenable in goroutine 1
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:201 +0xa5

goroutine 18 [finalizer wait]:
runtime.gopark(0x0?, 0x82fe48?, 0x90?, 0x41?, 0x1000000010?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc000059e20 sp=0xc000059e00 pc=0x448e8e
runtime.runfinq()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mfinal.go:193 +0x107 fp=0xc000059fe0 sp=0xc000059e20 pc=0x[427](https://github.com/jdblischak/centralized-tiledb-nightlies/actions/runs/10428720345/job/28885471574#step:5:428)127
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000059fe8 sp=0xc000059fe0 pc=0x47bba1
created by runtime.createfing in goroutine 1
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mfinal.go:163 +0x3d

goroutine 48 [GC worker (idle)]:
runtime.gopark(0x19a80b2d85?, 0xc0001484e0?, 0xb0?, 0xe7?, 0x526bff?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc00003e750 sp=0xc00003e730 pc=0x448e8e
runtime.gcBgMarkWorker()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1295 +0xe5 fp=0xc00003e7e0 sp=0xc00003e750 pc=0x[429](https://github.com/jdblischak/centralized-tiledb-nightlies/actions/runs/10428720345/job/28885471574#step:5:430)d05
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00003e7e8 sp=0xc00003e7e0 pc=0x47bba1
created by runtime.gcBgMarkStartWorkers in goroutine 47
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1219 +0x1c

goroutine 50 [GC worker (idle)]:
runtime.gopark(0x19a809c925?, 0x3?, 0x67?, 0x87?, 0x0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc000486750 sp=0xc000486730 pc=0x448e8e
runtime.gcBgMarkWorker()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1295 +0xe5 fp=0xc0004867e0 sp=0xc000486750 pc=0x429d05
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0004867e8 sp=0xc0004867e0 pc=0x47bba1
created by runtime.gcBgMarkStartWorkers in goroutine 47
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1219 +0x1c

goroutine 49 [GC worker (idle)]:
runtime.gopark(0x19a809cbd8?, 0x0?, 0x0?, 0x0?, 0x0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc00003ef50 sp=0xc00003ef30 pc=0x448e8e
runtime.gcBgMarkWorker()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1295 +0xe5 fp=0xc00003efe0 sp=0xc00003ef50 pc=0x429d05
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00003efe8 sp=0xc00003efe0 pc=0x47bba1
created by runtime.gcBgMarkStartWorkers in goroutine 47
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1219 +0x1c

goroutine 51 [GC worker (idle)]:
runtime.gopark(0x19a809c79c?, 0x3?, 0x46?, 0xa3?, 0x0?)
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/proc.go:398 +0xce fp=0xc000486f50 sp=0xc000486f30 pc=0x[448](https://github.com/jdblischak/centralized-tiledb-nightlies/actions/runs/10428720345/job/28885471574#step:5:449)e8e
runtime.gcBgMarkWorker()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1295 +0xe5 fp=0xc000486fe0 sp=0xc000486f50 pc=0x429d05
runtime.goexit()
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000486fe8 sp=0xc000486fe0 pc=0x47bba1
created by runtime.gcBgMarkStartWorkers in goroutine 47
	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/mgc.go:1219 +0x1c

rax    0x0
rbx    0x7fe54d197340
rcx    0x1
rdx    0x7fe54d197360
rdi    0x1
rsi    0x7fe54d1972f0
rbp    0x7fe54d1b8b20
rsp    0x7fe55a7e8110
r8     0x0
r9     0x7fe55a7e7a50
r10    0x7fe5a49ff220
r11    0x0
r12    0xc000371640
r13    0x1
r14    0x7fe54d1b8b80
r15    0x1
rip    0x7fe5a2bfc2b0
rflags 0x10283
cs     0x33
fs     0x0
gs     0x0
FAIL	github.com/TileDB-Inc/TileDB-Go	2.480s

jdblischak · 2024-08-19T14:44:18Z

Friday night a TileDB-SOMA-Py test caused a segfault:

apis/python/tests/test_sparse_nd_array.py .............................. [ 60%]
........................................................................ [ 64%]
........................................................................ [ 68%]
................................................................x....... [ 73%]
........................................................................ [ 77%]
........................................................................ [ 81%]
........................................................................ [ 85%]
........................................................................ [ 89%]
Fatal Python error: Segmentation fault

I assume Saturday night also failed with the same segfault. Unfortunately GitHub didn't save the log for this build, but the tests ran for the same amount of time as Friday prior to failing.

I couldn't find a potential cause for this failure. The test file, test_sparse_nd_array.py, was last edited 2 weeks ago, which doesn't line up with a failure that started on Friday.

Click here for full traceback

apis/python/tests/test_sparse_nd_array.py .............................. [ 60%]
........................................................................ [ 64%]
........................................................................ [ 68%]
................................................................x....... [ 73%]
........................................................................ [ 77%]
........................................................................ [ 81%]
........................................................................ [ 85%]
........................................................................ [ 89%]
Fatal Python error: Segmentation fault

Thread 0x00007ffaaeefe640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ffa1d8d3640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 327 in wait
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/_base.py", line 451 in result
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/somacore/query/_eager_iter.py", line 31 in __next__
  File "/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/src/tiledbsoma/_read_iters.py", line 346 in _sorted_tbl_reader
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 58 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 83 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ffa1e8d4640 (most recent call first):
  File "/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/src/tiledbsoma/_read_iters.py", line 462 in _arrow_table_reader
  File "/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/src/tiledbsoma/_read_iters.py", line 243 in _table_reader
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 58 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 83 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff9beb86640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Current thread 0x00007ff9bfb87640 (most recent call first):
  File "/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/src/tiledbsoma/_indexer.py", line 86 in get_indexer
  File "/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/src/tiledbsoma/_read_iters.py", line 259 in _reindexed_table_reader
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 58 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 83 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff9c0b88640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff90301f640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ffa36ae1640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff8d6015640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff85dff8640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff85eff9640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff865feb640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff90101d640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff91e025640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff91f026640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff9d80a7640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff91b022640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ffa208d6640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff8cd013640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff8ce014640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff8cf015640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ff9a7b8b640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ffa42fe6640 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/concurrent/futures/thread.py", line 81 in _worker
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 982 in run
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1045 in _bootstrap_inner
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/threading.py", line 1002 in _bootstrap

Thread 0x00007ffb09639b80 (most recent call first):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/capture.py", line 573 in snap
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/capture.py", line 685 in readouterr
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/capture.py", line 782 in read_global_capture
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/capture.py", line 846 in item_capture
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/contextlib.py", line 144 in __exit__
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/capture.py", line 878 in pytest_runtest_call
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_callers.py", line 124 in _multicall
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_manager.py", line 120 in _hookexec
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_hooks.py", line 513 in __call__
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/runner.py", line 242 in <lambda>
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/runner.py", line 341 in from_call
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/runner.py", line 241 in call_and_report
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/runner.py", line 132 in runtestprotocol
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/runner.py", line 113 in pytest_runtest_protocol
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_callers.py", line 103 in _multicall
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_manager.py", line 120 in _hookexec
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_hooks.py", line 513 in __call__
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/main.py", line 362 in pytest_runtestloop
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_callers.py", line 103 in _multicall
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_manager.py", line 120 in _hookexec
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_hooks.py", line 513 in __call__
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/main.py", line 337 in _main
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/main.py", line 283 in wrap_session
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/main.py", line 330 in pytest_cmdline_main
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_callers.py", line 103 in _multicall
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_manager.py", line 120 in _hookexec
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pluggy/_hooks.py", line 513 in __call__
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/config/__init__.py", line 175 in main
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/_pytest/config/__init__.py", line 201 in console_main
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pytest/__main__.py", line 9 in <module>
  File "<frozen runpy>", line 88 in _run_code
  File "<frozen runpy>", line 198 in _run_module_as_main

Extension modules: numpy.core._multiarray_umath, numpy.core._multiarray_tests, numpy.linalg._umath_linalg, numpy.fft._pocketfft_internal, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, pyarrow.lib/home/runner/work/_temp/6c083140-d852-4377-97d9-d93cf38a31bc.sh: line 9:  1951 Segmentation fault      (core dumped) PYTHONPATH=$(pwd)/apis/python/src python -m pytest apis/python/tests

jdblischak · 2024-08-19T14:52:39Z

And Sunday night a TileDB-Py test caused a segfault (thus TileDB-SOMA-Py was never built).

=========================== short test summary info ============================
FAILED tiledb/tests/test_enumeration.py::EnumerationTest::test_array_schema_enumeration_nullable[True-False] - ValueError: NumPy array conversion check failed for attr 'a'
== 1 failed, 953 passed, 6 skipped, 8 xfailed, 1 xpassed in 90.07s (0:01:30) ===

This same error also broke the upstream TileDB-Py nightlies over the weekend (TileDB-Inc/TileDB-Py#2040 (comment))

Given that this is also related to ArraySchema, I again suspect this is due to the recently merged TileDB-Inc/TileDB#5225

Click here for full traceback

=================================== FAILURES ===================================
______ EnumerationTest.test_array_schema_enumeration_nullable[True-False] ______

>   ???
E   RuntimeWarning: invalid value encountered in cast

libtiledb.pyx:2710: RuntimeWarning

The above exception was the direct cause of the following exception:

self = <tiledb.tests.test_enumeration.EnumerationTest object at 0x7ff5b6ace6d0>
sparse = False, pass_df = True

    @pytest.mark.skipif(
        not has_pyarrow() or not has_pandas(),
        reason="pyarrow>=1.0 and/or pandas>=1.0,<3.0 not installed",
    )
    @pytest.mark.parametrize("sparse", [True, False])
    @pytest.mark.parametrize("pass_df", [True, False])
    def test_array_schema_enumeration_nullable(self, sparse, pass_df):
        import pyarrow as pa
    
        uri = self.path("test_array_schema_enumeration_nullable")
        enmr = tiledb.Enumeration("e", False, ["alpha", "beta", "gamma"])
        dom = tiledb.Domain(tiledb.Dim("d", domain=(1, 5), dtype="int64"))
        att = tiledb.Attr("a", dtype="int8", nullable=True, enum_label="e")
        schema = tiledb.ArraySchema(
            domain=dom, attrs=[att], enums=[enmr], sparse=sparse
        )
        tiledb.Array.create(uri, schema)
    
        with tiledb.open(uri, "w") as A:
            dims = pa.array([1, 2, 3, 4, 5])
            data = pa.array([1.0, 2.0, None, 0, 1.0])
            if pass_df:
                dims = dims.to_pandas()
                data = data.to_pandas()
    
            if sparse:
                A[dims] = data
            else:
>               A[:] = data

tiledb/tests/test_enumeration.py:137: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
libtiledb.pyx:2576: in tiledb.libtiledb.DenseArrayImpl.__setitem__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   ValueError: NumPy array conversion check failed for attr 'a'

libtiledb.pyx:2712: ValueError
=========================== short test summary info ============================
FAILED tiledb/tests/test_enumeration.py::EnumerationTest::test_array_schema_enumeration_nullable[True-False] - ValueError: NumPy array conversion check failed for attr 'a'
== 1 failed, 953 passed, 6 skipped, 8 xfailed, 1 xpassed in 90.07s (0:01:30) ===

jdblischak · 2024-08-19T16:13:21Z

I couldn't find a potential cause for this failure. The test file, test_sparse_nd_array.py, was last edited 2 weeks ago, which doesn't line up with a failure that started on Friday.

Actually, skimming test_sparse_nd_array.py, there are various tests of the "array schema", so I assume the TileDB-SOMA-Py test segfault is also related to TileDB-Inc/TileDB#5225

github-actions · 2024-08-20T02:40:49Z

The centralized nightlies job failed on Monday (2024-08-19) in run 10464107494

jdblischak · 2024-08-20T18:43:39Z

The TileDB-Py test failure was fixed today in TileDB-Inc/TileDB-Py#2042, which I confirmed via a manual nightly run on main. Thanks @kounelisagis!

Note that TileDB-Go and TileDB-SOMA-Py continue to fail their tests

github-actions · 2024-08-21T02:43:10Z

The centralized nightlies job failed on Tuesday (2024-08-20) in run 10482317549

jdblischak · 2024-08-21T14:13:20Z

As anticipated, TileDB-Go and TileDB-SOMA-Py continue to fail their tests

github-actions · 2024-08-22T02:44:31Z

The centralized nightlies job failed on Wednesday (2024-08-21) in run 10500657691

github-actions · 2024-08-23T02:41:41Z

The centralized nightlies job failed on Thursday (2024-08-22) in run 10518723380

github-actions · 2024-08-23T14:07:15Z

The centralized nightlies job failed on Friday (2024-08-23) in run 10518723380

github-actions · 2024-08-23T14:23:44Z

The centralized nightlies job failed on Friday (2024-08-23) in run 10518723380

jdblischak · 2024-08-23T14:36:30Z

Lots going on here.

Last night there were two failures: TileDB-Go (expected) and TileDB-Py (unexpected)

TileDB-Py failed a single test, and it passed on the second attempt

=========================== short test summary info ============================
FAILED tiledb/tests/test_libtiledb.py::DenseArrayTest::test_open_with_timestamp[False] - AssertionError
== 1 failed, 961 passed, 6 skipped, 8 xfailed, 1 xpassed in 88.51s (0:01:28) ===

After TileDB-Py passed, there were two failures: TileDB-SOMA-Py (expected) and TileDB-VCF-Py (unexpected)

I restarted TileDB-VCF-Py, but it failed a second time with the same error while building the wheel:

  -- Searching for libtiledbvcf in /home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/../../dist/lib;/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/dist_links/dist/lib;/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/install-libtiledbvcf/lib/
  -- Found libtiledbvcf: /home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/install-libtiledbvcf/lib/libtiledbvcf.so
  CMake Error at CMakeLists.txt:114 (find_package):
    By not providing "FindTileDB.cmake" in CMAKE_MODULE_PATH this project has
    asked CMake to find a package configuration file provided by "TileDB", but
    CMake did not find one.

    Could not find a package configuration file provided by "TileDB" with any
    of the following names:

      TileDBConfig.cmake
      tiledb-config.cmake

    Add the installation prefix of "TileDB" to CMAKE_PREFIX_PATH or set
    "TileDB_DIR" to a directory containing one of the above files.  If "TileDB"
    provides a separate development package or SDK, be sure it has been
    installed.


  -- Configuring incomplete, errors occurred!

  *** CMake configuration failed
  error: subprocess-exited-with-error

I don't understand why this error is just surfacing now. There have been no updates to TileDB-VCF and none related to the CMake configuration of TileDB.

github-actions · 2024-08-24T02:42:24Z

The centralized nightlies job failed on Friday (2024-08-23) in run 10534890732

github-actions · 2024-08-25T02:47:00Z

The centralized nightlies job failed on Saturday (2024-08-24) in run 10543362914

github-actions · 2024-08-26T02:43:43Z

The centralized nightlies job failed on Sunday (2024-08-25) in run 10552357727

jdblischak · 2024-08-26T15:04:10Z

TileDB-Go and TileDB-SOMA-Py continue to fail their tests. TileDB-VCF-Py continues to fail to build a wheel due to some CMake configuration issue

jdblischak · 2024-08-26T16:39:54Z

Investigating more what could have changed in the TileDB-VCF-Py wheel build.

The last passing build was the evening of Wed, Aug 21st. The first failing build was the morning of Fri, Aug 23nd. It didn't run Thursday evening because the TileDB-Py build spuriously failed, so the problem didn't appear until I restarted it Friday morning.

Looking for changes in dependencies. The version of CMake is the same, 3.30.2. The version of scikit-build-core changed from 0.10.4 to 0.10.5, but I don't see anything obvious in the 0.10.5 changelog that would affect this behavior. packaging, setuptools, and setuptools-scm are all the same version.

I noticed that the failing build checked for the CXX compiler, but presumably the passing build retrieved this from the cache. Maybe the cache has been masking this error until now.

# passing build
  Using pip 24.2 from /opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pip (python 3.11)
  Successfully installed packaging-24.1 pathspec-0.12.1 pybind11-2.13.4 pybind11-global-2.13.4 scikit-build-core-0.10.4
  Successfully installed ninja-1.11.1.1 packaging-24.1 setuptools-73.0.1 setuptools-scm-8.1.0
  *** scikit-build-core 0.10.4 using CMake 3.30.2 (metadata_wheel)
Building wheels for collected packages: tiledbvcf
  Building wheel for tiledbvcf (pyproject.toml): started
  Running command Building wheel for tiledbvcf (pyproject.toml)
  *** scikit-build-core 0.10.4 using CMake 3.30.2 (wheel)
  *** Configuring CMake...
  loading initial cache file build/cp311-cp311-linux_x86_64/CMakeInit.txt
  -- Searching for libtiledbvcf in /home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/../../dist/lib;/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/dist_links/dist/lib;/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/install-libtiledbvcf/lib/
  -- Found libtiledbvcf: /home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/install-libtiledbvcf/lib/libtiledbvcf.so
  -- Found TileDB: /home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/install-libtiledb/lib/libtiledb.so.2.27

# failing build
  Using pip 24.2 from /opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/site-packages/pip (python 3.11)
  Successfully installed packaging-24.1 pathspec-0.12.1 pybind11-2.13.5 pybind11-global-2.13.5 scikit-build-core-0.10.5
  Successfully installed ninja-1.11.1.1 packaging-24.1 setuptools-73.0.1 setuptools-scm-8.1.0
  *** scikit-build-core 0.10.5 using CMake 3.30.2 (metadata_wheel)
Building wheels for collected packages: tiledbvcf
  Building wheel for tiledbvcf (pyproject.toml): started
  Running command Building wheel for tiledbvcf (pyproject.toml)
  *** scikit-build-core 0.10.5 using CMake 3.30.2 (wheel)
  *** Configuring CMake...
  loading initial cache file build/cp311-cp311-linux_x86_64/CMakeInit.txt
  -- The CXX compiler identification is GNU 11.4.0
  -- Detecting CXX compiler ABI info
  -- Detecting CXX compiler ABI info - done
  -- Check for working CXX compiler: /usr/bin/g++ - skipped
  -- Detecting CXX compile features
  -- Detecting CXX compile features - done
  -- Searching for libtiledbvcf in /home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/../../dist/lib;/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/apis/python/dist_links/dist/lib;/home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/install-libtiledbvcf/lib/
  -- Found libtiledbvcf: /home/runner/work/centralized-tiledb-nightlies/centralized-tiledb-nightlies/install-libtiledbvcf/lib/libtiledbvcf.so
  CMake Error at CMakeLists.txt:114 (find_package):
    By not providing "FindTileDB.cmake" in CMAKE_MODULE_PATH this project has
    asked CMake to find a package configuration file provided by "TileDB", but
    CMake did not find one.

    Could not find a package configuration file provided by "TileDB" with any
    of the following names:

      TileDBConfig.cmake
      tiledb-config.cmake

    Add the installation prefix of "TileDB" to CMAKE_PREFIX_PATH or set
    "TileDB_DIR" to a directory containing one of the above files.  If "TileDB"
    provides a separate development package or SDK, be sure it has been
    installed.


  -- Configuring incomplete, errors occurred!

  *** CMake configuration failed

github-actions · 2024-08-27T02:45:40Z

The centralized nightlies job failed on Monday (2024-08-26) in run 10570516527

jdblischak · 2024-08-27T14:05:51Z

The TileDB-Go and TileDB-SOMA-Py test failures were fixed by TileDB-Inc/TileDB#5256

jdblischak · 2024-08-27T14:06:05Z

So now the only remaining nightly error is the TileDB-VCF-Py wheel build

github-actions · 2024-08-28T02:46:05Z

The centralized nightlies job failed on Tuesday (2024-08-27) in run 10589002726

jdblischak · 2024-08-28T15:09:35Z

Still just the failing TileDB-VCF-Py wheel build

github-actions · 2024-08-29T02:47:01Z

The centralized nightlies job failed on Wednesday (2024-08-28) in run 10607538194

jdblischak · 2024-08-29T17:43:30Z

Add the installation prefix of "TileDB" to CMAKE_PREFIX_PATH or set "TileDB_DIR" to a directory containing one of the above files.

Turns out I had seen this same error previously for the TileDB-VCF-Py build (https://github.com/jdblischak/centralized-tiledb-nightlies/issues/6#issuecomment-2113166645), which I had fixed in jdblischak@61765c5 by setting TileDB_DIR. Thus I fixed the wheel build using the same method in jdblischak@bce055e.

Here's what I don't understand though: why was the behavior so different for the standard build/installation via python -m pip install versus the wheel build via python -m pip wheel???? For the former, I had to update it back in May when the change was first introduced to TileDB-VCF-Py upstream. That makes total sense. But for the latter, it continued to work for months, and only just now started failing. Bizarre.

github-actions bot added the nightly-failure The scheduled nightly builds failed label Aug 17, 2024

github-actions bot assigned jdblischak Aug 17, 2024

jdblischak mentioned this issue Aug 19, 2024

The nightly build with earliest supported numpy job failed on Sunday (2024-08-18) TileDB-Inc/TileDB-Py#2040

Closed

jdblischak closed this as completed Aug 29, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

The centralized nightlies job failed on Friday (2024-08-16) #18

The centralized nightlies job failed on Friday (2024-08-16) #18

github-actions bot commented Aug 17, 2024

github-actions bot commented Aug 18, 2024

github-actions bot commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

github-actions bot commented Aug 20, 2024

jdblischak commented Aug 20, 2024

github-actions bot commented Aug 21, 2024

jdblischak commented Aug 21, 2024

github-actions bot commented Aug 22, 2024

github-actions bot commented Aug 23, 2024

github-actions bot commented Aug 23, 2024

github-actions bot commented Aug 23, 2024

jdblischak commented Aug 23, 2024

github-actions bot commented Aug 24, 2024

github-actions bot commented Aug 25, 2024

github-actions bot commented Aug 26, 2024

jdblischak commented Aug 26, 2024

jdblischak commented Aug 26, 2024

github-actions bot commented Aug 27, 2024

jdblischak commented Aug 27, 2024

jdblischak commented Aug 27, 2024

github-actions bot commented Aug 28, 2024

jdblischak commented Aug 28, 2024

github-actions bot commented Aug 29, 2024

jdblischak commented Aug 29, 2024

The centralized nightlies job failed on Friday (2024-08-16) #18

The centralized nightlies job failed on Friday (2024-08-16) #18

Comments

github-actions bot commented Aug 17, 2024

github-actions bot commented Aug 18, 2024

github-actions bot commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

jdblischak commented Aug 19, 2024

github-actions bot commented Aug 20, 2024

jdblischak commented Aug 20, 2024

github-actions bot commented Aug 21, 2024

jdblischak commented Aug 21, 2024

github-actions bot commented Aug 22, 2024

github-actions bot commented Aug 23, 2024

github-actions bot commented Aug 23, 2024

github-actions bot commented Aug 23, 2024

jdblischak commented Aug 23, 2024

github-actions bot commented Aug 24, 2024

github-actions bot commented Aug 25, 2024

github-actions bot commented Aug 26, 2024

jdblischak commented Aug 26, 2024

jdblischak commented Aug 26, 2024

github-actions bot commented Aug 27, 2024

jdblischak commented Aug 27, 2024

jdblischak commented Aug 27, 2024

github-actions bot commented Aug 28, 2024

jdblischak commented Aug 28, 2024

github-actions bot commented Aug 29, 2024

jdblischak commented Aug 29, 2024