Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GC error #2157

Closed
swilliamson7 opened this issue Dec 2, 2024 · 11 comments · Fixed by #2172
Closed

GC error #2157

swilliamson7 opened this issue Dec 2, 2024 · 11 comments · Fixed by #2172
Labels
gc garbage collection segfault

Comments

@swilliamson7
Copy link
Collaborator

@wsmoses Opening an issue for the GC error I see, I added you to the repo where my code is, it's a private one in the DJ4Earth repository. Running julia --project=. technical_paper.jl should reproduce what I've been seeing. The error is

swilliamson@CRIOS-A66253 ~/D/G/S/eddy-stresses> julia --project=. technical_paper.jl      :) main!?#
[ Info: Revolve: Number of checkpoints: 82
[ Info: Revolve: Number of steps: 6733
[ Info: Prediction:
[ Info: Forward steps   : 16629
[ Info: Overhead factor : 2.469775731471855

[87818] signal (11.2): Segmentation fault: 11
in expression starting at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:598
gc_mark_obj8 at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:0
gc_mark_outrefs at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2623 [inlined]
gc_mark_loop_serial_ at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2679
gc_mark_loop_serial at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2702
gc_mark_loop at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2897 [inlined]
_jl_gc_collect at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:3221
ijl_gc_collect at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:3518
maybe_collect at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:937 [inlined]
jl_gc_pool_alloc_inner at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:1293 [inlined]
jl_gc_pool_alloc_noinline at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:1350 [inlined]
jl_gc_alloc_ at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/./julia_internal.h:477
_new_array_ at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/array.c:144
_new_array at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/array.c:198
ijl_alloc_array_2d at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/array.c:443
Array at ./boot.jl:479 [inlined]
Array at ./boot.jl:487 [inlined]
Array at ./boot.jl:494 [inlined]
similar at ./abstractarray.jl:874 [inlined]
similar at ./abstractarray.jl:873 [inlined]
similar at ./broadcast.jl:224 [inlined]
similar at ./broadcast.jl:223 [inlined]
copy at ./broadcast.jl:928 [inlined]
materialize at ./broadcast.jl:903 [inlined]
macro expansion at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper_integration.jl:216 [inlined]
#1 at /Users/swilliamson/.julia/packages/Checkpointing/RmOhf/src/Checkpointing.jl:181
checkpoint_struct_for at /Users/swilliamson/.julia/packages/Checkpointing/RmOhf/src/Checkpointing.jl:135
advection_coriolis! at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/ShallowWaters.jl/src/rhs.jl:0
length at ./essentials.jl:10 [inlined]
copyto! at ./array.jl:388 [inlined]
checkpointed_integration at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper_integration.jl:62 [inlined]
diffejulia_checkpointed_integration_4198wrap at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper_integration.jl:0
macro expansion at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/compiler.jl:8839 [inlined]
enzyme_call at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/compiler.jl:8405 [inlined]
CombinedAdjointThunk at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/compiler.jl:8178 [inlined]
autodiff at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/Enzyme.jl:491 [inlined]
autodiff at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/Enzyme.jl:537
unknown function (ip: 0x3868a4077)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
autodiff at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/Enzyme.jl:504 [inlined]
#run_adjoint_plusfd#3 at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:24
run_adjoint_plusfd at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:5 [inlined]
run_adjoint_plusfd at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:5
unknown function (ip: 0x33ef501ab)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
do_call at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/interpreter.c:126
eval_body at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/interpreter.c:0
jl_interpret_toplevel_thunk at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/interpreter.c:775
jl_toplevel_eval_flex at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:934
jl_toplevel_eval_flex at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:877
ijl_toplevel_eval at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:943 [inlined]
ijl_toplevel_eval_in at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:985
eval at ./boot.jl:385 [inlined]
include_string at ./loading.jl:2070
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
_include at ./loading.jl:2130
include at ./Base.jl:495
jfptr_include_46486 at /Applications/Julia-1.10.app/Contents/Resources/julia/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
exec_options at ./client.jl:318
_start at ./client.jl:552
jfptr__start_82831 at /Applications/Julia-1.10.app/Contents/Resources/julia/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
true_main at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/jlapi.c:582
jl_repl_entrypoint at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/jlapi.c:731
Allocations: 146347978 (Pool: 146155869; Big: 192109); GC: 174
[1]    87818 segmentation fault  julia --project=. technical_paper.jl
swilliamson@CRIOS-A66253 ~/D/G/S/eddy-stresses> julia --project=. technical_paper.jl      :( main!?#
[ Info: Revolve: Number of checkpoints: 82
[ Info: Revolve: Number of steps: 6733
[ Info: Prediction:
[ Info: Forward steps   : 16629
[ Info: Overhead factor : 2.469775731471855

[88064] signal (11.2): Segmentation fault: 11
in expression starting at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:598
gc_mark_obj8 at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:0
gc_mark_outrefs at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2623 [inlined]
gc_mark_loop_serial_ at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2679
gc_mark_loop_serial at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2702
gc_mark_loop at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:2897 [inlined]
_jl_gc_collect at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:3221
ijl_gc_collect at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:3518
maybe_collect at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:937 [inlined]
jl_gc_pool_alloc_inner at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:1293 [inlined]
jl_gc_pool_alloc_noinline at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gc.c:1350 [inlined]
jl_gc_alloc_ at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/./julia_internal.h:477
_new_array_ at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/array.c:144
_new_array at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/array.c:198
ijl_alloc_array_2d at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/array.c:443
Array at ./boot.jl:479 [inlined]
Array at ./boot.jl:487 [inlined]
Array at ./boot.jl:494 [inlined]
similar at ./abstractarray.jl:874 [inlined]
similar at ./abstractarray.jl:873 [inlined]
similar at ./broadcast.jl:224 [inlined]
similar at ./broadcast.jl:223 [inlined]
copy at ./broadcast.jl:928 [inlined]
materialize at ./broadcast.jl:903 [inlined]
macro expansion at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper_integration.jl:203 [inlined]
#1 at /Users/swilliamson/.julia/packages/Checkpointing/RmOhf/src/Checkpointing.jl:181
checkpoint_struct_for at /Users/swilliamson/.julia/packages/Checkpointing/RmOhf/src/Checkpointing.jl:135
advection_coriolis! at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/ShallowWaters.jl/src/rhs.jl:0
length at ./essentials.jl:10 [inlined]
copyto! at ./array.jl:388 [inlined]
checkpointed_integration at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper_integration.jl:62 [inlined]
diffejulia_checkpointed_integration_4080wrap at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper_integration.jl:0
macro expansion at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/compiler.jl:8839 [inlined]
enzyme_call at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/compiler.jl:8405 [inlined]
CombinedAdjointThunk at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/compiler.jl:8178 [inlined]
autodiff at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/Enzyme.jl:491 [inlined]
autodiff at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/Enzyme.jl:537
unknown function (ip: 0x38577c077)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
autodiff at /Users/swilliamson/.julia/packages/Enzyme/Vjlrr/src/Enzyme.jl:504 [inlined]
#run_adjoint_plusfd#3 at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:24
run_adjoint_plusfd at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:5 [inlined]
run_adjoint_plusfd at /Users/swilliamson/Documents/GitHub/ShallowWaters_work/eddy-stresses/technical_paper.jl:5
unknown function (ip: 0x341b3417b)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
do_call at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/interpreter.c:126
eval_body at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/interpreter.c:0
jl_interpret_toplevel_thunk at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/interpreter.c:775
jl_toplevel_eval_flex at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:934
jl_toplevel_eval_flex at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:877
ijl_toplevel_eval at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:943 [inlined]
ijl_toplevel_eval_in at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/toplevel.c:985
eval at ./boot.jl:385 [inlined]
include_string at ./loading.jl:2070
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
_include at ./loading.jl:2130
include at ./Base.jl:495
jfptr_include_46486 at /Applications/Julia-1.10.app/Contents/Resources/julia/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
exec_options at ./client.jl:318
_start at ./client.jl:552
jfptr__start_82831 at /Applications/Julia-1.10.app/Contents/Resources/julia/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/gf.c:3076
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
true_main at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/jlapi.c:582
jl_repl_entrypoint at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-HL2F7YQ3XH.0/build/default-honeycrisp-HL2F7YQ3XH-0/julialang/julia-release-1-dot-10/src/jlapi.c:731
Allocations: 145697242 (Pool: 145505391; Big: 191851); GC: 141
[1]    88064 segmentation fault  julia --project=. technical_paper.jl

Notes on the bug:

I tried changing package versions, so attempted

  1. Enzyme v0.13.18 + Checkpointing#main
  2. Enzyme v0.13.8 + Checkpointing#main
  3. and Enzyme v0.13.8 + Checkpointing v0.9.6 ----- This combination is the one that I've been consistently using for ~the past month or so, because I knew it worked. The error above though was from me running on these versions.

Also, I'm currently on Julia 1.10.0.

@swilliamson7
Copy link
Collaborator Author

When you have a moment, I'd be really curious for you to run the code and see if it reproduces on your machine, because again I'm so confused about what's happened

@wsmoses
Copy link
Member

wsmoses commented Dec 3, 2024

oh, can you update to the latest Julia 1.10 -- in this case Julia 1.10.7 ?

@swilliamson7
Copy link
Collaborator Author

How do I update to a specific Julia? I only know how to go to the latest version, which I think is 1.11 (and still doesn't work with Enzyme?)

@swilliamson7
Copy link
Collaborator Author

Or should I try 1.11?

@wsmoses
Copy link
Member

wsmoses commented Dec 3, 2024

I think juliaup update 1.10 or something?

@swilliamson7
Copy link
Collaborator Author

juliaup is giving me issues (namely I installed and then it still says it's not installed), give me a sec

@swilliamson7
Copy link
Collaborator Author

Still hit it on Julia v1.10.7 (sorry for the delay, it took a really long time to compile packages and then it takes a while for the code to run/hit the error)

@swilliamson7
Copy link
Collaborator Author

In the repo I made a new branch called old-state that is one commit behind the one I reverted to, and the code still runs there, so it's something between this and the one you're on that causes the error maybe?

@wsmoses
Copy link
Member

wsmoses commented Dec 3, 2024

I made a branch on the repo sigseg trying to minimize it

@wsmoses
Copy link
Member

wsmoses commented Dec 4, 2024

wmoses@beast:~/git/eddy-stresses (sigseg) $ cat technical_paper.jl 
using Enzyme
using Checkpointing

Enzyme.API.printall!(true)
Enzyme.Compiler.DumpPostOpt[] = true

mutable struct MyPrognosticVars{T<:AbstractFloat}
    u::Array{T,2}           # u-velocity
    nu::Array{T,2}           # sea surface height / interface displacement
end

mutable struct MyModelSetup
    i::Int
    J::Float64
    halo::Int
    nt::Int
    Prog::MyPrognosticVars{Float32}
    h::Matrix{Float64}
    t::Int                              # SW: I believe this has something to do with Checkpointing, need to verify
end

function checkpointed_integration(S, scheme)
    h = S.h
    nu = S.Prog.nu
      @inbounds  h[1] = 2.7
    
    @checkpoint_struct scheme S for S.i = 1:S.nt
        Prog = S.Prog

        halo = S.halo

        # undo scaling as well
        @views ucut = S.Prog.u[halo+1:end-halo,halo+1:end-halo]
        ηcut = S.Prog.nu

        temp = MyPrognosticVars{Float64}(ucut, ηcut) 

        energy_lr = first(temp.u.^2)

        S.J += energy_lr
    end

    return
end

function mymodel_setup()
    nt = 6733
    S = MyModelSetup(0, 0.0, 2, nt,MyPrognosticVars{Float32}(ones(Float32, 131, 132),ones(Float32, 130, 130)),ones(Float64, 130, 130),0)

    return S

end
function run_adjoint_plusfd()

    S = mymodel_setup()


    dS = Enzyme.make_zero(S)
    snaps = Int(floor(sqrt(S.nt)))
    revolve = Revolve{MyModelSetup}(S.nt,
        snaps;
        gc=true,
    )

    autodiff(Enzyme.Reverse, checkpointed_integration, Duplicated(S, dS), Const(revolve))

end

diffs, enzyme_deriv, S, dS = run_adjoint_plusfd()

@wsmoses wsmoses added gc garbage collection segfault labels Dec 4, 2024
@wsmoses
Copy link
Member

wsmoses commented Dec 5, 2024

using Enzyme, LLVM
ctx = LLVM.Context()

activate(ctx)

mod = """
source_filename = "start"
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:10:11:12:13"
target triple = "x86_64-linux-gnu"

declare noalias nonnull {} addrspace(10)* @julia.gc_alloc_obj({}**, i64, {} addrspace(10)*) local_unnamed_addr #5

define internal fastcc nonnull {} addrspace(10)* @julia_MyPrognosticVars_161({} addrspace(10)* %v1, {} addrspace(10)* %v2) {
top:
  %newstruct = call noalias nonnull dereferenceable(16) {} addrspace(10)* @julia.gc_alloc_obj({}** null, i64 16, {} addrspace(10)* addrspacecast ({}* inttoptr (i64 129778359735376 to {}*) to {} addrspace(10)*)) #30
  %a31 = addrspacecast {} addrspace(10)* %newstruct to {} addrspace(10)* addrspace(11)*
  %a32 = getelementptr inbounds {} addrspace(10)*, {} addrspace(10)* addrspace(11)* %a31, i64 1
  store atomic {} addrspace(10)* %v1, {} addrspace(10)* addrspace(11)* %a31 release, align 8
  %a33 = addrspacecast {} addrspace(10)* %newstruct to i8 addrspace(11)*
  %a34 = getelementptr inbounds i8, i8 addrspace(11)* %a33, i64 8
  %a35 = bitcast i8 addrspace(11)* %a34 to {} addrspace(10)* addrspace(11)*
  store atomic {} addrspace(10)* %v2, {} addrspace(10)* addrspace(11)* %a35 release, align 8
  ret {} addrspace(10)* %newstruct
}

define {} addrspace(10)* @caller({} addrspace(10)* %v1, {} addrspace(10)* %v2) {
top:
  %ac = call fastcc nonnull {} addrspace(10)* @julia_MyPrognosticVars_161({} addrspace(10)* %v1, {} addrspace(10)* %v2)
  %b = addrspacecast {} addrspace(10)* %ac to {} addrspace(10)* addrspace(11)*
  %c = load atomic {} addrspace(10)*, {} addrspace(10)* addrspace(11)* %b unordered, align 8
  ret {} addrspace(10)* %c
}

attributes #5 = { inaccessiblememonly mustprogress nofree nounwind willreturn allockind("alloc,uninitialized") allocsize(1) "enzyme_no_escaping_allocation" "enzymejl_world"="31504" }
"""

mod = parse(LLVM.Module, mod)

Enzyme.Compiler.removeDeadArgs!(mod, Enzyme.Compiler.JIT.get_tm())
println(string(mod))

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
gc garbage collection segfault
Projects
None yet
Development

Successfully merging a pull request may close this issue.

2 participants