chore: Update vendored sources to duckdb/duckdb@4d24f5c

Merge pull request duckdb/duckdb#10607 from carlopi/secret_folder
Merge pull request duckdb/duckdb#10603 from carlopi/extensions_patches
Merge pull request duckdb/duckdb#10606 from Mytherin/duckdbmemoryinit
Merge pull request duckdb/duckdb#10604 from Mytherin/aggregatelimit
Merge pull request duckdb/duckdb#10592 from carlopi/error_message_lock
Merge pull request duckdb/duckdb#10600 from samansmink/secret-manager-fixes-2
Merge pull request duckdb/duckdb#10589 from pdet/csv_result_buffer_hold
Merge pull request duckdb/duckdb#10605 from Tishj/julia_remove_dataframe
Merge pull request duckdb/duckdb#10596 from pdet/null_padding_projection_finalize
Merge pull request duckdb/duckdb#10590 from Tmonster/investigate-cardinality-estimates-parquet
Merge pull request duckdb/duckdb#10598 from szarnyasg/add-support-options
Merge pull request duckdb/duckdb#10579 from Tishj/unittest_require_flag
Merge pull request duckdb/duckdb#10202 from Tishj/sorted_aggregate_function_small_vector_size
Merge pull request duckdb/duckdb#10553 from Tmonster/fuzzer_column_binding_issues
Merge pull request duckdb/duckdb#10513 from Maxxen/array-tupledata-fix
Merge pull request duckdb/duckdb#10584 from Mause/datetime-delta
Merge pull request duckdb/duckdb#10582 from carlopi/fix_extension_distribution
Merge pull request duckdb/duckdb#10580 from Maxxen/bump-spatial
Merge pull request duckdb/duckdb#10532 from Tishj/deduplicate_columns
Merge pull request duckdb/duckdb#10573 from carlopi/fixthreads
Merge pull request duckdb/duckdb#10572 from carlopi/noexit
Merge pull request duckdb/duckdb#10571 from Tishj/time_cast_heap_overflow
Merge pull request duckdb/duckdb#10569 from Tishj/fix_export_of_types
Merge pull request duckdb/duckdb#10568 from Tishj/pytest_does_not_throw_httpexception
Merge pull request duckdb/duckdb#10567 from Tishj/require_no_alternative_verify
Merge pull request duckdb/duckdb#10565 from Tishj/arrow_support_time_tz
Merge pull request duckdb/duckdb#10563 from Tishj/python_exception_bug
Merge pull request duckdb/duckdb#10559 from Mytherin/issue10528
Merge pull request duckdb/duckdb#10548 from ywelsch/yw/duckdb-only-server-threads
Merge pull request duckdb/duckdb#10564 from carlopi/fix01init
Merge pull request duckdb/duckdb#10560 from carlopi/rowsorts
Merge pull request duckdb/duckdb#10558 from pdet/bug_9952
Merge pull request duckdb/duckdb#10555 from carlopi/refactorcommit
Merge pull request duckdb/duckdb#10557 from pdet/reset-quoted-proj
Merge pull request duckdb/duckdb#10504 from Mytherin/queryid
Merge pull request duckdb/duckdb#10487 from felipecrv/upper_bound
Merge pull request duckdb/duckdb#10552 from pdet/fix_glob_partition_test
Merge pull request duckdb/duckdb#10551 from pdet/segfault-fix
Merge pull request duckdb/duckdb#10554 from Mytherin/benchmarkerrors
Merge pull request duckdb/duckdb#10485 from Tmonster/list_zip_no_args
Merge pull request duckdb/duckdb#10536 from motherduckdb/flo/add-order-to-tests
Merge pull request duckdb/duckdb#10549 from lnkuiper/radix_ht_reservation
Merge pull request duckdb/duckdb#10525 from Tishj/parallel_checkpoint_error_race
Merge pull request duckdb/duckdb#10541 from Mytherin/arrayextractunnamedstruct
Merge pull request duckdb/duckdb#10538 from Maxxen/copy-file-path-fix
Merge pull request duckdb/duckdb#10544 from hawkfish/tz2024a
Merge pull request duckdb/duckdb#10542 from carlopi/fixes
Merge pull request duckdb/duckdb#10408 from Mause/feature/jdbc-set-date
Merge pull request duckdb/duckdb#10531 from ywelsch/yw/client-context-in-attach
Merge pull request duckdb/duckdb#10537 from Mytherin/structcastoutoforder
Merge pull request duckdb/duckdb#10534 from Mytherin/applypatchesfix
Merge pull request duckdb/duckdb#10488 from Tishj/python_editable_build
Merge pull request duckdb/duckdb#10527 from Mytherin/linenoiseenterbehavior
Merge pull request duckdb/duckdb#10530 from pdet/bug_null_csv_ignore
Merge pull request duckdb/duckdb#10533 from Tishj/python_fixes_2024_2_8
Merge pull request duckdb/duckdb#10526 from lnkuiper/null_to_json_cast
Merge pull request duckdb/duckdb#10529 from philippmd/philippmd/errno-ioexception
Merge pull request duckdb/duckdb#10503 from lnkuiper/temporary_memory_manager_tweaks
Merge pull request duckdb/duckdb#10500 from Mytherin/issue910
Merge pull request duckdb/duckdb#10357 from kryonix/main
Merge pull request duckdb/duckdb#10521 from Mause/patch-4
Merge pull request duckdb/duckdb#10518 from motherduckdb/secrets-changes
Merge pull request duckdb/duckdb#10514 from Mytherin/exceptionstacktrace
Merge pull request duckdb/duckdb#10512 from Tishj/executor_race_condition
Merge pull request duckdb/duckdb#10510 from Tishj/gcc_is_stupid
Merge pull request duckdb/duckdb#10509 from pdet/csv_big_data
Merge pull request duckdb/duckdb#10505 from Mytherin/issue10363
Merge pull request duckdb/duckdb#10507 from carlopi/bundle-libraries
Merge pull request duckdb/duckdb#10459 from Maxxen/copy-tmp-ext
Merge pull request duckdb/duckdb#10483 from Maxxen/array-bugfix
Merge pull request duckdb/duckdb#10502 from Mytherin/autocompletefix
Merge pull request duckdb/duckdb#10499 from Mytherin/issue1164
Merge pull request duckdb/duckdb#10498 from Mytherin/disablecompletionrendering
Merge pull request duckdb/duckdb#10496 from Mytherin/memorymanagement
Merge pull request duckdb/duckdb#10492 from SChakravorti21/fix-empty-normalized-version
Merge pull request duckdb/duckdb#10484 from Flogex/extension-build-script
Merge pull request duckdb/duckdb#10493 from Mytherin/altcommands
Merge pull request duckdb/duckdb#10475 from rdavis120/main
Merge pull request duckdb/duckdb#10480 from Mytherin/autocompletedisplay
Merge pull request duckdb/duckdb#10473 from pdet/bug_10283
Merge pull request duckdb/duckdb#10479 from hannes/windowsarm
Merge pull request duckdb/duckdb#10471 from Maxxen/custom-index-fork
Merge pull request duckdb/duckdb#10474 from Mytherin/gzipfs
Merge pull request duckdb/duckdb#10472 from pdet/bug_10273
Merge pull request duckdb/duckdb#10470 from Mytherin/clearcontinuationmarkers
Merge pull request duckdb/duckdb#10467 from carlopi/fixes
Merge pull request duckdb/duckdb#10465 from pdet/type_misdetection
Merge pull request duckdb/duckdb#10462 from pdet/double_quoted_header
Merge pull request duckdb/duckdb#10461 from Mytherin/linenoisecleanup
Merge pull request duckdb/duckdb#10464 from Tishj/improve_julia_dev_readme
Merge pull request duckdb/duckdb#10245 from Tishj/parallel_streaming_query_result
Merge pull request duckdb/duckdb#10442 from peteraisher/bugfix/duckdb/duckdb#10441-validate-url
Merge pull request duckdb/duckdb#10449 from rdavis120/main
Merge pull request duckdb/duckdb#10457 from hawkfish/sortedagg-dry
Merge pull request duckdb/duckdb#10453 from Mytherin/highlight
Merge pull request duckdb/duckdb#10372 from samansmink/comment-on
Merge pull request duckdb/duckdb#10438 from pdet/empty_value_quote
Merge pull request duckdb/duckdb#10445 from Mause/enforce-pep517
Merge pull request duckdb/duckdb#10430 from samansmink/cloudflare-invalidation
Merge pull request duckdb/duckdb#10429 from samansmink/full-file-download-bugfix
Merge pull request duckdb/duckdb#10420 from Mytherin/pyexceptions
Merge pull request duckdb/duckdb#10425 from Mytherin/shellcontinuemarker
Merge pull request duckdb/duckdb#10447 from Mytherin/coverityscan
Merge pull request duckdb/duckdb#10347 from hawkfish/first-argmin
Merge pull request duckdb/duckdb#10433 from moshekaplan/patch-2
Merge pull request duckdb/duckdb#10424 from carlopi/remove_prints
Merge pull request duckdb/duckdb#10426 from Mytherin/nowpushdown
Merge pull request duckdb/duckdb#10376 from Maxxen/read-files
Merge pull request duckdb/duckdb#10419 from Tishj/skip_pyarrow_struct_pushdown_on_3_8
Merge pull request duckdb/duckdb#10421 from samansmink/secret-manager-refactor
Merge pull request duckdb/duckdb#10423 from Tishj/deprecate_patas_and_chimp
Merge pull request duckdb/duckdb#10413 from Tmonster/10406-fix-anti-joins-on-strings
Merge pull request duckdb/duckdb#10401 from maiadegraaf/c_api_additions
Merge pull request duckdb/duckdb#10411 from pdet/projection_pushdown_csv
Merge pull request duckdb/duckdb#10405 from szarnyasg/issue-template-nightly-build
Merge pull request duckdb/duckdb#10185 from samansmink/secret-manager-fixes-1
Merge pull request duckdb/duckdb#10418 from zhouzilong2020/apeace-llvm-warning
Merge pull request duckdb/duckdb#10410 from Mytherin/exceptionrework
Merge pull request duckdb/duckdb#10403 from szarnyasg/check-issue-formatting-fix
Merge pull request duckdb/duckdb#10390 from pdet/cast_csv
Merge pull request duckdb/duckdb#10398 from carlopi/fixplatformdetection
Merge pull request duckdb/duckdb#10380 from carlopi/extensioncasing
Merge pull request duckdb/duckdb#10392 from carlopi/noexpect
Merge pull request duckdb/duckdb#10399 from carlopi/fixpyci
Merge pull request duckdb/duckdb#10389 from Maxxen/fix-missing-checkpoint
Merge pull request duckdb/duckdb#10329 from carlopi/reworkplatform
Merge pull request duckdb/duckdb#10388 from Tishj/python_use_duckdb_cursor
Merge pull request duckdb/duckdb#10378 from Tmonster/add_empty_samples_to_table_statistics
Merge pull request duckdb/duckdb#10386 from kryonix/issue10260
Merge pull request duckdb/duckdb#10373 from taniabogatsch/additional-expansion-tests
Merge pull request duckdb/duckdb#10149 from szarnyasg/check-issues-for-code-formatting
Merge pull request duckdb/duckdb#9967 from taniabogatsch/block-size
Merge pull request duckdb/duckdb#10365 from Mause/feature/set-bytes
Merge pull request duckdb/duckdb#10359 from carlopi/changeextensiondirectory
Merge pull request duckdb/duckdb#10354 from carlopi/fixblack
Merge pull request duckdb/duckdb#10369 from gitccl/fix_set_list_size
Merge pull request duckdb/duckdb#10370 from carlopi/autocomplete_6030
Merge pull request duckdb/duckdb#10343 from Maxxen/feat/digit-separator
Merge pull request duckdb/duckdb#10358 from Mytherin/rcfix
Merge pull request duckdb/duckdb#10325 from Mytherin/artnullvalue
Merge pull request duckdb/duckdb#10353 from carlopi/fixpyformat
Merge pull request duckdb/duckdb#10344 from samansmink/pin-ccache-action-for-glibc
Merge pull request duckdb/duckdb#10340 from Tmonster/fix_plan_cost_regression_3
Merge pull request duckdb/duckdb#10318 from lnkuiper/distinct_threads
Merge pull request duckdb/duckdb#10310 from Mytherin/updatesqlitepostgres
Merge pull request duckdb/duckdb#10339 from Tishj/python_fix_conversion_of_negative_intervals
Merge pull request duckdb/duckdb#10337 from Mytherin/interruptexception
Merge pull request duckdb/duckdb#10338 from Mause/split-jdbc-tests
Merge pull request duckdb/duckdb#10335 from lkuffo/alp_negative_zero
Merge pull request duckdb/duckdb#9836 from Tishj/arrow_run_end_encoding
Merge pull request duckdb/duckdb#9656 from renevdzee/add-icon
Merge pull request duckdb/duckdb#10243 from rhizo-co/fix-sniff-csv
Merge pull request duckdb/duckdb#10314 from Maxxen/parquet-struct-filter-pushdown
Merge pull request duckdb/duckdb#10320 from Tmonster/fix_issue_999_2
Merge pull request duckdb/duckdb#10327 from Mytherin/rendercontrolcharacters
Merge pull request duckdb/duckdb#10328 from Mytherin/modenooutputresult
Merge pull request duckdb/duckdb#10321 from Mytherin/createsortkey
Merge pull request duckdb/duckdb#10317 from taniabogatsch/capi-fixes
Merge pull request duckdb/duckdb#10319 from xuke-hat/hugeint-mul
Merge pull request duckdb/duckdb#10300 from lnkuiper/json_spinlock
Merge pull request duckdb/duckdb#10306 from pdet/parallel_null_pad
Merge pull request duckdb/duckdb#10311 from Mytherin/issue10308
Merge pull request duckdb/duckdb#10295 from samansmink/bp-bug
Merge pull request duckdb/duckdb#10307 from pdet/quotes_benchmark
Merge pull request duckdb/duckdb#10301 from lnkuiper/table_macro_definition
Merge pull request duckdb/duckdb#10299 from lnkuiper/fts_cast
Merge pull request duckdb/duckdb#10258 from pdet/flipity_flip
Merge pull request duckdb/duckdb#10297 from v1gnesh/patch-1
Merge pull request duckdb/duckdb#10284 from Mytherin/progressbarrendering
Merge pull request duckdb/duckdb#10290 from Tishj/pandas_2_2_0
Merge pull request duckdb/duckdb#10291 from gitccl/fix_10180
Merge pull request duckdb/duckdb#9545 from david-cortes/pop_kurtosis
Merge pull request duckdb/duckdb#9635 from lkuffo/alp_compression
Merge pull request duckdb/duckdb#10208 from chrisiou/parse_path
Merge pull request duckdb/duckdb#10288 from Mytherin/issue10279
Merge pull request duckdb/duckdb#10274 from hawkfish/date-interval
Merge pull request duckdb/duckdb#10277 from Tishj/run_listtests_with_start_offset
Merge pull request duckdb/duckdb#10255 from Maxxen/bind-file-extension
Merge pull request duckdb/duckdb#10268 from Mytherin/issue10212
Merge pull request duckdb/duckdb#10267 from lnkuiper/alter_add_extension_type_column
Merge pull request duckdb/duckdb#10266 from lnkuiper/macro_stuff
Merge pull request duckdb/duckdb#10265 from lnkuiper/fix_10254
Merge pull request duckdb/duckdb#10259 from hawkfish/window-case
Merge pull request duckdb/duckdb#10226 from motherduckdb/user_agent_missed_bits
Merge pull request duckdb/duckdb#10248 from goldmedal/enhance-pg-proc
Merge pull request duckdb/duckdb#10262 from motherduckdb/dsdgen_interrupted
Merge pull request duckdb/duckdb#10256 from gitccl/rm-c-cast
Merge pull request duckdb/duckdb#10246 from Mytherin/copycolid
Merge pull request duckdb/duckdb#10209 from pdet/parallel_csv_state
Merge pull request duckdb/duckdb#10242 from Mytherin/orderbycolumnlifetime
Merge pull request duckdb/duckdb#10240 from Mytherin/unpivottypes
Merge pull request duckdb/duckdb#10235 from lnkuiper/hash_join_duplicate_columns
Merge pull request duckdb/duckdb#10236 from Mytherin/sequencerace
Merge pull request duckdb/duckdb#10238 from Mytherin/valueslistbinding
Merge pull request duckdb/duckdb#10234 from lnkuiper/file_size_bytes
Merge pull request duckdb/duckdb#10011 from osidekyle/add-batch-functionality
Merge pull request duckdb/duckdb#10150 from taniabogatsch/lambda-scoping
Merge pull request duckdb/duckdb#10207 from Mause/bugfix/capi-time-tz
Merge pull request duckdb/duckdb#10181 from hawkfish/window-first
Merge pull request duckdb/duckdb#10174 from Tishj/python_fetch_unnamed_struct_as_tuple
Merge pull request duckdb/duckdb#10204 from samansmink/oote-rtools-build
Merge pull request duckdb/duckdb#10203 from lnkuiper/hash_join_duplicate_columns
Merge pull request duckdb/duckdb#10229 from Mytherin/jsonfloat
Merge pull request duckdb/duckdb#10232 from szarnyasg/embedded-in-process
Merge pull request duckdb/duckdb#10217 from motherduckdb/fix-drop-if-exists
Merge pull request duckdb/duckdb#10195 from nickgerrets/uhugeint_compression
Merge pull request duckdb/duckdb#10188 from lnkuiper/json_many_files
Merge pull request duckdb/duckdb#9976 from Tmonster/cardinality-estimates-right-semi-anti
Merge pull request duckdb/duckdb#9971 from hawkfish/absorb
Merge pull request duckdb/duckdb#10123 from xuke-hat/merge-null
Merge pull request duckdb/duckdb#10222 from goldmedal/feature/add-info-schema-tables
Merge pull request duckdb/duckdb#10220 from gitccl/fix_bug
Merge pull request duckdb/duckdb#10215 from hawkfish/tz-2023d
Merge pull request duckdb/duckdb#10210 from Mytherin/querydescribe
Merge pull request duckdb/duckdb#10182 from motherduckdb/copy-ctr
Merge pull request duckdb/duckdb#10190 from Tmonster/issue_10046_even_better_solution
Merge pull request duckdb/duckdb#10199 from motherduckdb/allow-persistent
Merge pull request duckdb/duckdb#10206 from carlopi/fixtest
Merge pull request duckdb/duckdb#10032 from Maxxen/custom-index-fork
Merge pull request duckdb/duckdb#10197 from hawkfish/sem-test
Merge pull request duckdb/duckdb#10187 from carlopi/fixmap
Merge pull request duckdb/duckdb#10194 from Mytherin/issue10096
Merge pull request duckdb/duckdb#10186 from samansmink/fix-missing-move
Merge pull request duckdb/duckdb#10184 from Tmonster/crash-when-no-columns-in-returning
Merge pull request duckdb/duckdb#10183 from hannes/bug10148
Merge pull request duckdb/duckdb#9920 from lnkuiper/file_size_bytes
Merge pull request duckdb/duckdb#10175 from Mytherin/detachkeyword
Merge pull request duckdb/duckdb#10176 from Mytherin/issue10057
Merge pull request duckdb/duckdb#10107 from hawkfish/timetz-cmp
Merge pull request duckdb/duckdb#10157 from hawkfish/infinite-c
Merge pull request duckdb/duckdb#10147 from lnkuiper/concurrent_operator_memory_manager
Merge pull request duckdb/duckdb#9993 from lnkuiper/deliminator_stuff
Merge pull request duckdb/duckdb#10172 from gsauthof/ctrl-z
Merge pull request duckdb/duckdb#10164 from Mytherin/issue10141
Merge pull request duckdb/duckdb#10165 from Tishj/python_fix_timestamptz_issue
Merge pull request duckdb/duckdb#10163 from Mytherin/issue10074
Merge pull request duckdb/duckdb#10038 from Tmonster/remove_chunk_collection_from_reservoir_sampler
Merge pull request duckdb/duckdb#10151 from samansmink/speed-up-json-httpfs-reads
Merge pull request duckdb/duckdb#10162 from carlopi/fixsignaturemissigntruncate
Merge pull request duckdb/duckdb#10044 from chrisiou/regexp-escape-func
Merge pull request duckdb/duckdb#10117 from nickgerrets/hugeint_faster_math
Merge pull request duckdb/duckdb#10045 from hawkfish/sorted-agg
Merge pull request duckdb/duckdb#10110 from Tmonster/pushdown_filters_into_semi_and_anti_joins
Merge pull request duckdb/duckdb#10146 from ywelsch/yw/sequence-serialization
Merge pull request duckdb/duckdb#10160 from Mause/bugfix/py312-win
Merge pull request duckdb/duckdb#10159 from Mause/feature/py312-win
Merge pull request duckdb/duckdb#10152 from How-u-doing/non-cse
Merge pull request duckdb/duckdb#10156 from goldmedal/feature/add-session-func-pgcatalog
Merge pull request duckdb/duckdb#9883 from carlopi/extension-upload-wasm-nightly
Merge pull request duckdb/duckdb#9672 from TomBurdge/extend-pyspark
Merge pull request duckdb/duckdb#9957 from lnkuiper/first
Merge pull request duckdb/duckdb#10137 from samansmink/bump-manylinux-arm-python
Merge pull request duckdb/duckdb#10145 from Mause/bugfix/jemalloc-on-windows
Merge pull request duckdb/duckdb#10132 from yiyuanliu/lyy/fix-python-stubs
Merge pull request duckdb/duckdb#10144 from Mause/feature/python-312
Merge pull request duckdb/duckdb#10142 from hawkfish/list-cast
Merge pull request duckdb/duckdb#10135 from Mytherin/defaultpreparedstatement
Merge pull request duckdb/duckdb#10134 from Mytherin/unnamedstructtostring
Merge pull request duckdb/duckdb#10126 from Mytherin/walchecksum
Merge pull request duckdb/duckdb#10130 from Mytherin/querylocation
Merge pull request duckdb/duckdb#10115 from Mytherin/stringliteralbinding
Merge pull request duckdb/duckdb#10086 from taniabogatsch/lazy-wal
Merge pull request duckdb/duckdb#10119 from szarnyasg/streamline-issues-with-prs-2
Merge pull request duckdb/duckdb#10118 from Mause/bugfix/blind-catches
Merge pull request duckdb/duckdb#10113 from yiyuanliu/lyy/fix-parquet-thread
Merge pull request duckdb/duckdb#10116 from szarnyasg/streamline-issues-with-prs
Merge pull request duckdb/duckdb#10111 from szarnyasg/add-original-issue-number-to-mirror-issue
Merge pull request duckdb/duckdb#10055 from hawkfish/icu-serialize
Merge pull request duckdb/duckdb#10102 from samansmink/bump-iceberg
Merge pull request duckdb/duckdb#10104 from hannes/nostdlib-cheader
Merge pull request duckdb/duckdb#10103 from szarnyasg/update-license-year-to-2024
Merge pull request duckdb/duckdb#9918 from tom-s-powell/tp/s3-subpath
Merge pull request duckdb/duckdb#9909 from maiadegraaf/list_reduce
Merge pull request duckdb/duckdb#10063 from samansmink/deploy-script-refactor
Merge pull request duckdb/duckdb#9499 from braintrustdata/json-dot-syntax
Merge pull request duckdb/duckdb#10082 from ywgrit/main
Merge pull request duckdb/duckdb#10083 from ywelsch/yw/pending-query-cleanup-internal
Merge pull request duckdb/duckdb#10097 from Mause/bugfix/capi-union
Merge pull request duckdb/duckdb#9544 from jkub/working_memory
Merge pull request duckdb/duckdb#10072 from motherduckdb/patch/missing-header-dependency
Merge pull request duckdb/duckdb#10061 from Mytherin/issue10058
Merge pull request duckdb/duckdb#10054 from hawkfish/distinct-gcc
Merge pull request duckdb/duckdb#9989 from lnkuiper/issue9718
Merge pull request duckdb/duckdb#10042 from samansmink/stored-credentials
Merge pull request duckdb/duckdb#10039 from mcmcgrath13/patch-1
Merge pull request duckdb/duckdb#10049 from Mause/python-pkg-version
Merge pull request duckdb/duckdb#8635 from nickgerrets/uhugeint
Merge pull request duckdb/duckdb#10036 from motherduckdb/fix-test
Merge pull request duckdb/duckdb#9754 from hawkfish/window-distinct
Merge pull request duckdb/duckdb#10025 from chenzl25/fix_array_subquery
Merge pull request duckdb/duckdb#9990 from lnkuiper/issue9380
Merge pull request duckdb/duckdb#9954 from hawkfish/interval-seconds
Merge pull request duckdb/duckdb#10026 from Mytherin/lldbfix
Merge pull request duckdb/duckdb#9968 from Tishj/stream_query_owning_result
Merge pull request duckdb/duckdb#10013 from yiyuanliu/lyy/fix-parquet-progress
Merge pull request duckdb/duckdb#10014 from sundy-li/ceil-fix
Merge pull request duckdb/duckdb#9988 from hannes/lockerrormsg
Merge pull request duckdb/duckdb#10004 from taniabogatsch/invalid-ptr-analyze
Merge pull request duckdb/duckdb#10006 from szarnyasg/increase-cli-history-size
Merge pull request duckdb/duckdb#10015 from Mytherin/multilinemode

src/duckdb/extension/parquet/column_reader.cpp

@@ -20,7 +20,6 @@
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/types/bit.hpp"
 #include "duckdb/common/types/blob.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
 #endif
 
 namespace duckdb {
@@ -181,11 +180,7 @@ idx_t ColumnReader::GroupRowsAvailable() {
 }
 
 unique_ptr<BaseStatistics> ColumnReader::Stats(idx_t row_group_idx_p, const vector<ColumnChunk> &columns) {
-	if (Type().id() == LogicalTypeId::LIST || Type().id() == LogicalTypeId::STRUCT ||
-	    Type().id() == LogicalTypeId::MAP || Type().id() == LogicalTypeId::ARRAY) {
-		return nullptr;
-	}
-	return ParquetStatisticsUtils::TransformColumnStatistics(Schema(), Type(), columns[file_idx]);
+	return ParquetStatisticsUtils::TransformColumnStatistics(*this, columns);
 }
 
 void ColumnReader::Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, idx_t num_values, // NOLINT

src/duckdb/extension/parquet/column_writer.cpp

@@ -13,9 +13,9 @@
 #include "duckdb/common/serializer/memory_stream.hpp"
 #include "duckdb/common/serializer/write_stream.hpp"
 #include "duckdb/common/string_map_set.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/common/types/date.hpp"
 #include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/common/types/uhugeint.hpp"
 #include "duckdb/common/types/string_heap.hpp"
 #include "duckdb/common/types/time.hpp"
 #include "duckdb/common/types/timestamp.hpp"
@@ -824,6 +824,22 @@ struct ParquetHugeintOperator {
 	}
 };
 
+struct ParquetUhugeintOperator {
+	template <class SRC, class TGT>
+	static TGT Operation(SRC input) {
+		return Uhugeint::Cast<double>(input);
+	}
+
+	template <class SRC, class TGT>
+	static unique_ptr<ColumnWriterStatistics> InitializeStats() {
+		return make_uniq<ColumnWriterStatistics>();
+	}
+
+	template <class SRC, class TGT>
+	static void HandleStats(ColumnWriterStatistics *stats, SRC source_value, TGT target_value) {
+	}
+};
+
 template <class SRC, class TGT, class OP = ParquetCastOperator>
 static void TemplatedWritePlain(Vector &col, ColumnWriterStatistics *stats, idx_t chunk_start, idx_t chunk_end,
                                 ValidityMask &mask, WriteStream &ser) {
@@ -1997,6 +2013,9 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 	case LogicalTypeId::HUGEINT:
 		return make_uniq<StandardColumnWriter<hugeint_t, double, ParquetHugeintOperator>>(
 		    writer, schema_idx, std::move(schema_path), max_repeat, max_define, can_have_nulls);
+	case LogicalTypeId::UHUGEINT:
+		return make_uniq<StandardColumnWriter<uhugeint_t, double, ParquetUhugeintOperator>>(
+		    writer, schema_idx, std::move(schema_path), max_repeat, max_define, can_have_nulls);
 	case LogicalTypeId::TIMESTAMP_NS:
 		return make_uniq<StandardColumnWriter<int64_t, int64_t, ParquetTimestampNSOperator>>(
 		    writer, schema_idx, std::move(schema_path), max_repeat, max_define, can_have_nulls);

src/duckdb/extension/parquet/include/column_reader.hpp

@@ -19,7 +19,6 @@
 #ifndef DUCKDB_AMALGAMATION
 
 #include "duckdb/common/operator/cast_operators.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/types/vector_cache.hpp"

src/duckdb/extension/parquet/include/decode_utils.hpp

@@ -7,8 +7,8 @@ class ParquetDecodeUtils {
 
 public:
 	template <class T>
-	static T ZigzagToInt(const T n) {
-		return (n >> 1) ^ -(n & 1);
+	static T ZigzagToInt(const uint64_t n) {
+		return T(n >> 1) ^ -T(n & 1);
 	}
 
 	static const uint64_t BITPACK_MASKS[];

src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp

@@ -10,7 +10,7 @@ class DbpDecoder {
 		block_value_count = ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_);
 		miniblocks_per_block = ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_);
 		total_value_count = ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_);
-		start_value = ParquetDecodeUtils::ZigzagToInt(ParquetDecodeUtils::VarintDecode<int64_t>(buffer_));
+		start_value = ParquetDecodeUtils::ZigzagToInt<int64_t>(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_));
 
 		// some derivatives
 		D_ASSERT(miniblocks_per_block > 0);
@@ -61,7 +61,8 @@ class DbpDecoder {
 				if (bitpack_pos > 0) {       // have to eat the leftovers if any
 					buffer_.inc(1);
 				}
-				min_delta = ParquetDecodeUtils::ZigzagToInt(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_));
+				min_delta =
+				    ParquetDecodeUtils::ZigzagToInt<int64_t>(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_));
 				for (idx_t miniblock_idx = 0; miniblock_idx < miniblocks_per_block; miniblock_idx++) {
 					miniblock_bit_widths[miniblock_idx] = buffer_.read<uint8_t>();
 					// TODO what happens if width is 0?
@@ -80,7 +81,7 @@ class DbpDecoder {
 			ParquetDecodeUtils::BitUnpack<T>(buffer_, bitpack_pos, &values[value_offset], read_now,
 			                                 miniblock_bit_widths[miniblock_offset]);
 			for (idx_t i = value_offset; i < value_offset + read_now; i++) {
-				values[i] = ((i == 0) ? start_value : values[i - 1]) + min_delta + values[i];
+				values[i] = T(uint64_t((i == 0) ? start_value : values[i - 1]) + min_delta + uint64_t(values[i]));
 			}
 			value_offset += read_now;
 			values_left_in_miniblock -= read_now;

src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp

@@ -17,18 +17,26 @@ class ParquetDecimalUtils {
 public:
 	template <class PHYSICAL_TYPE>
 	static PHYSICAL_TYPE ReadDecimalValue(const_data_ptr_t pointer, idx_t size,
-	                                      const duckdb_parquet::format::SchemaElement &schema_ele) {
-		D_ASSERT(size <= sizeof(PHYSICAL_TYPE));
+	                                      const duckdb_parquet::format::SchemaElement &) {
 		PHYSICAL_TYPE res = 0;
 
 		auto res_ptr = (uint8_t *)&res;
 		bool positive = (*pointer & 0x80) == 0;
 
 		// numbers are stored as two's complement so some muckery is required
-		for (idx_t i = 0; i < size; i++) {
+		for (idx_t i = 0; i < MinValue<idx_t>(size, sizeof(PHYSICAL_TYPE)); i++) {
 			auto byte = *(pointer + (size - i - 1));
 			res_ptr[i] = positive ? byte : byte ^ 0xFF;
 		}
+		// Verify that there are only 0s here
+		if (size > sizeof(PHYSICAL_TYPE)) {
+			for (idx_t i = sizeof(PHYSICAL_TYPE); i < size; i++) {
+				auto byte = *(pointer + (size - i - 1));
+				if (byte != 0) {
+					throw InvalidInputException("Invalid decimal encoding in Parquet file");
+				}
+			}
+		}
 		if (!positive) {
 			res += 1;
 			return -res;

src/duckdb/extension/parquet/include/parquet_statistics.hpp

@@ -12,11 +12,12 @@ using duckdb_parquet::format::ColumnChunk;
 using duckdb_parquet::format::SchemaElement;
 
 struct LogicalType;
+class ColumnReader;
 
 struct ParquetStatisticsUtils {
 
-	static unique_ptr<BaseStatistics> TransformColumnStatistics(const SchemaElement &s_ele, const LogicalType &type,
-	                                                            const ColumnChunk &column_chunk);
+	static unique_ptr<BaseStatistics> TransformColumnStatistics(const ColumnReader &reader,
+	                                                            const vector<ColumnChunk> &columns);
 
 	static Value ConvertValue(const LogicalType &type, const duckdb_parquet::format::SchemaElement &schema_ele,
 	                          const std::string &stats);

src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -87,6 +87,10 @@ class ParquetWriter {
 	BufferedFileWriter &GetWriter() {
 		return *writer;
 	}
+	idx_t FileSize() {
+		lock_guard<mutex> glock(lock);
+		return writer->total_written;
+	}
 
 	static CopyTypeSupport TypeIsSupported(const LogicalType &type);
 

src/duckdb/extension/parquet/parquet_extension.cpp

@@ -25,7 +25,6 @@
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/common/serializer/serializer.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/function/pragma_function.hpp"
 #include "duckdb/function/table_function.hpp"
@@ -491,10 +490,9 @@ class ParquetScanFunction {
 		if (bind_data.initial_file_cardinality == 0) {
 			return (100.0 * (bind_data.cur_file + 1)) / bind_data.files.size();
 		}
-		auto percentage = (bind_data.chunk_count * STANDARD_VECTOR_SIZE * 100.0 / bind_data.initial_file_cardinality) /
-		                  bind_data.files.size();
-		percentage += 100.0 * bind_data.cur_file / bind_data.files.size();
-		return percentage;
+		auto percentage = std::min(
+		    100.0, (bind_data.chunk_count * STANDARD_VECTOR_SIZE * 100.0 / bind_data.initial_file_cardinality));
+		return (percentage + 100.0 * bind_data.cur_file) / bind_data.files.size();
 	}
 
 	static unique_ptr<LocalTableFunctionState>
@@ -630,7 +628,10 @@ class ParquetScanFunction {
 
 	static idx_t ParquetScanMaxThreads(ClientContext &context, const FunctionData *bind_data) {
 		auto &data = bind_data->Cast<ParquetReadBindData>();
-		return data.initial_file_row_groups * data.files.size();
+		if (data.files.size() > 1) {
+			return TaskScheduler::GetScheduler(context).NumberOfThreads();
+		}
+		return MaxValue(data.initial_file_row_groups, (idx_t)1);
 	}
 
 	// This function looks for the next available row group. If not available, it will open files from bind_data.files
@@ -910,12 +911,12 @@ static void GetFieldIDs(const Value &field_ids_value, ChildFieldIDs &field_ids,
 	}
 }
 
-unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, const CopyInfo &info, const vector<string> &names,
-                                          const vector<LogicalType> &sql_types) {
+unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBindInput &input,
+                                          const vector<string> &names, const vector<LogicalType> &sql_types) {
 	D_ASSERT(names.size() == sql_types.size());
 	bool row_group_size_bytes_set = false;
 	auto bind_data = make_uniq<ParquetWriteBindData>();
-	for (auto &option : info.options) {
+	for (auto &option : input.info.options) {
 		const auto loption = StringUtil::Lower(option.first);
 		if (option.second.size() != 1) {
 			// All parquet write options require exactly one argument
@@ -986,7 +987,13 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, const CopyInfo
 			throw NotImplementedException("Unrecognized option for PARQUET: %s", option.first.c_str());
 		}
 	}
-	if (!row_group_size_bytes_set) {
+	if (row_group_size_bytes_set) {
+		if (DBConfig::GetConfig(context).options.preserve_insertion_order) {
+			throw BinderException("ROW_GROUP_SIZE_BYTES does not work while preserving insertion order. Use \"SET "
+			                      "preserve_insertion_order=false;\" to disable preserving insertion order.");
+		}
+	} else {
+		// We always set a max row group size bytes so we don't use too much memory
 		bind_data->row_group_size_bytes = bind_data->row_group_size * ParquetWriteBindData::BYTES_PER_ROW;
 	}
 
@@ -1179,6 +1186,14 @@ idx_t ParquetWriteDesiredBatchSize(ClientContext &context, FunctionData &bind_da
 	return bind_data.row_group_size;
 }
 
+//===--------------------------------------------------------------------===//
+// Current File Size
+//===--------------------------------------------------------------------===//
+idx_t ParquetWriteFileSize(GlobalFunctionData &gstate) {
+	auto &global_state = gstate.Cast<ParquetWriteGlobalState>();
+	return global_state.writer->FileSize();
+}
+
 //===--------------------------------------------------------------------===//
 // Scan Replacement
 //===--------------------------------------------------------------------===//
@@ -1240,6 +1255,7 @@ void ParquetExtension::Load(DuckDB &db) {
 	function.prepare_batch = ParquetWritePrepareBatch;
 	function.flush_batch = ParquetWriteFlushBatch;
 	function.desired_batch_size = ParquetWriteDesiredBatchSize;
+	function.file_size_bytes = ParquetWriteFileSize;
 	function.serialize = ParquetCopySerialize;
 	function.deserialize = ParquetCopyDeserialize;
 	function.supports_type = ParquetWriter::TypeIsSupported;

src/duckdb/extension/parquet/parquet_reader.cpp

@@ -25,6 +25,7 @@
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/planner/filter/null_filter.hpp"
+#include "duckdb/planner/filter/struct_filter.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/object_cache.hpp"
 #endif
@@ -874,6 +875,11 @@ static void ApplyFilter(Vector &v, TableFilter &filter, parquet_filter_t &filter
 	case TableFilterType::IS_NULL:
 		FilterIsNull(v, filter_mask, count);
 		break;
+	case TableFilterType::STRUCT_EXTRACT: {
+		auto &struct_filter = filter.Cast<StructFilter>();
+		auto &child = StructVector::GetEntries(v)[struct_filter.child_idx];
+		ApplyFilter(*child, *struct_filter.child_filter, filter_mask, count);
+	} break;
 	default:
 		D_ASSERT(0);
 		break;