From b55fcc6d8291b8b538ab1b149c58c94d063a56b8 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 05:15:34 +0200 Subject: [PATCH 01/22] remove comment - see description for details Using cmake here is good as it mirrors the actual build so we fail fast. Replacing the test compile with the cmake check would require cmake even if we use pre-compiled binary which we don't want. --- r/tools/nixlibs.R | 5 ----- 1 file changed, 5 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 60deca05cd172..c785cc5f75fa1 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -736,11 +736,6 @@ with_cloud_support <- function(env_var_list) { cat("**** ", start_msg, " support ", msg, "; building with ", off_flags, "\n") } - # Check the features - # This duplicates what we do with the test program above when we check - # capabilities for using binaries. We could consider consolidating this - # logic, though these use cmake in order to match exactly what we do in the - # libarrow build, and maybe that increases the fidelity. if (!cmake_find_package("CURL", NULL, env_var_list)) { # curl on macos should be installed, so no need to alter this for macos # TODO: check for apt/yum/etc. and message the right thing? From 7d0091f3239c20e32469c4ec83d0ab0b63845afa Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 05:25:52 +0200 Subject: [PATCH 02/22] default to build with cloud support if libs are available --- r/tools/nixlibs.R | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index c785cc5f75fa1..a3cfae4ffe54f 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -720,8 +720,14 @@ is_feature_requested <- function(env_varname, default = env_is("LIBARROW_MINIMAL } with_cloud_support <- function(env_var_list) { - arrow_s3 <- is_feature_requested("ARROW_S3") - arrow_gcs <- is_feature_requested("ARROW_GCS") + have_cloud_libs <- cmake_find_package("CURL", NULL, env_var_list) && + cmake_find_package("OpenSSL", "1.0.2", env_var_list) + # Build with cloud support when libs are available unless user explicitly turns them off + # Only show warnings if user explicitly requests cloud support. + arrow_s3 <- !env_is("ARROW_S3", "off") && have_cloud_libs || + is_feature_requested("ARROW_S3") + arrow_gcs <- !env_is("ARROW_GCS", "off") && have_cloud_libs || + is_feature_requested("ARROW_GCS") if (arrow_s3 || arrow_gcs) { # User wants S3 or GCS support. # Make sure that we have curl and openssl system libs From 9804981d30e61671c8c39d344ece301aa34e2454 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 05:26:42 +0200 Subject: [PATCH 03/22] update opennssl search logic to also use pkg-config --- r/configure | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/r/configure b/r/configure index ced3b40a68901..240a45184829b 100755 --- a/r/configure +++ b/r/configure @@ -121,16 +121,25 @@ else ARROW_USE_PKG_CONFIG="false" fi -# find openssl on macos. macOS ships with libressl. openssl is installable -# with brew, but it is generally not linked. We can over-ride this and find -# openssl but setting OPENSSL_ROOT_DIR (which cmake will pick up later in -# the installation process). FWIW, arrow's cmake process uses this -# same process to find openssl, but doing it now allows us to catch it in -# nixlibs.R and throw a nicer error. +## Find openssl +# Arrow's cmake process uses this same process to find openssl, +# but doing it now allows us to catch it in +# nixlibs.R and activate S3 and GCS support for the source build. + +# macOS ships with libressl. openssl is installable with brew, but it is +# generally not linked. We can over-ride this and find +# openssl by setting OPENSSL_ROOT_DIR (which cmake will pick up later in +# the installation process). if [ "${OPENSSL_ROOT_DIR}" = "" ] && brew --prefix openssl >/dev/null 2>&1; then export OPENSSL_ROOT_DIR="`brew --prefix openssl`" export PKG_CONFIG_PATH="${OPENSSL_ROOT_DIR}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" fi +# Look for openssl with pkg-config for non-brew sources(e.g. CRAN) and Linux +if [ "${OPENSSL_ROOT_DIR}" = ""] && [ "${PKG_CONFIG_AVAILABLE}" = "true" ]; then + if ${PKG_CONFIG} --exists openssl; then + export OPENSSL_ROOT_DIR="`${PKG_CONFIG} --variable=prefix openssl`" + fi +fi ############# # Functions # From 107fe7e5c36221211afd0110c086c7b8fe0550ad Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 05:47:33 +0200 Subject: [PATCH 04/22] test macos source build --- dev/tasks/r/github.packages.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 760e3b6da448d..c3f5e887a5f51 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -375,29 +375,38 @@ jobs: read_parquet(system.file("v0.7.1.parquet", package = "arrow")) print(arrow_info()) - #TODO test macos source build? - test-linux-source: + test-source: needs: source - name: Test linux source build - runs-on: ubuntu-latest + name: Test {{ '${{ matrix.platform.name }}' }} source build + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + strategy: + fail-fast: false + matrix: + platform: + - {runs_on: "ubuntu-latest", name: "Linux"} + - {runs_on: "macos-latest", name: "macOS"} steps: - name: Install R uses: r-lib/actions/setup-r@v2 - with: - install-r: false {{ macros.github_setup_local_r_repo(false, false)|indent }} {{ macros.github_checkout_arrow()|indent }} - name: Install sccache + if: matrix.platform.name == 'Linux' shell: bash run: | arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - - name: Install R package system dependencies + - name: Install R package system dependencies (Linux) + if: matrix.platform.name == 'Linux' run: | sudo arrow/ci/scripts/r_install_system_dependencies.sh env: ARROW_GCS: "ON" ARROW_S3: "ON" ARROW_SOURCE_HOME: arrow + - name: Install R package system dependencies (macOS) + run: | + brew sccache + brew bundle --file arrow/cpp/Brewfile - name: Remove arrow/ run: | rm -rf arrow/ From 845e6070a0441487f7d4208d017c64c41b503581 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 05:52:05 +0200 Subject: [PATCH 05/22] revert - disable m1 package build --- dev/tasks/r/github.packages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index c3f5e887a5f51..c27a166eddc23 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -197,7 +197,7 @@ jobs: platform: - { runs_on: 'windows-latest', name: "Windows"} - { runs_on: ["self-hosted", "macos-10.13"], name: "macOS High Sierra"} - - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], name: "macOS Big Sur" } + # - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], name: "macOS Big Sur" } r_version: - { rtools: "{{ macros.r_release.rt }}", r: "{{ macros.r_release.ver }}" } - { rtools: "{{ macros.r_oldrel.rt }}", r: "{{ macros.r_oldrel.ver }}" } @@ -378,11 +378,11 @@ jobs: test-source: needs: source name: Test {{ '${{ matrix.platform.name }}' }} source build - runs-on: {{ '${{ matrix.platform.runs_on }}' }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} strategy: fail-fast: false matrix: - platform: + platform: - {runs_on: "ubuntu-latest", name: "Linux"} - {runs_on: "macos-latest", name: "macOS"} steps: From 08b463f2a1d3ac90725651ca53abde7eecd12aed Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 06:06:13 +0200 Subject: [PATCH 06/22] fix needs --- dev/tasks/r/github.packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index c27a166eddc23..e10a9212ea920 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -435,7 +435,7 @@ jobs: upload-binaries: # Only upload binaries if all tests pass. - needs: [r-packages, test-linux-source, test-linux-binary] + needs: [r-packages, test-source, test-linux-binary] name: Upload artifacts runs-on: ubuntu-latest steps: From 5c304610dffee7e48f8e6b7131e36108cdd95708 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 06:27:28 +0200 Subject: [PATCH 07/22] fix typo --- dev/tasks/r/github.packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index e10a9212ea920..930eda3c3b9a3 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -405,7 +405,7 @@ jobs: ARROW_SOURCE_HOME: arrow - name: Install R package system dependencies (macOS) run: | - brew sccache + brew install sccache brew bundle --file arrow/cpp/Brewfile - name: Remove arrow/ run: | From 115968fe51745ff08349ea54cc583670a98362a6 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 06:33:52 +0200 Subject: [PATCH 08/22] don't install mac deps on linux --- dev/tasks/r/github.packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 930eda3c3b9a3..75042bca6c55f 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -404,6 +404,7 @@ jobs: ARROW_S3: "ON" ARROW_SOURCE_HOME: arrow - name: Install R package system dependencies (macOS) + if: matrix.platform.name != 'Linux' run: | brew install sccache brew bundle --file arrow/cpp/Brewfile From 10ea3a31a863f9bcc2f8a7c9c20e082d344d927a Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 06:37:22 +0200 Subject: [PATCH 09/22] force source build in install.packages --- dev/tasks/macros.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 2a6cbe3e61109..3bcff2f63541f 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -278,6 +278,7 @@ on: "arrow", repos = c(getOption("arrow.dev_repo"), getOption("repos")), verbose = TRUE, + type = "source", INSTALL_opts = "--build" ) From 0b5bd3e29f58fc31f085158f0cae9c446b536b80 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 07:45:23 +0200 Subject: [PATCH 10/22] fix configure if --- r/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/configure b/r/configure index 240a45184829b..addf7b59c7fbf 100755 --- a/r/configure +++ b/r/configure @@ -135,7 +135,7 @@ if [ "${OPENSSL_ROOT_DIR}" = "" ] && brew --prefix openssl >/dev/null 2>&1; then export PKG_CONFIG_PATH="${OPENSSL_ROOT_DIR}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" fi # Look for openssl with pkg-config for non-brew sources(e.g. CRAN) and Linux -if [ "${OPENSSL_ROOT_DIR}" = ""] && [ "${PKG_CONFIG_AVAILABLE}" = "true" ]; then +if [ "${OPENSSL_ROOT_DIR}" = "" -a "${PKG_CONFIG_AVAILABLE}" = "true" ]; then if ${PKG_CONFIG} --exists openssl; then export OPENSSL_ROOT_DIR="`${PKG_CONFIG} --variable=prefix openssl`" fi From 9d4018473900bf9127c6f52fbd18ab2b0d47bf35 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 07:46:01 +0200 Subject: [PATCH 11/22] install crc32c from brew as a workaround for missing symbol when bundled --- dev/tasks/r/github.packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 75042bca6c55f..bbbf740c6ef6d 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -406,7 +406,7 @@ jobs: - name: Install R package system dependencies (macOS) if: matrix.platform.name != 'Linux' run: | - brew install sccache + brew install sccache crc32c brew bundle --file arrow/cpp/Brewfile - name: Remove arrow/ run: | From 4ddbeaf8d5df518f9a5d68a31021fc1b55eaa248 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 17:11:54 +0200 Subject: [PATCH 12/22] add crc32c to static bundled libs --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a6b5db382421a..68c2960aab353 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4086,6 +4086,7 @@ macro(build_crc32c_once) ${_CRC32C_STATIC_LIBRARY}) target_include_directories(Crc32c::crc32c BEFORE INTERFACE "${CRC32C_INCLUDE_DIR}") add_dependencies(Crc32c::crc32c crc32c_ep) + list(APPEND ARROW_BUNDLED_STATIC_LIBS Crc32c::crc32c) endif() endmacro() From f878645b53244c9f51728311b23c31eb32643f1c Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 18:01:14 +0200 Subject: [PATCH 13/22] Fix linux deps install sudo was run in clean env as `-E` was missing, so nothing was ever installed. As we don't need test bench etc we can just call apt directly. --- dev/tasks/r/github.packages.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index bbbf740c6ef6d..d21eaa0d6b9d4 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -397,12 +397,7 @@ jobs: arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - name: Install R package system dependencies (Linux) if: matrix.platform.name == 'Linux' - run: | - sudo arrow/ci/scripts/r_install_system_dependencies.sh - env: - ARROW_GCS: "ON" - ARROW_S3: "ON" - ARROW_SOURCE_HOME: arrow + run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Install R package system dependencies (macOS) if: matrix.platform.name != 'Linux' run: | From 647f84c47747baad4c63171cbe0a66a5b55f1dce Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 20:17:28 +0200 Subject: [PATCH 14/22] Revert "revert - disable m1 package build" This reverts commit 845e6070a0441487f7d4208d017c64c41b503581. --- dev/tasks/r/github.packages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index d21eaa0d6b9d4..2a71246c792b2 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -197,7 +197,7 @@ jobs: platform: - { runs_on: 'windows-latest', name: "Windows"} - { runs_on: ["self-hosted", "macos-10.13"], name: "macOS High Sierra"} - # - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], name: "macOS Big Sur" } + - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], name: "macOS Big Sur" } r_version: - { rtools: "{{ macros.r_release.rt }}", r: "{{ macros.r_release.ver }}" } - { rtools: "{{ macros.r_oldrel.rt }}", r: "{{ macros.r_oldrel.ver }}" } @@ -378,11 +378,11 @@ jobs: test-source: needs: source name: Test {{ '${{ matrix.platform.name }}' }} source build - runs-on: {{ '${{ matrix.platform.runs_on }}' }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} strategy: fail-fast: false matrix: - platform: + platform: - {runs_on: "ubuntu-latest", name: "Linux"} - {runs_on: "macos-latest", name: "macOS"} steps: From ab8c7f86c99194d3e45a69b43fd362b1e98b3355 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 20:20:13 +0200 Subject: [PATCH 15/22] use self-hosted 10.13 run to test macos source build --- dev/tasks/r/github.packages.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 2a71246c792b2..54837d14d9a14 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -384,7 +384,7 @@ jobs: matrix: platform: - {runs_on: "ubuntu-latest", name: "Linux"} - - {runs_on: "macos-latest", name: "macOS"} + - {runs_on: ["self-hosted", "macos-10.13"] , name: "macOS"} steps: - name: Install R uses: r-lib/actions/setup-r@v2 @@ -398,11 +398,6 @@ jobs: - name: Install R package system dependencies (Linux) if: matrix.platform.name == 'Linux' run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev - - name: Install R package system dependencies (macOS) - if: matrix.platform.name != 'Linux' - run: | - brew install sccache crc32c - brew bundle --file arrow/cpp/Brewfile - name: Remove arrow/ run: | rm -rf arrow/ From 36537e250aa92fb47a4d6239bdd5cef26a99941f Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 20:40:23 +0200 Subject: [PATCH 16/22] don't install R on self-hosted --- dev/tasks/r/github.packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 54837d14d9a14..ccd8e1e657e48 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -387,6 +387,7 @@ jobs: - {runs_on: ["self-hosted", "macos-10.13"] , name: "macOS"} steps: - name: Install R + if: matrix.platform.name == 'Linux' uses: r-lib/actions/setup-r@v2 {{ macros.github_setup_local_r_repo(false, false)|indent }} {{ macros.github_checkout_arrow()|indent }} From d8aff44be00688ed3a4be59ce4d82bea8677f175 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 11 Oct 2023 23:40:29 +0200 Subject: [PATCH 17/22] use checkout v3 --- dev/tasks/r/github.packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index ccd8e1e657e48..d4117b972f742 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -390,7 +390,7 @@ jobs: if: matrix.platform.name == 'Linux' uses: r-lib/actions/setup-r@v2 {{ macros.github_setup_local_r_repo(false, false)|indent }} - {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_checkout_arrow(action_v="3")|indent }} - name: Install sccache if: matrix.platform.name == 'Linux' shell: bash From 8ba38a10b2ea7c5881eb1e5a0c7d7982722034f0 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 12 Oct 2023 03:30:04 +0200 Subject: [PATCH 18/22] remove superflous crc --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 68c2960aab353..a579757e64c11 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4317,8 +4317,7 @@ macro(build_google_cloud_cpp_storage) absl::synchronization absl::throw_delegate absl::time - absl::time_zone - Crc32c::crc32c) + absl::time_zone) endif() endmacro() From 66401fc6f827aadac8857a5e0c042a05ffc46605 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 12 Oct 2023 03:32:59 +0200 Subject: [PATCH 19/22] restore previous behavior of linux min build but full on macos --- r/tools/nixlibs.R | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index a3cfae4ffe54f..9d1fa3cebb18e 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -56,7 +56,8 @@ try_download <- function(from_url, to_file, hush = quietly) { } not_cran <- env_is("NOT_CRAN", "true") -if (not_cran) { +# enable full featured builds for macOS in case of CRAN source builds. +if (not_cran || on_macos) { # Set more eager defaults if (env_is("LIBARROW_BINARY", "")) { Sys.setenv(LIBARROW_BINARY = "true") @@ -720,14 +721,9 @@ is_feature_requested <- function(env_varname, default = env_is("LIBARROW_MINIMAL } with_cloud_support <- function(env_var_list) { - have_cloud_libs <- cmake_find_package("CURL", NULL, env_var_list) && - cmake_find_package("OpenSSL", "1.0.2", env_var_list) - # Build with cloud support when libs are available unless user explicitly turns them off - # Only show warnings if user explicitly requests cloud support. - arrow_s3 <- !env_is("ARROW_S3", "off") && have_cloud_libs || - is_feature_requested("ARROW_S3") - arrow_gcs <- !env_is("ARROW_GCS", "off") && have_cloud_libs || - is_feature_requested("ARROW_GCS") + arrow_s3 <- is_feature_requested("ARROW_S3") + arrow_gcs <- is_feature_requested("ARROW_GCS") + if (arrow_s3 || arrow_gcs) { # User wants S3 or GCS support. # Make sure that we have curl and openssl system libs From db2ea4cd8a8c5b0c05f220a37ed9cc9bdc156d34 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 12 Oct 2023 17:22:46 +0200 Subject: [PATCH 20/22] clarify step name --- dev/tasks/r/github.packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index d4117b972f742..a1c7c2f246438 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -405,7 +405,7 @@ jobs: - name: Enable parallel build run: | echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV - - name: Install arrow from nightly repo + - name: Install arrow source package env: # Test source build so be sure not to download a binary LIBARROW_BINARY: "FALSE" From e2c05f8d7c65d42e1178c1d3ec759db099a889c9 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 12 Oct 2023 18:40:32 +0200 Subject: [PATCH 21/22] fix core detection on mac --- dev/tasks/r/github.packages.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index a1c7c2f246438..5c41eff43c426 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -404,7 +404,8 @@ jobs: rm -rf arrow/ - name: Enable parallel build run: | - echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV + cores=`nproc || sysctl -n hw.logicalcpu` + echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV - name: Install arrow source package env: # Test source build so be sure not to download a binary From 16e72d45b7081f683f45b4a57c6f0106bb4b7b12 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 12 Oct 2023 16:18:54 -0300 Subject: [PATCH 22/22] Update r/tools/nixlibs.R Co-authored-by: Jonathan Keane --- r/tools/nixlibs.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 65e3e656a0564..3d651ec6ac9ea 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -57,7 +57,7 @@ try_download <- function(from_url, to_file, hush = quietly) { } not_cran <- env_is("NOT_CRAN", "true") -# enable full featured builds for macOS in case of CRAN source builds. +# enable full featured builds and binaries for macOS (or if the NOT_CRAN variable has been set) if (not_cran || on_macos) { # Set more eager defaults if (env_is("LIBARROW_BINARY", "")) {