Make azure blob storage can be compiled in arrow (#44)

Signed-off-by: sunby <[email protected]>

arrow/all/conanfile.py

@@ -60,6 +60,7 @@ class ArrowConan(ConanFile):
         "with_protobuf": ["auto", True, False],
         "with_re2": ["auto", True, False],
         "with_s3": [True, False],
+        "with_azure": [True, False],
         "with_utf8proc": ["auto", True, False],
         "with_brotli": [True, False],
         "with_bz2": [True, False],
@@ -109,6 +110,7 @@ class ArrowConan(ConanFile):
         "with_protobuf": False,
         "with_re2": False,
         "with_s3": False,
+        "with_azure": False,
         "with_utf8proc": False,
         "with_lz4": False,
         "with_snappy": False,
@@ -170,7 +172,7 @@ def requirements(self):
         if self.options.with_mimalloc:
             self.requires("mimalloc/1.7.6")
         if self.options.with_boost:
-            self.requires("boost/1.84.0")
+            self.requires("boost/1.85.0")
         if self.options.with_gflags:
             self.requires("gflags/2.2.2")
         if self.options.with_glog:
@@ -216,6 +218,8 @@ def requirements(self):
             self.requires("libbacktrace/cci.20210118")
         if self.options.with_orc:
             self.requires("orc/2.0.0")
+        if self.options.with_azure:
+            self.requires("azure-sdk-for-cpp/1.11.3")
 
     def validate(self):
         # Do not allow options with 'auto' value
@@ -361,6 +365,7 @@ def generate(self):
             tc.variables["ARROW_USE_BOOST"] = True
             tc.variables["ARROW_BOOST_USE_SHARED"] = bool(self.dependencies["boost"].options.shared)
         tc.variables["ARROW_S3"] = bool(self.options.with_s3)
+        tc.variables["ARROW_AZURE"] = bool(self.options.with_azure)
         tc.variables["AWSSDK_SOURCE"] = "SYSTEM"
         tc.variables["ARROW_BUILD_UTILITIES"] = bool(self.options.cli)
         tc.variables["ARROW_BUILD_INTEGRATION"] = False
@@ -579,7 +584,10 @@ def package_info(self):
         if self._requires_rapidjson():
             self.cpp_info.components["libarrow"].requires.append("rapidjson::rapidjson")
         if self.options.with_s3:
-            self.cpp_info.components["libarrow"].requires.append("aws-sdk-cpp::s3")
+            # https://github.com/apache/arrow/blob/6b268f62a8a172249ef35f093009c740c32e1f36/cpp/src/arrow/CMakeLists.txt#L98
+            self.cpp_info.components["libarrow"].requires.extend([f"aws-sdk-cpp::{x}" for x in ["cognito-identity", "core", "identity-management", "s3", "sts"]])
+        if self.options.with_azure:
+            self.cpp_info.components["libarrow"].requires.extend([f"azure-sdk-for-cpp::{x}" for x in ["azure-storage-common","azure-storage-blobs","azure-identity","azure-storage-files-shares","azure-storage-files-datalake"]])
         if self.options.get_safe("with_gcs"):
             self.cpp_info.components["libarrow"].requires.append("google-cloud-cpp::storage")
         if self.options.with_orc:

azure-sdk-for-cpp/all/CMakeLists.txt

@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.15)
+project(cmake_wrapper)
+
+# The cmake_wrapper allows users to build only modules they want and not the entire sdk,
+# the CMakeLists.txt from source does not provide this modularity to users (it's all or nothing).
+
+foreach(sdk ${BUILD_LIST})
+    if(${sdk} STREQUAL azure-core)
+        # Always build Core
+        add_subdirectory("src/sdk/core")
+    elseif(${sdk} STREQUAL azure-storage-common)
+        add_subdirectory("src/sdk/storage/azure-storage-common")
+    elseif(${sdk} STREQUAL azure-storage-blobs)
+        add_subdirectory("src/sdk/storage/azure-storage-blobs")
+    elseif(${sdk} STREQUAL azure-storage-files-shares)
+        add_subdirectory("src/sdk/storage/azure-storage-files-shares")
+    endif()
+endforeach()

azure-sdk-for-cpp/all/conandata.yml

@@ -0,0 +1,4 @@
+sources:
+  "1.11.3":
+    url: "https://github.com/Azure/azure-sdk-for-cpp/archive/refs/tags/azure-core_1.11.3.tar.gz"
+    sha256: "c67e42622bf1ebafee29aa09f333e41adc24712b0c993ada5dd97c9265b444cc"

azure-sdk-for-cpp/all/conanfile.py

@@ -0,0 +1,115 @@
+from conan import ConanFile
+from conan.errors import ConanInvalidConfiguration
+from conan.tools.build import check_min_cppstd
+from conan.tools.cmake import CMakeToolchain, CMake, CMakeDeps, cmake_layout
+from conan.tools.files import get, copy, rmdir
+from conan.tools.scm import Version
+import os
+
+required_conan_version = ">=1.54.0"
+
+AZURE_SDK_MODULES = (
+    "azure-storage-common",
+    "azure-storage-blobs",
+    "azure-identity",
+    "azure-storage-files-shares",
+    "azure-storage-files-datalake",
+    "azure-data-tables"
+)
+
+class AzureSDKForCppConan(ConanFile):
+    name = "azure-sdk-for-cpp"
+    description = "Microsoft Azure Storage Client Library for C++"
+    license = "Apache-2.0"
+    url = "https://github.com/conan-io/conan-center-index"
+    homepage = "https://github.com/Azure/azure-sdk-for-cpp"
+    topics = ("azure", "cpp", "cross-platform", "microsoft", "cloud")
+    package_type = "library"
+    settings = "os", "arch", "compiler", "build_type"
+    options = {"shared": [True, False], "fPIC": [True, False]}
+    options.update({_name: [True, False] for _name in AZURE_SDK_MODULES})
+    default_options = {"shared": False, "fPIC": True}
+    default_options.update({_name: True for _name in AZURE_SDK_MODULES}) # Build all modules by default, let users pick what they do not want
+
+    def export_sources(self):
+        copy(self, "CMakeLists.txt", src=self.recipe_folder, dst=self.export_sources_folder)
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version], strip_root=True)
+
+    def configure(self):
+        if self.options.get_safe("shared"):
+            self.options.rm_safe("fPIC")
+
+    def requirements(self):
+        self.requires("openssl/[>=1.1 <4]")
+        self.requires("libcurl/[>=7.78 <9]")
+        self.requires("libxml2/[>=2.12.5 <3]")
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def validate(self):
+        if self.settings.compiler.get_safe("cppstd"):
+            check_min_cppstd(self, 14)
+
+        # Open to contributions for windows and apple
+        if self.settings.os != "Linux":
+            raise ConanInvalidConfiguration(
+                f"{self.ref} Conan recipe in ConanCenter still does not support {self.settings.os}, contributions to the recipe welcome.")
+
+        if self.settings.compiler != "gcc":
+            raise ConanInvalidConfiguration(
+                f"{self.ref} Conan recipe in ConanCenter still does not support {self.settings.compiler}, contributions to the recipe welcome.")
+
+        if self.settings.compiler == 'gcc' and Version(self.settings.compiler.version) < "6":
+            raise ConanInvalidConfiguration("Building requires GCC >= 6")
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+
+        build_list = ["azure-core"]
+        for sdk in AZURE_SDK_MODULES:
+            if self.options.get_safe(sdk):
+                build_list.append(sdk)
+        tc.cache_variables["BUILD_LIST"] = ";".join(build_list)
+
+        tc.variables["AZ_ALL_LIBRARIES"] = "ON"
+        tc.variables["FETCH_SOURCE_DEPS"] = "OFF"
+        tc.cache_variables["BUILD_TESTING"] = "OFF"
+        tc.cache_variables["BUILD_WINDOWS_UWP"] = "ON"
+        tc.cache_variables["DISABLE_AZURE_CORE_OPENTELEMETRY"] = "ON"
+        tc.cache_variables["BUILD_TRANSPORT_CURL"] = "ON"
+        tc.generate()
+
+        deps = CMakeDeps(self)
+        deps.generate()
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def package(self):
+        copy(self, "LICENSE.txt",
+             dst=os.path.join(self.package_folder, "licenses"),
+             src=self.source_folder)
+        cmake = CMake(self)
+        cmake.install()
+
+        rmdir(self, os.path.join(self.package_folder, "share"))
+
+    def package_info(self):
+        self.cpp_info.set_property("cmake_file_name", "Azure")
+
+        # core component
+        self.cpp_info.components["azure-core"].set_property("cmake_target_name", "Azure::azure-core")
+        self.cpp_info.components["azure-core"].libs = ["azure-core"]
+        self.cpp_info.components["azure-core"].requires.extend(["openssl::openssl", "libcurl::curl", "libxml2::libxml2"])
+
+        enabled_sdks = [sdk for sdk in AZURE_SDK_MODULES if self.options.get_safe(sdk)]
+        for sdk in enabled_sdks:
+            self.cpp_info.components[sdk].set_property("cmake_target_name", f"Azure::{sdk}")
+            self.cpp_info.components[sdk].libs = [sdk]
+            self.cpp_info.components[sdk].requires.extend(["openssl::openssl", "libcurl::curl", "libxml2::libxml2"])
+

azure-sdk-for-cpp/all/test_package/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.8)
+
+project(test_package LANGUAGES CXX)
+
+find_package(AzureSDK CONFIG REQUIRED)
+
+add_executable(test_azure-core test_azure-core.cc)
+target_link_libraries(test_azure-core PRIVATE Azure::azure-core)
+
+add_executable(test_azure-storage-common test_azure-storage-common.cc)
+target_link_libraries(test_azure-storage-common PRIVATE Azure::azure-core Azure::azure-storage-common)
+
+add_executable(test_azure-storage-blobs test_azure-storage-blobs.cc)
+target_link_libraries(test_azure-storage-blobs PRIVATE Azure::azure-core Azure::azure-storage-common Azure::azure-storage-blobs)
+
+add_executable(test_azure-storage-files-shares test_azure-storage-files-shares.cc)
+target_link_libraries(test_azure-storage-files-shares PRIVATE Azure::azure-core Azure::azure-storage-common Azure::azure-storage-files-shares)

azure-sdk-for-cpp/all/test_package/conanfile.py

@@ -0,0 +1,33 @@
+from conan import ConanFile
+from conan.tools.build import can_run
+from conan.tools.cmake import CMake, cmake_layout
+import os
+
+
+class TestPackageConan(ConanFile):
+    settings = "os", "arch", "compiler", "build_type"
+    generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv"
+
+    @property
+    def _tested_modules(self):
+        return ["azure-core",
+                "azure-storage-common",
+                "azure-storage-blobs",
+                "azure-storage-files-shares"]
+
+    def layout(self):
+        cmake_layout(self)
+
+    def requirements(self):
+        self.requires(self.tested_reference_str)
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def test(self):
+        if can_run(self):
+            for module in self._tested_modules:
+                bin_path = os.path.join(self.cpp.build.bindirs[0], f"test_{module}")
+                self.run(bin_path, env="conanrun")

azure-sdk-for-cpp/all/test_package/test_azure-core.cc

@@ -0,0 +1,10 @@
+#include <iostream>
+#include <azure/core.hpp>
+
+int main()
+{
+  std::vector<uint8_t> data = {1, 2, 3, 4};
+  Azure::Core::IO::MemoryBodyStream stream(data);
+
+  return 0;
+}

azure-sdk-for-cpp/all/test_package/test_azure-storage-blobs.cc

@@ -0,0 +1,10 @@
+#include <azure/storage/blobs.hpp>
+
+using namespace Azure::Storage::Blobs;
+
+int main()
+{
+  BlobAudience audience{"TEST"};
+
+  return 0;
+}

azure-sdk-for-cpp/all/test_package/test_azure-storage-common.cc

@@ -0,0 +1,10 @@
+#include <azure/storage/common/storage_common.hpp>
+
+using namespace Azure::Storage;
+
+int main()
+{
+  ContentHash contentHash{};
+
+  return 0;
+}

azure-sdk-for-cpp/all/test_package/test_azure-storage-files-shares.cc

@@ -0,0 +1,10 @@
+#include <azure/storage/files/shares.hpp>
+
+using namespace Azure::Storage::Files::Shares;
+
+int main()
+{
+  SetSharePropertiesOptions options;
+
+  return 0;
+}

azure-sdk-for-cpp/config.yml

@@ -0,0 +1,3 @@
+versions:
+  "1.11.3":
+    folder: "all"