Improve wheels and sdist

MANIFEST.in

@@ -1,3 +1,6 @@
-include curl_cffi/_wrapper.*
-include curl_cffi/ffi/*
-include curl_cffi/include/curl/*
+recursive-include tests/*
+
+include ffi/*
+include include/curl/*
+include scripts/build.py
+include Makefile

Makefile

@@ -18,8 +18,8 @@ curl-impersonate-$(VERSION)/chrome/patches: $(CURL_VERSION)
 	for p in $</curl-*.patch; do patch -p1 < ../$$p; done
 	# Re-generate the configure script
 	autoreconf -fi
-	mkdir -p ../curl_cffi/include/curl
-	cp -R include/curl/* ../curl_cffi/include/curl/
+	mkdir -p ../include/curl
+	cp -R include/curl/* ../include/curl/
 	# Sentinel files: https://tech.davis-hansson.com/p/make/
 	touch .preprocessed
 
@@ -44,6 +44,6 @@ clean:
 	rm -rf build/ dist/ curl_cffi.egg-info/ $(CURL_VERSION)/ curl-impersonate-$(VERSION)/
 	rm -rf curl_cffi/*.o curl_cffi/*.so curl_cffi/_wrapper.c
 	rm -rf .preprocessed $(CURL_VERSION).tar.xz curl-impersonate-$(VERSION).tar.gz
-	rm -rf curl_cffi/include/
+	rm -rf include/
 
 .PHONY: clean build test install-editable preprocess gen-const

curl_cffi/build.py

@@ -1,47 +1,76 @@
 import os
-import struct
 import platform
+import struct
 
 from cffi import FFI
 
-ffibuilder = FFI()
-# arch = "%s-%s" % (os.uname().sysname, os.uname().machine)
-uname = platform.uname()
 
+def abs_machine():
+    machine = platform.machine()
 
-if uname.system == "Windows":
-    if struct.calcsize("P") * 8 == 64:
-        libdir = "./lib64"
+    pointer_bits = struct.calcsize("P") * 8
+    if pointer_bits not in (32, 64):
+        raise Exception("Unsupported pointer size")
+
+    is_64 = pointer_bits == 64
+
+    # x86 based archs
+    if machine in ('AMD64', 'x86_64', 'i686'):
+        return "x86_64" if is_64 else "i686"
+    # arm based archs
+    elif machine in ('aarch64', 'arm64', 'armv6l', 'armv7l'):
+        return "aarch64" if is_64 else "arm"
     else:
+        raise Exception("Unsupported processor")
+
+
+ffibuilder = FFI()
+system = platform.system()
+machine = abs_machine()
+parent_dir = os.path.dirname(os.path.dirname(__file__))
+
+if system == "Windows":
+    if machine == "x86_64":
         libdir = "./lib32"
-elif uname.system == "Darwin":
-    libdir = "/Users/runner/work/_temp/install/lib"
+    elif machine == "i686":
+        libdir = "./lib64"
+    else:
+        libdir = "ERROR"
+elif system == "Darwin":
+    if machine in ("x86_64", "aarch64"):
+        libdir = "/Users/runner/work/_temp/install/lib"
+    else:
+        libdir = "ERROR"
 else:
-    libdir = os.path.expanduser("~/.local/lib")
+    if machine in ("x86_64", "arm", "aarch64"):
+        libdir = os.path.expanduser("~/.local/lib")
+    else:
+        libdir = "ERROR"
 
+if libdir == "ERROR":
+    raise Exception("Unsupported platform")
 
 ffibuilder.set_source(
     "curl_cffi._wrapper",
     """
         #include "shim.h"
     """,
-    libraries=["curl-impersonate-chrome"] if uname.system != "Windows" else ["libcurl"],
+    libraries=["curl-impersonate-chrome"] if system != "Windows" else ["libcurl"],
     library_dirs=[libdir],
     source_extension=".c",
     include_dirs=[
-        os.path.join(os.path.dirname(__file__), "include"),
-        os.path.join(os.path.dirname(__file__), "ffi"),
+        os.path.join(parent_dir, "include"),
+        os.path.join(parent_dir, "ffi"),
     ],
     sources=[
-        os.path.join(os.path.dirname(__file__), "ffi/shim.c"),
+        os.path.join(parent_dir, "ffi/shim.c"),
     ],
     extra_compile_args=(
-        ["-Wno-implicit-function-declaration"] if uname.system == "Darwin" else []
+        ["-Wno-implicit-function-declaration"] if system == "Darwin" else []
     ),
-    # extra_link_args=["-Wl,-rpath,$ORIGIN/../libcurl/" + arch],
 )
 
-with open(os.path.join(os.path.dirname(__file__), "ffi/cdef.c")) as f:
+with open(os.path.join(parent_dir, "ffi/cdef.c")) as f:
     cdef_content = f.read()
     ffibuilder.cdef(cdef_content)
 

pyproject.toml

@@ -5,7 +5,7 @@ authors = [{ name = "Yifei Kong", email = "[email protected]" }]
 description = "libcurl ffi bindings for Python, with impersonation support"
 license = { file = "LICENSE" }
 dependencies = [
-    "cffi >=1.12.0",
+    "cffi>=1.12.0",
     "certifi",
 ]
 readme = "README.md"
@@ -27,14 +27,12 @@ classifiers = [
 [project.optional-dependencies]
 dev = [
     "autoflake==1.4",
-    "black==22.8.0",
     "coverage==6.4.1",
     "cryptography==38.0.3",
     "flake8==6.0.0",
     "flake8-bugbear==22.7.1",
     "flake8-pie==0.15.0",
     "httpx==0.23.1",
-    "isort==5.10.1",
     "mypy==0.971",
     "types-certifi==2021.10.8.2",
     "pytest==7.1.2",
@@ -49,7 +47,7 @@ dev = [
 ]
 build = [
     "cibuildwheel",
-    "wheel"
+    "wheel",
 ]
 test = [
     "cryptography==38.0.3",
@@ -68,20 +66,15 @@ test = [
     "fastapi==0.100.0",
 ]
 
-[tool.ruff.lint]
-# Enable the isort rules.
-extend-select = ["I"]
+
+[build-system]
+requires = ["wheel", "setuptools", "cffi>=1.12.0"]
+build-backend = "setuptools.build_meta"
 
 
 [tool.setuptools]
 packages = ["curl_cffi", "curl_cffi.requests"]
 package-data = { curl_cffi = ["libcurl.dll"] }
-# include-package-data = true
-
-
-[build-system]
-requires = ["wheel", "setuptools", "cffi>=1.12.0"]
-build-backend = "setuptools.build_meta"
 
 
 [tool.cibuildwheel]
@@ -103,6 +96,7 @@ test-command = "python -bb -m pytest {project}/tests/unittest"
 test-extras = ["test"]
 test-skip = "pp*"
 
+
 # configure cibuildwheel to build native archs ('auto'), and some emulated ones
 [tool.cibuildwheel.linux]
 archs = ["auto", "aarch64"]
@@ -117,3 +111,8 @@ before-all = "gmake preprocess"
 [tool.pytest.ini_options]
 # pythonpath = [ "." ]
 asyncio_mode = "auto"
+
+
+[tool.ruff.lint]
+# Enable the isort rules.
+extend-select = ["I"]

setup.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import platform
 import shutil
 import struct
@@ -23,33 +24,66 @@ def get_tag(self):
         return python, abi, plat
 
 
-def download_so():
-    uname = platform.uname()
-    machine = uname.machine
+def abs_machine():
+    machine = platform.machine()
+
+    pointer_bits = struct.calcsize("P") * 8
+    if pointer_bits not in (32, 64):
+        raise Exception("Unsupported pointer size")
+
+    is_64 = pointer_bits == 64
+
+    # x86 based archs
+    if machine in ('AMD64', 'x86_64', 'i686'):
+        return "x86_64" if is_64 else "i686"
+    # arm based archs
+    elif machine in ('aarch64', 'arm64', 'armv6l', 'armv7l'):
+        return "aarch64" if is_64 else "arm"
+    else:
+        raise Exception("Unsupported processor")
+
 
-    # do not download if target platfrom dll not found
+def download_so():
+    system = platform.system()
+    machine = abs_machine()
 
-    if uname.system == "Windows":
+    if system == "Windows":
         sysname = "win32"
-        if struct.calcsize("P") * 8 == 64:
-            machine = "x86_64"
+        so_name = "libcurl.dll"
+
+        if machine == "x86_64":
             libdir = "./lib64"
-        else:
-            machine = "i686"
+        elif machine == "i686":
             libdir = "./lib32"
-        so_name = "libcurl.dll"
-    elif uname.system == "Darwin":
+        else:
+            so_name = "SKIP"
+
+    elif system == "Darwin":
         sysname = "macos"
-        libdir = "/Users/runner/work/_temp/install/lib"
         so_name = "libcurl-impersonate-chrome.4.dylib"
+
+        if machine in ("x86_64", "aarch64"):
+            libdir = "/Users/runner/work/_temp/install/lib"
+        else:
+            so_name = "SKIP"
+
     else:
         sysname = "linux-gnu"
-        libdir = os.path.expanduser("~/.local/lib")
         so_name = "libcurl-impersonate-chrome.so"
 
+        if machine in ("x86_64", "arm", "aarch64"):
+            libdir = os.path.expanduser("~/.local/lib")
+        else:
+            so_name = "SKIP"
+
+    if so_name == "SKIP":
+        print(f"libcurl for {sysname} platform is not available on github.")
+        return
+
     if (Path(libdir) / so_name).exists():
         print(".so files alreay downloaded.")
         return
+
     file = "libcurl-impersonate.tar.gz"
     url = (
         f"https://github.com/yifeikong/curl-impersonate/releases/download/"
@@ -63,7 +97,8 @@ def download_so():
     print("Unpacking downloaded files...")
     os.makedirs(libdir, exist_ok=True)
     shutil.unpack_archive(file, libdir)
-    if uname.system == "Windows":
+
+    if system == "Windows":
         shutil.copy2(f"{libdir}/libcurl.dll", "curl_cffi")
 
 
@@ -73,11 +108,15 @@ def run(self):
         super().run()
 
 
+# this option is only valid in setup.py
+kwargs = {"cffi_modules": ["curl_cffi/build.py:ffibuilder"]}
+if len(sys.argv) > 1 and sys.argv[1] != 'bdist_wheel':
+    kwargs = {}
+
 setup(
-    # this option is only valid in setup.py
-    cffi_modules=["curl_cffi/build.py:ffibuilder"],
     cmdclass={
         "bdist_wheel": bdist_wheel_abi3,  # type: ignore
         "build": my_build,  # type: ignore
     },
+    **kwargs,
 )