diff --git a/MANIFEST.in b/MANIFEST.in index 97b70a83..bb194bb8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,7 @@ +recursive-include tests * + include ffi/* -include curl_cffi/_wrapper.* -include curl_cffi/include/curl/* +include include/curl/* include scripts/build.py +include Makefile +include libs.json diff --git a/Makefile b/Makefile index ad8f4c58..1633082f 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,8 @@ curl-impersonate-$(VERSION)/chrome/patches: $(CURL_VERSION) for p in $=1.12.0", + "cffi>=1.12.0", "certifi", ] readme = "README.md" @@ -27,14 +27,12 @@ classifiers = [ [project.optional-dependencies] dev = [ "autoflake==1.4", - "black==22.8.0", "coverage==6.4.1", "cryptography==38.0.3", "flake8==6.0.0", "flake8-bugbear==22.7.1", "flake8-pie==0.15.0", "httpx==0.23.1", - "isort==5.10.1", "mypy==0.971", "types-certifi==2021.10.8.2", "pytest==7.1.2", @@ -49,7 +47,7 @@ dev = [ ] build = [ "cibuildwheel", - "wheel" + "wheel", ] test = [ "cryptography==38.0.3", @@ -68,20 +66,15 @@ test = [ "fastapi==0.100.0", ] -[tool.ruff.lint] -# Enable the isort rules. -extend-select = ["I"] + +[build-system] +requires = ["wheel", "setuptools", "cffi>=1.12.0"] +build-backend = "setuptools.build_meta" [tool.setuptools] packages = ["curl_cffi", "curl_cffi.requests"] package-data = { curl_cffi = ["libcurl.dll"] } -# include-package-data = true - - -[build-system] -requires = ["wheel", "setuptools", "cffi>=1.12.0"] -build-backend = "setuptools.build_meta" [tool.cibuildwheel] @@ -103,6 +96,7 @@ test-command = "python -bb -m pytest {project}/tests/unittest" test-extras = ["test"] test-skip = "pp*" + # configure cibuildwheel to build native archs ('auto'), and some emulated ones [tool.cibuildwheel.linux] archs = ["auto", "aarch64"] @@ -117,3 +111,8 @@ before-all = "gmake preprocess" [tool.pytest.ini_options] # pythonpath = [ "." ] asyncio_mode = "auto" + + +[tool.ruff.lint] +# Enable the isort rules. +extend-select = ["I"] diff --git a/scripts/build.py b/scripts/build.py index b303d2d2..d6d73dad 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -1,46 +1,82 @@ +import json import os -import struct import platform +import shutil +import struct from pathlib import Path +from urllib.request import urlretrieve from cffi import FFI -ffibuilder = FFI() -# arch = "%s-%s" % (os.uname().sysname, os.uname().machine) -uname = platform.uname() +__version__ = "0.6.0b9" + + +def detect_arch(): + with open(Path(__file__).parent.parent / "libs.json") as f: + archs = json.loads(f.read()) + uname = platform.uname() + pointer_size = struct.calcsize("P") * 8 + for arch in archs: + if ( + arch["system"] == uname.system + and arch["machine"] == uname.machine + and arch["pointer_size"] == pointer_size + ): + arch["libdir"] = os.path.expanduser(arch["libdir"]) + return arch + raise Exception(f"Unsupported arch: {uname}") + + +arch = detect_arch() +def download_so(): + if (Path(arch["libdir"]) / arch["so_name"]).exists(): + print(".so files alreay downloaded.") + return -if uname.system == "Windows": - if struct.calcsize("P") * 8 == 64: - libdir = "./lib64" - else: - libdir = "./lib32" -elif uname.system == "Darwin": - libdir = "/Users/runner/work/_temp/install/lib" -else: - libdir = os.path.expanduser("~/.local/lib") + file = "libcurl-impersonate.tar.gz" + url = ( + f"https://github.com/yifeikong/curl-impersonate/releases/download/" + f"v{__version__}/libcurl-impersonate-v{__version__}" + f".{arch['so_arch']}-{arch['sysname']}.tar.gz" + ) + print(f"Downloading libcurl-impersonate-chrome from {url}...") + urlretrieve(url, file) + + print("Unpacking downloaded files...") + os.makedirs(arch["libdir"], exist_ok=True) + shutil.unpack_archive(file, arch["libdir"]) + + if arch["system"] == "Windows": + shutil.copy2(f"{arch['libdir']}/libcurl.dll", "curl_cffi") + + +ffibuilder = FFI() +system = platform.system() root_dir = Path(__file__).parent.parent +download_so() + ffibuilder.set_source( "curl_cffi._wrapper", """ #include "shim.h" """, - libraries=["curl-impersonate-chrome"] if uname.system != "Windows" else ["libcurl"], - library_dirs=[libdir], + # FIXME from `curl-impersonate` + libraries=["curl-impersonate-chrome"] if system != "Windows" else ["libcurl"], + library_dirs=[arch["libdir"]], source_extension=".c", include_dirs=[ - str(root_dir / "curl_cffi/include"), + str(root_dir / "include"), str(root_dir / "ffi"), ], sources=[ str(root_dir / "ffi/shim.c"), ], extra_compile_args=( - ["-Wno-implicit-function-declaration"] if uname.system == "Darwin" else [] + ["-Wno-implicit-function-declaration"] if system == "Darwin" else [] ), - # extra_link_args=["-Wl,-rpath,$ORIGIN/../libcurl/" + arch], ) with open(root_dir / "ffi/cdef.c") as f: diff --git a/setup.py b/setup.py index f33836a7..711ff902 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,5 @@ -import os -import platform -import shutil -import struct -from pathlib import Path from setuptools import setup -from urllib.request import urlretrieve from wheel.bdist_wheel import bdist_wheel -from distutils.command.build import build - - -__version__ = "0.6.0b9" class bdist_wheel_abi3(bdist_wheel): @@ -23,61 +13,10 @@ def get_tag(self): return python, abi, plat -def download_so(): - uname = platform.uname() - machine = uname.machine - - # do not download if target platfrom dll not found - - if uname.system == "Windows": - sysname = "win32" - if struct.calcsize("P") * 8 == 64: - machine = "x86_64" - libdir = "./lib64" - else: - machine = "i686" - libdir = "./lib32" - so_name = "libcurl.dll" - elif uname.system == "Darwin": - sysname = "macos" - libdir = "/Users/runner/work/_temp/install/lib" - so_name = "libcurl-impersonate-chrome.4.dylib" - else: - sysname = "linux-gnu" - libdir = os.path.expanduser("~/.local/lib") - so_name = "libcurl-impersonate-chrome.so" - - if (Path(libdir) / so_name).exists(): - print(".so files alreay downloaded.") - return - file = "libcurl-impersonate.tar.gz" - url = ( - f"https://github.com/yifeikong/curl-impersonate/releases/download/" - f"v{__version__}/libcurl-impersonate-v{__version__}" - f".{machine}-{sysname}.tar.gz" - ) - - print(f"Downloading libcurl-impersonate-chrome from {url}...") - urlretrieve(url, file) - - print("Unpacking downloaded files...") - os.makedirs(libdir, exist_ok=True) - shutil.unpack_archive(file, libdir) - if uname.system == "Windows": - shutil.copy2(f"{libdir}/libcurl.dll", "curl_cffi") - - -class my_build(build): - def run(self): - download_so() - super().run() - - setup( # this option is only valid in setup.py cffi_modules=["scripts/build.py:ffibuilder"], cmdclass={ "bdist_wheel": bdist_wheel_abi3, # type: ignore - "build": my_build, # type: ignore }, )