Enhance Cython integration: Add support for c++ mode, use hostpytho…

…n3-provided Cython, allow pinning of specific Cython version in recipe (#957)

* Add support to build cpp files from Cython, use Cython from hostpython3, and allow configuring Cython version to be used in recipe

* PEP8 (noqa as ATM we do not care why it failed to Cythonize)

* Add cymunk to broken recipes

* Remove cpplink, now it uses the same liblink

* Remove unused import

* Avoid DRY violation

.ci/constants.py

@@ -1,4 +1,4 @@
-BROKEN_RECIPES = set(["netifaces", "kivent_core"])
+BROKEN_RECIPES = set(["netifaces", "kivent_core", "cymunk"])
 
 # recipes that were already built will be skipped
 CORE_RECIPES = set(["kivy", "hostpython3", "python3"])

kivy_ios/recipes/audiostream/__init__.py

@@ -7,6 +7,7 @@ class AudiostreamRecipe(CythonRecipe):
     library = "libaudiostream.a"
     depends = ["python", "sdl2", "sdl2_mixer"]
     pre_build_ext = True
+    hostpython_prerequisites = ["Cython==0.29.37"]
 
 
 recipe = AudiostreamRecipe()

kivy_ios/recipes/curly/__init__.py

@@ -7,6 +7,7 @@ class CurlyRecipe(CythonRecipe):
     library = "libcurly.a"
     depends = ["python", "libcurl", "sdl2", "sdl2_image"]
     pre_build_ext = True
+    hostpython_prerequisites = ["Cython==0.29.37"]
 
 
 recipe = CurlyRecipe()

kivy_ios/recipes/cymunk/__init__.py

@@ -12,7 +12,7 @@ class CymunkRecipe(CythonRecipe):
     name = 'cymunk'
     pre_build_ext = True
     library = 'libcymunk.a'
-
+    hostpython_prerequisites = ["Cython==0.29.37"]
     depends = ['python']
 
     def get_recipe_env(self, arch):

kivy_ios/recipes/ffpyplayer/__init__.py

@@ -12,6 +12,7 @@ class FFPyplayerRecipe(CythonRecipe):
         "CoreMotion"]
     pbx_libraries = ["libiconv"]
     pre_build_ext = True
+    hostpython_prerequisites = ["Cython==0.29.37"]
 
     def get_recipe_env(self, plat):
         env = super(FFPyplayerRecipe, self).get_recipe_env(plat)

kivy_ios/recipes/hostpython3/__init__.py

@@ -100,11 +100,22 @@ def install(self):
                 self.ctx.dist_dir,
                 "hostpython3",
                 "lib",
-                "python3.11",
+                f"python{self.ctx.hostpython_ver}",
                 "site-packages",
                 "setuptools",
             ),
         )
+        self.apply_patch(
+             "fix-ldshared-override.patch",
+             join(
+                 self.ctx.dist_dir,
+                 "hostpython3",
+                 "lib",
+                 f"python{self.ctx.hostpython_ver}",
+                 "site-packages",
+                 "setuptools",
+             ),
+        )
 
 
 recipe = Hostpython3Recipe()

kivy_ios/recipes/hostpython3/fix-ldshared-override.patch

@@ -0,0 +1,31 @@
+diff -Naur setuptools.orig/_distutils/unixccompiler.py setuptools/_distutils/unixccompiler.py
+--- setuptools.orig/_distutils/unixccompiler.py	2024-02-11 18:42:58
++++ setuptools/_distutils/unixccompiler.py	2024-02-11 18:45:30
+@@ -253,14 +253,20 @@
+                 building_exe = target_desc == CCompiler.EXECUTABLE
+                 linker = (self.linker_exe if building_exe else self.linker_so)[:]
+
+-                if target_lang == "c++" and self.compiler_cxx:
+-                    env, linker_ne = _split_env(linker)
+-                    aix, linker_na = _split_aix(linker_ne)
+-                    _, compiler_cxx_ne = _split_env(self.compiler_cxx)
+-                    _, linker_exe_ne = _split_env(self.linker_exe)
++                # Mirko: We need our LDSHARED also for c++ things, 
++                # otherwise our hack to have static libs does not work
++                # properly.
++                # We will likely remove all these caveats once PEP 730
++                # is implemented (and we will conform to it).
+
+-                    params = _linker_params(linker_na, linker_exe_ne)
+-                    linker = env + aix + compiler_cxx_ne + params
++                #if target_lang == "c++" and self.compiler_cxx:
++                #    env, linker_ne = _split_env(linker)
++                #    aix, linker_na = _split_aix(linker_ne)
++                #    _, compiler_cxx_ne = _split_env(self.compiler_cxx)
++                #    _, linker_exe_ne = _split_env(self.linker_exe)
++
++                #    params = _linker_params(linker_na, linker_exe_ne)
++                #    linker = env + aix + compiler_cxx_ne + params
+
+                 linker = compiler_fixup(linker, ld_args)
+

kivy_ios/recipes/ios/__init__.py

@@ -7,6 +7,7 @@ class IosRecipe(CythonRecipe):
     library = "libios.a"
     depends = ["python"]
     pbx_frameworks = ["MessageUI", "CoreMotion", "UIKit", "WebKit", "Photos"]
+    hostpython_prerequisites = ["Cython==0.29.37"]
 
     def install(self):
         self.install_python_package(

kivy_ios/recipes/kivent_core/__init__.py

@@ -21,6 +21,7 @@ class KiventCoreRecipe(CythonRecipe):
     subbuilddir = False
     cythonize = True
     pbx_frameworks = ["OpenGLES"]  # note: This line may be unnecessary
+    hostpython_prerequisites = ["Cython==0.29.37"]
 
     def get_recipe_env(self, plat):
         env = super(KiventCoreRecipe, self).get_recipe_env(plat)

kivy_ios/recipes/kiwisolver/__init__.py

@@ -2,14 +2,10 @@
 This file is derived from the p4a recipe for kiwisolver.
 It is a dependency of matplotlib.
 
-It is a C++ library, and it utilizes the cpplink script to handle
-creating the library files needed for inclusion in an iOS project.
-
 It also depends on the headers from the cppy package.
 '''
 
 from kivy_ios.toolchain import CythonRecipe
-from os.path import join
 
 
 class KiwiSolverRecipe(CythonRecipe):
@@ -18,21 +14,9 @@ class KiwiSolverRecipe(CythonRecipe):
     version = '1.3.2'
     url = 'https://github.com/nucleic/kiwi/archive/{version}.zip'
     depends = ["python"]
-    hostpython_prerequisites = ["cppy"]
+    hostpython_prerequisites = ["cppy", "Cython==0.29.37"]
     cythonize = False
     library = "libkiwisolver.a"
 
-    def get_recipe_env(self, plat):
-        env = super().get_recipe_env(plat)
-
-        # cpplink setup
-        env['CXX_ORIG'] = env['CXX']
-        env['CXX'] = join(self.ctx.root_dir, "tools", "cpplink")
-
-        # setuptools uses CC for compiling and CXX for linking
-        env['CC'] = env['CXX']
-        env['CFLAGS'] += ' -isysroot {}'.format(env['IOSSDKROOT'])
-        return env
-
 
 recipe = KiwiSolverRecipe()

kivy_ios/recipes/matplotlib/__init__.py

@@ -2,9 +2,6 @@
 This file is derived from the p4a recipe for matplotlib.
 It is a dependency of matplotlib.
 
-It is a C++ library, and it utilizes the cpplink script to handle
-creating the library files needed for inclusion in an iOS project.
-
 In addition to the original patch files for p4a, additional patch files
 are necessary to prevent duplicate symbols from appearing in the final
 link of a kivy-ios application.
@@ -24,7 +21,7 @@ class MatplotlibRecipe(CythonRecipe):
     pre_build_ext = True
     python_depends = ['cycler', 'fonttools', 'packaging',
                       'pyparsing', 'python-dateutil', 'six']
-    hostpython_prerequisites = ['pybind11', 'certifi']
+    hostpython_prerequisites = ['pybind11', 'certifi', "Cython==0.29.37"]
     cythonize = False
 
     def generate_libraries_pc_files(self, plat):
@@ -113,11 +110,6 @@ def get_recipe_env(self, plat):
         numpy_inc_dir = dirname(sh.glob(numpytype.get_build_dir(plat) + '/**/_numpyconfig.h', recursive=True)[0])
 
         env['CFLAGS'] += f' -I{free_inc_dir} -I{numpy_inc_dir}'
-        env['CXX_ORIG'] = env['CXX']
-        env['CXX'] = join(self.ctx.root_dir, "tools", "cpplink")
-
-        # setuptools uses CC for compiling and CXX for linking
-        env['CFLAGS'] += ' -isysroot {}'.format(env['IOSSDKROOT'])
 
         return env
 

kivy_ios/recipes/netifaces/__init__.py

@@ -11,6 +11,7 @@ class NetifacesRecipe(CythonRecipe):
     url = "https://pypi.io/packages/source/n/netifaces/netifaces-{version}.tar.gz"
     depends = ["python3"]
     python_depends = ["setuptools"]
+    hostpython_prerequisites = ["Cython==0.29.37"]
     library = "libnetifaces.a"
     cythonize = False
 

kivy_ios/recipes/numpy/__init__.py

@@ -11,7 +11,7 @@ class NumpyRecipe(CythonRecipe):
     libraries = ["libnpymath.a", "libnpyrandom.a"]
     include_dir = "numpy/core/include"
     depends = ["python"]
-    hostpython_prerequisites = ["Cython==0.29.36"]
+    hostpython_prerequisites = ["Cython==0.29.37"]
     cythonize = False
 
     def prebuild_platform(self, plat):

kivy_ios/recipes/photolibrary/__init__.py

@@ -7,6 +7,7 @@ class PhotoRecipe(CythonRecipe):
     library = "libphotolibrary.a"
     depends = ["python"]
     pbx_frameworks = ["UIKit", "Photos", "MobileCoreServices"]
+    hostpython_prerequisites = ["Cython==0.29.37"]
 
     def install(self):
         self.install_python_package(name="photolibrary.so", is_dir=False)

kivy_ios/recipes/pillow/__init__.py

@@ -18,6 +18,7 @@ class PillowRecipe(CythonRecipe):
     python_depends = ["setuptools"]
     pbx_libraries = ["libz", "libbz2"]
     include_per_platform = True
+    hostpython_prerequisites = ["Cython==0.29.37"]
     cythonize = False
 
     def prebuild_platform(self, plat):

kivy_ios/recipes/pycrypto/__init__.py

@@ -12,6 +12,7 @@ class PycryptoRecipe(CythonRecipe):
     depends = ["python", "openssl"]
     include_per_platform = True
     library = "libpycrypto.a"
+    hostpython_prerequisites = ["Cython==0.29.37"]
 
     def build_platform(self, plat):
         build_env = plat.get_env()

kivy_ios/toolchain.py

@@ -176,6 +176,19 @@ def get_env(self):
         include_dirs += ["-I{}".format(
             join(self.ctx.dist_dir, "include", self.name))]
 
+        # Add Python include directories
+        include_dirs += [
+            "-I{}".format(
+                join(
+                    self.ctx.dist_dir,
+                    "root",
+                    "python3",
+                    "include",
+                    f"python{self.ctx.hostpython_ver}",
+                )
+            )
+        ]
+
         env = {}
         cc = sh.xcrun("-find", "-sdk", self.sdk, "clang").strip()
         cxx = sh.xcrun("-find", "-sdk", self.sdk, "clang++").strip()
@@ -249,6 +262,7 @@ def noicctempfile():
             "-O3",
             self.version_min,
         ] + include_dirs)
+        env["CXXFLAGS"] = env["CFLAGS"]
         env["LDFLAGS"] = " ".join([
             "-arch", self.arch,
             # "--sysroot", self.sysroot,
@@ -1134,6 +1148,7 @@ def reduce_python_package(self):
 class CythonRecipe(PythonRecipe):
     pre_build_ext = False
     cythonize = True
+    hostpython_prerequisites = ["Cython==3.0.11"]
 
     def cythonize_file(self, filename):
         if filename.startswith(self.build_dir):
@@ -1143,7 +1158,8 @@ def cythonize_file(self, filename):
         # doesn't (yet) have the executable bit hence we explicitly call it
         # with the Python interpreter
         cythonize_script = join(self.ctx.root_dir, "tools", "cythonize.py")
-        shprint(sh.Command(sys.executable), cythonize_script, filename)
+
+        shprint(sh.Command(self.ctx.hostpython), cythonize_script, filename)
 
     def cythonize_build(self):
         if not self.cythonize: