Update to python 3.8.2 (#443)

* bump python version to 3.8.2

* Revert to cython 0.28.1

.github/workflows/kivy_ios.yml

@@ -34,7 +34,7 @@ jobs:
         pip3 install sh
         brew install autoconf automake libtool pkg-config
         brew link libtool
-        pip3 install Cython==0.29.10
+        pip3 install Cython==0.28.1
     - name: Build updated recipes
       run: |
         python3 .ci/rebuild_updated_recipes.py

recipes/host_setuptools3/__init__.py

@@ -20,7 +20,7 @@ def prebuild_arch(self, arch):
         # Setuptools version 28.3.0
         site_packages_path = join(
             self.ctx.dist_dir, 'hostpython3',
-            'lib', 'python3.7', 'site-packages')
+            'lib', 'python3.8', 'site-packages')
         os.chdir(site_packages_path)
         with open('setuptools.pth', 'r') as f:
             setuptools_egg_path = f.read().strip('./').strip('\n')

recipes/hostopenssl/__init__.py

@@ -0,0 +1,39 @@
+from toolchain import Recipe, shprint
+from os.path import join
+import sh
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class HostOpensslRecipe(Recipe):
+    version = "1.1.1f"
+    url = "http://www.openssl.org/source/openssl-{version}.tar.gz"
+    archs = ["x86_64"]
+
+    def get_build_env(self):
+        build_env = self.ctx.env.copy()
+        self.build_env_x86_84 = build_env
+        return build_env
+
+    def build_x86_64(self):
+        build_env = self.get_build_env()
+        configure = sh.Command(join(self.build_dir, "Configure"))
+        shprint(configure,
+                "darwin64-x86_64-cc",
+                _env=build_env)
+        shprint(sh.make, "clean")
+        shprint(sh.make, self.ctx.concurrent_make, "build_libs")
+
+    def install(self):
+        sh.mkdir('-p', join(self.ctx.dist_dir, 'hostopenssl'))
+        sh.cp('-r', join(self.get_build_dir('x86_64'), 'include'),
+              join(self.ctx.dist_dir, 'hostopenssl', 'include'))
+        sh.mkdir('-p', join(self.ctx.dist_dir, 'hostopenssl', 'lib'))
+        sh.cp(join(self.get_build_dir('x86_64'), 'libssl.a'),
+              join(self.ctx.dist_dir, 'hostopenssl', 'lib'))
+        sh.cp(join(self.get_build_dir('x86_64'), 'libcrypto.a'),
+              join(self.ctx.dist_dir, 'hostopenssl', 'lib'))
+
+
+recipe = HostOpensslRecipe()

recipes/hostpython3/__init__.py

@@ -1,5 +1,5 @@
-from toolchain import Recipe, shprint, ensure_dir
-from os.path import join, exists
+from toolchain import Recipe, shprint
+from os.path import join
 import os
 import sh
 import shutil
@@ -9,16 +9,16 @@
 
 
 class Hostpython3Recipe(Recipe):
-    version = "3.7.1"
+    version = "3.8.2"
     url = "https://www.python.org/ftp/python/{version}/Python-{version}.tgz"
-    depends = []
-    optional_depends = ["openssl"]
+    depends = ["hostlibffi", "hostopenssl"]
+    optional_depends = []
     archs = ["x86_64"]
 
     def init_with_ctx(self, ctx):
         super(Hostpython3Recipe, self).init_with_ctx(ctx)
-        self.set_hostpython(self, "3.7")
-        self.ctx.so_suffix = ".cpython-37m-darwin.so"
+        self.set_hostpython(self, "3.8")
+        self.ctx.so_suffix = ".cpython-38m-darwin.so"
         self.ctx.hostpython = join(self.ctx.dist_dir, "hostpython3", "bin", "python")
         self.ctx.hostpgen = join(self.ctx.dist_dir, "hostpython3", "bin", "pgen")
         logger.info("Global: hostpython located at {}".format(self.ctx.hostpython))
@@ -28,33 +28,10 @@ def prebuild_arch(self, arch):
         if self.has_marker("patched"):
             return
         self.copy_file("ModulesSetup", "Modules/Setup.local")
-        # self.apply_patch("ssize-t-max.patch")
-        # self.apply_patch("dynload.patch")
-        # self.apply_patch("static-_sqlite3.patch")
-        # shutil.copy("Modules/Setup.dist", "Modules/Setup")
-        # if "openssl.build_all" in self.ctx.state:
-        #     self.append_file("ModulesSetup.openssl", "Modules/Setup.local")
         self.set_marker("patched")
 
     def postbuild_arch(self, arch):
         return
-        """
-        makefile_fn = join(self.build_dir, "Makefile")
-        with open(makefile_fn) as fd:
-            lines = fd.readlines()
-        for index, line in enumerate(lines):
-            if "-bundle" not in line:
-                continue
-            parts = line.split(" ")
-            parts.remove("-bundle")
-            if "-bundle_loader" in parts:
-                i = parts.index("-bundle_loader")
-                parts.pop(i)
-                parts.pop(i)
-            lines[index] = " ".join(parts)
-        with open(makefile_fn, "w") as fd:
-            fd.writelines(lines)
-        """
 
     def get_build_env(self):
         sdk_path = sh.xcrun("--sdk", "macosx", "--show-sdk-path").strip()
@@ -73,52 +50,40 @@ def get_build_env(self):
                 "-mmacosx-version-min=10.12",
                 "-I{}".format(join(self.ctx.dist_dir, "hostlibffi", "usr", "local", "include"))
                 ])
-        if "openssl.build_all" in self.ctx.state:
-            build_env["LDFLAGS"] += " -L{}".format(join(self.ctx.dist_dir, "lib"))
-            build_env["CFLAGS"] += " -I{}".format(join(self.ctx.dist_dir, "include",
-                                                       "x86_64", "openssl"))
         return build_env
 
     def build_x86_64(self):
         build_env = self.get_build_env()
         configure = sh.Command(join(self.build_dir, "configure"))
         shprint(configure,
                 "--prefix={}".format(join(self.ctx.dist_dir, "hostpython3")),
-                # "--disable-toolbox-glue",
-                # "--without-gcc",
+                "--with-openssl={}".format(join(self.ctx.dist_dir, 'hostopenssl')),
                 _env=build_env)
         shprint(sh.make, "-C", self.build_dir, self.ctx.concurrent_make,
                 _env=build_env)
-        # shutil.move("python", "hostpython")
-        # shutil.move("Parser/pgen", "Parser/hostpgen")
 
     def install(self):
         arch = list(self.filtered_archs)[0]
         build_env = self.get_build_env()
         build_dir = self.get_build_dir(arch.arch)
         build_env["PATH"] = os.environ["PATH"]
-        # Compiling sometimes looks for Python-ast.py in the 'Python' i.s.o.
-        # the 'hostpython' folder. Create a symlink to fix. See issue #201
-        # shprint(sh.ln, "-s",
-        #         join(build_dir, "hostpython3"),
-        #         join(build_dir, "Python"))
         shprint(sh.make, self.ctx.concurrent_make,
                 "-C", build_dir,
                 "install",
                 _env=build_env)
-        # pylib_dir = join(self.ctx.dist_dir, "hostpython3", "lib", "python3.7")
-        # if exists(pylib_dir):
-        #     shutil.rmtree(pylib_dir)
-        # shutil.copytree(
-        #     join(build_dir, "Lib"),
-        #     pylib_dir)
-        # ensure_dir(join(pylib_dir, "config"))
-        # shutil.copy(
-        #     join(build_dir, "Makefile"),
-        #     join(pylib_dir, "config", "Makefile"))
         shutil.copy(
             join(self.ctx.dist_dir, "hostpython3", "bin", "python3"),
             join(self.ctx.dist_dir, "hostpython3", "bin", "python"))
+        """
+        I don't like this kind of "patches".
+        sysconfig was overriding our cflags and extensions were failing to build.
+        This hack resets the cflags provided by sysconfig.
+        """
+        with open(join(self.ctx.dist_dir, "hostpython3", "lib", "python3.8", "distutils", "sysconfig.py"), 'r') as sysconfigfile:
+            lines = sysconfigfile.readlines()
+        lines[192] = '        cflags = ""\n'
+        with open(join(self.ctx.dist_dir, "hostpython3", "lib", "python3.8", "distutils", "sysconfig.py"), 'w') as sysconfigfile:
+            sysconfigfile.writelines(lines)
 
 
 recipe = Hostpython3Recipe()

recipes/kivy/__init__.py

@@ -9,8 +9,13 @@
 
 
 class KivyRecipe(CythonRecipe):
-    # post kivy 1.11.1, including statusbar/fullscreen fix
-    version = "38fcbd5b90c99a96d82682f14986836cde81412d"
+    """
+    post kivy 2.0.0rc1
+    Includes these iOS specific fixes:
+    - Statusbar / Fullscreen fix (PR #4589)
+    - Extend usage of certifi on iOS (PR #4648)
+    """
+    version = "067064c23a275187e67f1c9d7de7cc06f384af4d"
     url = "https://github.com/kivy/kivy/archive/{version}.zip"
     library = "libkivy.a"
     depends = ["sdl2", "sdl2_image", "sdl2_mixer", "sdl2_ttf", "ios",

recipes/python3/ModulesSetup

@@ -68,7 +68,7 @@ zlib zlibmodule.c -I$(prefix)/include -lz
 #####################################################################
 _ctypes_test _ctypes/_ctypes_test.c
 _testbuffer _testbuffer.c
-_testcapi  _testcapimodule.c
+_testinternalcapi  _testinternalcapi.c
 _testimportmultiple _testimportmultiple.c
 
 #####################################################################

recipes/python3/__init__.py

@@ -9,38 +9,41 @@
 
 
 class Python3Recipe(Recipe):
-    version = "3.7.1"
+    version = "3.8.2"
     url = "https://www.python.org/ftp/python/{version}/Python-{version}.tgz"
     depends = ["hostpython3", "libffi", "openssl"]
-    library = "libpython3.7m.a"
+    library = "libpython3.8.a"
     pbx_libraries = ["libz", "libbz2", "libsqlite3"]
 
     def init_with_ctx(self, ctx):
         super(Python3Recipe, self).init_with_ctx(ctx)
-        self.set_python(self, "3.7")
-        ctx.python_ver_dir = "python3.7"
+        self.set_python(self, "3.8")
+        ctx.python_ver_dir = "python3.8"
         ctx.python_prefix = join(ctx.dist_dir, "root", "python3")
         ctx.site_packages_dir = join(
             ctx.python_prefix, "lib", ctx.python_ver_dir, "site-packages")
 
     def prebuild_arch(self, arch):
         # common to all archs
-        if  self.has_marker("patched"):
+        if self.has_marker("patched"):
             return
-        self.apply_patch("dynload.patch")
+        self.apply_patch("config.sub.patch")
+        self.apply_patch("configure.patch")
+        self.apply_patch("posixmodule.patch")
+        self.apply_patch("dynload_shlib.patch")
+        self.apply_patch("disable_explicit_blake2.patch")
         self.copy_file("ModulesSetup", "Modules/Setup.local")
         self.append_file("ModulesSetup.mobile", "Modules/Setup.local")
-        self.apply_patch("xcompile.patch")
         self.set_marker("patched")
-    
+
     def postbuild_arch(self, arch):
         # include _sqlite module to .a
         py_arch = arch.arch
         if py_arch == "armv7":
             py_arch = "arm"
         elif py_arch == "arm64":
             py_arch = "aarch64"
-        tmp_folder = "temp.ios-{}-3.7{}".format(py_arch, self.build_dir)
+        tmp_folder = "temp.ios-{}-3.8{}".format(py_arch, self.build_dir)
         build_env = self.get_build_env(arch)
         for o_file in [
             "cache.o",
@@ -81,7 +84,6 @@ def build_arch(self, arch):
                 "LD={}".format(build_env["LD"]),
                 "CFLAGS={}".format(build_env["CFLAGS"]),
                 "LDFLAGS={} -undefined dynamic_lookup".format(build_env["LDFLAGS"]),
-                # "--without-pymalloc",
                 "ac_cv_file__dev_ptmx=yes",
                 "ac_cv_file__dev_ptc=no",
                 "ac_cv_little_endian_double=yes",
@@ -114,10 +116,13 @@ def build_arch(self, arch):
                 "--prefix={}".format(prefix),
                 "--without-ensurepip",
                 "--with-system-ffi",
-                # "--without-doc-strings",
                 "--enable-ipv6",
+                "PYTHON_FOR_BUILD=_PYTHON_PROJECT_BASE=$(abs_builddir) \
+                    _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) \
+                    PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib\
+                    _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH)\
+                    {}".format(sh.Command(self.ctx.hostpython)),
                 _env=build_env)
-
         self.apply_patch("ctypes_duplicate.patch")
         shprint(sh.make, self.ctx.concurrent_make)
 
@@ -130,32 +135,29 @@ def install(self):
                 "install",
                 "prefix={}".format(join(self.ctx.dist_dir, "root", "python3")),
                 _env=build_env)
-        # os.execve("/bin/bash", ["/bin/bash"], os.environ)
         self.reduce_python()
         self.install_mock_modules()
 
     def install_mock_modules(self):
         logger.info("Install mock modules")
         sqlite3_src = join(self.recipe_dir, 'mock_modules', '_sqlite3')
         site_packages_folder = join(
-                self.ctx.dist_dir, "root", "python3", "lib", "python3.7", "site-packages", "_sqlite3")
-        shutil.rmtree(site_packages_folder, ignore_errors=True) # Needed in case of rebuild
+                self.ctx.dist_dir, "root", "python3", "lib", "python3.8", "site-packages", "_sqlite3")
+        shutil.rmtree(site_packages_folder, ignore_errors=True)  # Needed in case of rebuild
         shutil.copytree(sqlite3_src, site_packages_folder)
-    
+
     def reduce_python(self):
         logger.info("Reduce python")
         oldpwd = os.getcwd()
         try:
             logger.info("Remove files unlikely to be used")
             os.chdir(join(self.ctx.dist_dir, "root", "python3"))
-            # os.execve("/bin/bash", ["/bin/bash"], env=os.environ)
             sh.rm("-rf", "bin", "share")
-
             # platform binaries and configuration
             os.chdir(join(
                 self.ctx.dist_dir, "root", "python3", "lib",
-                "python3.7", "config-3.7m-darwin"))
-            sh.rm("libpython3.7m.a")
+                "python3.8", "config-3.8-darwin"))
+            sh.rm("libpython3.8.a")
             sh.rm("python.o")
             sh.rm("config.c.in")
             sh.rm("makesetup")
@@ -164,11 +166,11 @@ def reduce_python(self):
             # cleanup pkgconfig and compiled lib
             os.chdir(join(self.ctx.dist_dir, "root", "python3", "lib"))
             sh.rm("-rf", "pkgconfig")
-            sh.rm("-f", "libpython3.7m.a")
+            sh.rm("-f", "libpython3.8.a")
 
             # cleanup python libraries
             os.chdir(join(
-                self.ctx.dist_dir, "root", "python3", "lib", "python3.7"))
+                self.ctx.dist_dir, "root", "python3", "lib", "python3.8"))
             sh.rm("-rf", "wsgiref", "curses", "idlelib", "lib2to3",
                   "ensurepip", "turtledemo", "lib-dynload", "venv",
                   "pydoc_data")
@@ -189,12 +191,12 @@ def reduce_python(self):
             sh.find(".", "-name", "__pycache__", "-type", "d", "-delete")
 
             # create the lib zip
-            logger.info("Create a python3.7.zip")
-            sh.mv("config-3.7m-darwin", "..")
+            logger.info("Create a python3.8.zip")
+            sh.mv("config-3.8-darwin", "..")
             sh.mv("site-packages", "..")
-            sh.zip("-r", "../python37.zip", sh.glob("*"))
+            sh.zip("-r", "../python38.zip", sh.glob("*"))
             sh.rm("-rf", sh.glob("*"))
-            sh.mv("../config-3.7m-darwin", ".")
+            sh.mv("../config-3.8-darwin", ".")
             sh.mv("../site-packages", ".")
         finally:
             os.chdir(oldpwd)

recipes/python3/config.sub.patch

@@ -0,0 +1,31 @@
+diff -Naur Python-3.8.2.orig/.building Python-3.8.2/.building
+--- Python-3.8.2.orig/.building	2020-04-11 23:53:30.000000000 +0200
++++ Python-3.8.2/.building	1970-01-01 01:00:00.000000000 +0100
+@@ -1 +0,0 @@
+-ok
+\ No newline at end of file
+diff -Naur Python-3.8.2.orig/config.sub Python-3.8.2/config.sub
+--- Python-3.8.2.orig/config.sub	2020-04-11 23:53:40.000000000 +0200
++++ Python-3.8.2/config.sub	2020-04-11 23:51:41.000000000 +0200
+@@ -249,7 +249,7 @@
+ 	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
+ 	| am33_2.0 \
+ 	| arc | arceb \
+-	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][arm] \
++	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][armk] \
+ 	| avr | avr32 \
+ 	| ba \
+ 	| be32 | be64 \
+@@ -1524,7 +1524,11 @@
+ 		;;
+ 	-nacl*)
+ 		;;
+-	-ios)
++	-ios*)
++		;;
++	-tvos*)
++		;;
++	-watchos*)
+ 		;;
+ 	-none)
+ 		;;