Skip to content

Commit

Permalink
Rework python3 on debian12
Browse files Browse the repository at this point in the history
- removes shell
- removes support for ctypes
- brings python3 out of experimental for debian12

Signed-off-by: Appu Goundan <[email protected]>
  • Loading branch information
loosebazooka committed Sep 29, 2023
1 parent 0aaf3a9 commit 51f992d
Show file tree
Hide file tree
Showing 6 changed files with 253 additions and 22 deletions.
20 changes: 16 additions & 4 deletions BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -184,18 +184,30 @@ PYTHON3 |= {
for (tag_base, label, user) in PYTHON3_VARIATIONS
}

# python on debian11 builds off of experimental
PYTHON3 |= {
"{REGISTRY}/{PROJECT_ID}/python3-" + distro + ":" + tag_base + "-" + arch: "//experimental/python3:" + label + "_" + user + "_" + arch + "_" + distro
"{REGISTRY}/{PROJECT_ID}/python3-debian11:" + tag_base + "-" + arch: "//experimental/python3:" + label + "_" + user + "_" + arch + "_debian11"
for arch in BASE_ARCHITECTURES
for (tag_base, label, user) in PYTHON3_VARIATIONS
for distro in DISTROS
}

# oci_image_index
PYTHON3 |= {
"{REGISTRY}/{PROJECT_ID}/python3-" + distro + ":" + tag_base: "//experimental/python3:" + label + "_" + user + "_" + distro
"{REGISTRY}/{PROJECT_ID}/python3-debian11:" + tag_base: "//experimental/python3:" + label + "_" + user + "_" + distro
for (tag_base, label, user) in PYTHON3_VARIATIONS
}

# python on debian12 has moved out of experimental
PYTHON3 |= {
"{REGISTRY}/{PROJECT_ID}/python3-debian12:" + tag_base + "-" + arch: "//python3:" + label + "_" + user + "_" + arch + "_debian12"
for arch in BASE_ARCHITECTURES
for (tag_base, label, user) in PYTHON3_VARIATIONS
}

# oci_image_index
PYTHON3 |= {
"{REGISTRY}/{PROJECT_ID}/python3-debian12:" + tag_base: "//python3:" + label + "_" + user + "_" + distro
for (tag_base, label, user) in PYTHON3_VARIATIONS
for distro in DISTROS
}

## NODEJS
Expand Down
25 changes: 7 additions & 18 deletions experimental/python3/BUILD
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
load("@contrib_rules_oci//oci:defs.bzl", "oci_image", "oci_image_index", "oci_tarball", "structure_test")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("//:checksums.bzl", ARCHITECTURES = "BASE_ARCHITECTURES")
load("//base:distro.bzl", "DISTROS")
load("//base:distro.bzl", DISTROS = "LANGUAGE_DISTROS")
load("//base:base.bzl", "NONROOT", "deb_pkg")

package(default_visibility = ["//visibility:public"])

DISTRO_VERSION = {
"debian11": "3.9",
"debian12": "3.11",
}

[
Expand Down Expand Up @@ -63,32 +62,22 @@ DISTRO_VERSION = {
deb_pkg(arch, distro, "zlib1g"),
deb_pkg(arch, distro, "libcom-err2"),
deb_pkg(arch, distro, "libcrypt1"),
deb_pkg(arch, distro, "libffi7"),
deb_pkg(arch, distro, "libgssapi-krb5-2"),
deb_pkg(arch, distro, "libk5crypto3"),
deb_pkg(arch, distro, "libkeyutils1"),
deb_pkg(arch, distro, "libkrb5-3"),
deb_pkg(arch, distro, "libkrb5support0"),
deb_pkg(arch, distro, "libmpdec3"),
deb_pkg(arch, distro, "libnsl2"),
deb_pkg(arch, distro, "libpython3.9-minimal"),
deb_pkg(arch, distro, "libpython3.9-stdlib"),
deb_pkg(arch, distro, "libreadline8"),
deb_pkg(arch, distro, "libtirpc3"),
deb_pkg(arch, distro, "python3.9-minimal"),
"ld_so_" + arch + "_cache.tar",
":python_aliases_%s" % distro,
] + {
# Distro-specific packages
"debian11": [
deb_pkg(arch, distro, "libffi7"),
deb_pkg(arch, distro, "libmpdec3"),
deb_pkg(arch, distro, "libpython3.9-minimal"),
deb_pkg(arch, distro, "libpython3.9-stdlib"),
deb_pkg(arch, distro, "python3.9-minimal"),
],
"debian12": [
deb_pkg(arch, distro, "libffi8"),
deb_pkg(arch, distro, "libpython3.11-minimal"),
deb_pkg(arch, distro, "libpython3.11-stdlib"),
deb_pkg(arch, distro, "python3.11-minimal"),
],
}[distro],
],
)
for mode in [
"",
Expand Down
125 changes: 125 additions & 0 deletions python/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
load("@contrib_rules_oci//oci:defs.bzl", "oci_image", "oci_image_index", "oci_tarball", "structure_test")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("//:checksums.bzl", ARCHITECTURES = "BASE_ARCHITECTURES")

package(default_visibility = ["//visibility:public"])

USERS = [
"root",
"nonroot",
]

DISTROS = {
"debian12",
}

DISTRO_VERSION = {
"debian12": "3.11",
}

[
pkg_tar(
name = "python_aliases_%s" % distro,
symlinks = {
"/usr/bin/python": "/usr/bin/python" + DISTRO_VERSION[distro],
"/usr/bin/python3": "/usr/bin/python" + DISTRO_VERSION[distro],
},
)
for distro in DISTROS
]

[
oci_image_index(
name = ("python3" if (not mode) else mode[1:]) + "_" + user + "_" + distro,
images = [
("python3" if (not mode) else mode[1:]) + "_" + user + "_" + arch + "_" + distro
for arch in ARCHITECTURES
],
)
for mode in [
"",
":debug",
]
for user in USERS
for distro in DISTROS
]

[
oci_image(
name = ("python3" if (not mode) else mode[1:]) + "_" + user + "_" + arch + "_" + distro,
# Based on //cc so that C extensions work properly.
base = "//cc" + (mode if mode else ":cc") + "_" + user + "_" + arch + "_" + distro,
entrypoint = [
"/usr/bin/python" + DISTRO_VERSION[distro],
],
# Use UTF-8 encoding for file system: match modern Linux
env = {"LANG": "C.UTF-8"},
tars = [
deb_pkg(arch, distro, "libbz2-1.0"),
deb_pkg(arch, distro, "libdb5.3"),
deb_pkg(arch, distro, "libexpat1"),
deb_pkg(arch, distro, "liblzma5"),
deb_pkg(arch, distro, "libsqlite3-0"),
deb_pkg(arch, distro, "libuuid1"),
deb_pkg(arch, distro, "libncursesw6"),
deb_pkg(arch, distro, "libtinfo6"),
deb_pkg(arch, distro, "python3-distutils"),
deb_pkg(arch, distro, "zlib1g"),
deb_pkg(arch, distro, "libcom-err2"),
deb_pkg(arch, distro, "libcrypt1"),
deb_pkg(arch, distro, "libgssapi-krb5-2"),
deb_pkg(arch, distro, "libk5crypto3"),
deb_pkg(arch, distro, "libkeyutils1"),
deb_pkg(arch, distro, "libkrb5-3"),
deb_pkg(arch, distro, "libkrb5support0"),
deb_pkg(arch, distro, "libnsl2"),
deb_pkg(arch, distro, "libreadline8"),
deb_pkg(arch, distro, "libtirpc3"),
deb_pkg(arch, distro, "libffi8"),
deb_pkg(arch, distro, "libpython3.11-minimal"),
deb_pkg(arch, distro, "libpython3.11-stdlib"),
deb_pkg(arch, distro, "python3.11-minimal"),
":python_aliases_%s" % distro,
],
)
for mode in [
"",
":debug",
]
for user in USERS
for arch in ARCHITECTURES
for distro in DISTROS
]

[
structure_test(
name = "python3_" + user + "_" + arch + "_" + distro + "_test",
size = "medium",
config = ["testdata/python3.yaml"],
image = ":python3_" + user + "_" + arch + "_" + distro,
tags = [
"manual",
arch,
],
)
for user in USERS
for arch in ARCHITECTURES
for distro in DISTROS
]

# tests for version-specific things
[
structure_test(
name = "version_specific_" + user + "_" + arch + "_" + distro + "_test",
size = "medium",
config = ["testdata/" + distro + ".yaml"],
image = ":python3_" + user + "_" + arch + "_" + distro,
tags = [
"manual",
arch,
],
)
for user in USERS
for arch in ARCHITECTURES
for distro in DISTROS
]
16 changes: 16 additions & 0 deletions python/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Documentation for `gcr.io/distroless/python3`

## Image Contents

This image contains a minimal Linux, Python-based runtime.

Specifically, the image contains everything in the [base image](../../base/README.md), plus:

* Python 3 and its dependencies.
* No shell and no support for ctypes

## Usage

The entrypoint of this image is set to "python", so this image expects users to supply a path to a .py file in the CMD.

See the Python [Hello World](../../examples/python3/) directory for an example.
8 changes: 8 additions & 0 deletions python/testdata/debian12.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
schemaVersion: "1.0.0"
commandTests:
- name: version
command: ["/usr/bin/python3", "--version"]
expectedOutput: ["Python 3.11.2"]
- name: symlink
command: ["/usr/bin/python", "--version"]
expectedOutput: ["Python 3.11.2"]
81 changes: 81 additions & 0 deletions python/testdata/python3.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
schemaVersion: "1.0.0"
commandTests:
- name: hello
command: ["/usr/bin/python3", "-c", "print('Hello World')"]
expectedOutput: ['Hello World']

# ensure there is no shell
- name: no_shell
command: ["/usr/bin/python3", "-c",
"import subprocess, sys; subprocess.check_call(sys.executable + ' -h', shell=True)"]
exitCode: 1

# debian's default python3 includes a partial version of distutils causing virtualenv to fail
# ensure we have the full version so virtualenvs work with distroless
- name: distutils_works
command: ["/usr/bin/python3", "-c", "import distutils.dist"]
exitCode: 0

# file names are UTF-8: default for modern Linux systems
# The \xe9 backslash must be double-escaped to avoid YAML string parsing weirdness
- name: filesystem_utf8
command: ["/usr/bin/python3", "-c", "open(u'h\\xe9llo', 'w'); import sys; print(sys.getfilesystemencoding())"]
expectedOutput: ['utf-8']

# the print function should output UTF-8
- name: print_utf8
command: ["/usr/bin/python3", "-c", "print(u'h\\xe9llo.txt')"]
expectedOutput: ['h\xe9llo']

# import every module installed with the Python package
- name: import_everything
exitCode: 0
expectedOutput: ['FINISHED ENTIRE SCRIPT']
command:
- "/usr/bin/python3"
- "-c"
# multi-line YAML string with Python script that imports all modules that are installed.
# This ensures we have the right native library dependencies.
- |
import pkgutil
skip_modules = frozenset((
# Windows-specific modules
'asyncio.windows_events',
'asyncio.windows_utils',
'ctypes.wintypes',
'distutils._msvccompiler',
'distutils.command.bdist_msi',
'distutils.msvc9compiler',
'encodings.cp65001',
'encodings.mbcs',
'encodings.oem',
'multiprocessing.popen_spawn_win32',
'winreg',
# Python regression tests "for internal use by Python only"
'test',
# calls sys.exit
'unittest.__main__',
'venv.__main__',
# depends on things not installed by default on Debian
'dbm.gnu',
'lib2to3.pgen2.conv',
'turtle',
))
# pass an error handler so the test fails if there are broken standard library packages
def walk_packages_onerror(failed_module_name):
raise Exception('failed to import module: {}'.format(repr(failed_module_name)))
for module_info in pkgutil.walk_packages(onerror=walk_packages_onerror):
module_name = module_info.name
if module_name in skip_modules or module_name.startswith('test.'):
continue
__import__(module_name)
print('imported {}'.format(module_name))
# ensures some module does not exit early (e.g unittest.__main__)
print('FINISHED ENTIRE SCRIPT')

0 comments on commit 51f992d

Please sign in to comment.