diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 235bc78599400e..1f5f7e57dc4859 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -243,6 +243,18 @@ Lib/test/support/interpreters/ @ericsnowcurrently Modules/_xx*interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently +# Android +**/*Android* @mhsmith +**/*android* @mhsmith + +# iOS (but not termios) +**/iOS* @freakboy3742 +**/ios* @freakboy3742 +**/*_iOS* @freakboy3742 +**/*_ios* @freakboy3742 +**/*-iOS* @freakboy3742 +**/*-ios* @freakboy3742 + # WebAssembly /Tools/wasm/ @brettcannon diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 299c02e0944d42..1b2677ff6fe889 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -218,7 +218,7 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: true - # macos-14 is M1 + # macos-14-large is Intel with 12 cores (most parallelism) os-matrix: '["macos-14"]' build_ubuntu: @@ -250,7 +250,7 @@ jobs: build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -316,7 +316,7 @@ jobs: test_hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' @@ -429,7 +429,7 @@ jobs: build_asan: name: 'Address sanitizer' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b37eb727955909..7152cde8f4607c 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -23,7 +23,7 @@ jobs: jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) runs-on: ${{ matrix.runner }} - timeout-minutes: 75 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -41,7 +41,7 @@ jobs: - true - false llvm: - - 16 + - 18 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -75,14 +75,10 @@ jobs: architecture: aarch64 runner: ubuntu-latest compiler: gcc - # These fail because of emulation, not because of the JIT: - exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 runner: ubuntu-latest compiler: clang - # These fail because of emulation, not because of the JIT: - exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection env: CC: ${{ matrix.compiler }} steps: @@ -94,37 +90,43 @@ jobs: - name: Native Windows if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No PGO or tests (yet): - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} - name: Native macOS if: runner.os == 'macOS' run: | + brew update brew install llvm@${{ matrix.llvm }} SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Native Linux if: runner.os == 'Linux' && matrix.architecture == 'x86_64' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }} make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" @@ -139,6 +141,6 @@ jobs: CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ CPP="$CC --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b766785de405d2..35996f237814ba 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/_pyrepl/**" - "Lib/test/libregrtest/**" - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" @@ -35,8 +36,9 @@ jobs: strategy: matrix: target: [ + "Lib/_pyrepl", "Lib/test/libregrtest", - "Tools/build/", + "Tools/build", "Tools/cases_generator", "Tools/clinic", "Tools/jit", diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0800401f4cd113..fb485bd4f82bd2 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -15,6 +15,7 @@ apt-get -yq install \ libgdbm-dev \ libgdbm-compat-dev \ liblzma-dev \ + libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index dabeca8c81ece1..d06a718d199c96 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -22,6 +22,7 @@ jobs: HOMEBREW_NO_INSTALL_CLEANUP: 1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux strategy: fail-fast: false matrix: @@ -49,7 +50,7 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: make -j4 + run: make -j8 - name: Display build info run: make pythoninfo - name: Tests diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 8ddb3b3ada32c2..48bd5b547e8cba 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -29,7 +29,14 @@ jobs: - name: Install Dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh - sudo apt install -y clang + # Install clang-18 + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 + sudo update-alternatives --set clang /usr/bin/clang-18 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-18 # Reduce ASLR to avoid TSAN crashing sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index e6fbaaf74c5a4b..fa450ed3376321 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -12,11 +12,12 @@ jobs: build_ubuntu_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: FORCE_COLOR: 1 OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux steps: - uses: actions/checkout@v4 - name: Register gcc problem matcher diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 4a509a8acfee96..c389fe9e173b38 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -9,7 +9,7 @@ jobs: build_wasi_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: WASMTIME_VERSION: 18.0.3 WASI_SDK_VERSION: 21 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 663a11897d98e2..fde9d9149bf62b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,14 @@ repos: args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.4.2 + hooks: + - id: black + name: Run Black on Tools/jit/ + files: ^Tools/jit/ + language_version: python3.12 + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: diff --git a/Android/README.md b/Android/README.md index 5ed186e06e3951..f5f463ca116589 100644 --- a/Android/README.md +++ b/Android/README.md @@ -22,12 +22,25 @@ you don't already have the SDK, here's how to install it: `android-sdk/cmdline-tools/latest`. * `export ANDROID_HOME=/path/to/android-sdk` +The `android.py` script also requires the following commands to be on the `PATH`: + +* `curl` +* `java` +* `tar` +* `unzip` + ## Building -Building for Android requires doing a cross-build where you have a "build" -Python to help produce an Android build of CPython. This procedure has been -tested on Linux and macOS. +Python can be built for Android on any POSIX platform supported by the Android +development tools, which currently means Linux or macOS. This involves doing a +cross-build where you use a "build" Python (for your development machine) to +help produce a "host" Python for Android. + +First, make sure you have all the usual tools and libraries needed to build +Python for your development machine. The only Android tool you need to install +is the command line tools package above: the build script will download the +rest. The easiest way to do a build is to use the `android.py` script. You can either have it perform the entire build process from start to finish in one step, or @@ -43,9 +56,10 @@ The discrete steps for building via `android.py` are: ./android.py make-host HOST ``` -To see the possible values of HOST, run `./android.py configure-host --help`. +`HOST` identifies which architecture to build. To see the possible values, run +`./android.py configure-host --help`. -Or to do it all in a single command, run: +To do all steps in a single command, run: ```sh ./android.py build HOST @@ -62,3 +76,22 @@ call. For example, if you want a pydebug build that also caches the results from ```sh ./android.py build HOST -- -C --with-pydebug ``` + + +## Testing + +To run the Python test suite on Android: + +* Install Android Studio, if you don't already have it. +* Follow the instructions in the previous section to build all supported + architectures. +* Run `./android.py setup-testbed` to download the Gradle wrapper. +* Open the `testbed` directory in Android Studio. +* In the *Device Manager* dock, connect a device or start an emulator. + Then select it from the drop-down list in the toolbar. +* Click the "Run" button in the toolbar. +* The testbed app displays nothing on screen while running. To see its output, + open the [Logcat window](https://developer.android.com/studio/debug/logcat). + +To run specific tests, or pass any other arguments to the test suite, edit the +command line in testbed/app/src/main/python/main.py. diff --git a/Android/android-env.sh b/Android/android-env.sh index 3ce3e035cfb8fe..545d559d93ab36 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -61,6 +61,12 @@ done export CFLAGS="" export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment" +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + # Many packages get away with omitting -lm on Linux, but Android is stricter. LDFLAGS="$LDFLAGS -lm" diff --git a/Android/android.py b/Android/android.py index 5c57e53c415d2b..0a1393e61ddb0e 100755 --- a/Android/android.py +++ b/Android/android.py @@ -7,8 +7,9 @@ import subprocess import sys import sysconfig -from os.path import relpath +from os.path import basename, relpath from pathlib import Path +from tempfile import TemporaryDirectory SCRIPT_NAME = Path(__file__).name CHECKOUT = Path(__file__).resolve().parent.parent @@ -102,11 +103,17 @@ def unpack_deps(host): for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1", "sqlite-3.45.1-0", "xz-5.4.6-0"]: filename = f"{name_ver}-{host}.tar.gz" - run(["wget", f"{deps_url}/{name_ver}/{filename}"]) + download(f"{deps_url}/{name_ver}/{filename}") run(["tar", "-xf", filename]) os.remove(filename) +def download(url, target_dir="."): + out_path = f"{target_dir}/{basename(url)}" + run(["curl", "-Lf", "-o", out_path, url]) + return out_path + + def configure_host_python(context): host_dir = subdir(context.host, clean=context.clean) @@ -160,6 +167,30 @@ def clean_all(context): delete_if_exists(CROSS_BUILD_DIR) +# To avoid distributing compiled artifacts without corresponding source code, +# the Gradle wrapper is not included in the CPython repository. Instead, we +# extract it from the Gradle release. +def setup_testbed(context): + ver_long = "8.7.0" + ver_short = ver_long.removesuffix(".0") + testbed_dir = CHECKOUT / "Android/testbed" + + for filename in ["gradlew", "gradlew.bat"]: + out_path = download( + f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}", + testbed_dir) + os.chmod(out_path, 0o755) + + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + os.chdir(temp_dir) + bin_zip = download( + f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip") + outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar" + run(["unzip", bin_zip, outer_jar]) + run(["unzip", "-o", "-d", f"{testbed_dir}/gradle/wrapper", outer_jar, + "gradle-wrapper.jar"]) + + def main(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") @@ -173,8 +204,11 @@ def main(): help="Run `configure` for Android") make_host = subcommands.add_parser("make-host", help="Run `make` for Android") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") + subcommands.add_parser( + "clean", help="Delete the cross-build directory") + subcommands.add_parser( + "setup-testbed", help="Download the testbed Gradle wrapper") + for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", @@ -194,7 +228,8 @@ def main(): "configure-host": configure_host_python, "make-host": make_host_python, "build": build_all, - "clean": clean_all} + "clean": clean_all, + "setup-testbed": setup_testbed} dispatch[context.subcommand](context) diff --git a/Android/testbed/.gitignore b/Android/testbed/.gitignore new file mode 100644 index 00000000000000..b9a7d611c943cf --- /dev/null +++ b/Android/testbed/.gitignore @@ -0,0 +1,21 @@ +# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`. +/gradlew +/gradlew.bat +/gradle/wrapper/gradle-wrapper.jar + +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/deploymentTargetDropdown.xml +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/Android/testbed/app/.gitignore b/Android/testbed/app/.gitignore new file mode 100644 index 00000000000000..42afabfd2abebf --- /dev/null +++ b/Android/testbed/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts new file mode 100644 index 00000000000000..7690d3fd86b2fd --- /dev/null +++ b/Android/testbed/app/build.gradle.kts @@ -0,0 +1,129 @@ +import com.android.build.api.variant.* + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") +} + +val PYTHON_DIR = File(projectDir, "../../..").canonicalPath +val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build" +val ABIS = mapOf( + "arm64-v8a" to "aarch64-linux-android", + "x86_64" to "x86_64-linux-android", +) + +val PYTHON_VERSION = File("$PYTHON_DIR/Include/patchlevel.h").useLines { + for (line in it) { + val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line) + if (match != null) { + return@useLines match.groupValues[1] + } + } + throw GradleException("Failed to find Python version") +} + + +android { + namespace = "org.python.testbed" + compileSdk = 34 + + defaultConfig { + applicationId = "org.python.testbed" + minSdk = 21 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + ndk.abiFilters.addAll(ABIS.keys) + externalNativeBuild.cmake.arguments( + "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR", + "-DPYTHON_VERSION=$PYTHON_VERSION") + } + + externalNativeBuild.cmake { + path("src/main/c/CMakeLists.txt") + } + + // Set this property to something non-empty, otherwise it'll use the default + // list, which ignores asset directories beginning with an underscore. + aaptOptions.ignoreAssetsPattern = ".git" + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") +} + + +// Create some custom tasks to copy Python and its standard library from +// elsewhere in the repository. +androidComponents.onVariants { variant -> + generateTask(variant, variant.sources.assets!!) { + into("python") { + for (triplet in ABIS.values) { + for (subDir in listOf("include", "lib")) { + into(subDir) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/$subDir") + include("python$PYTHON_VERSION/**") + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + } + } + into("lib/python$PYTHON_VERSION") { + // Uncomment this to pick up edits from the source directory + // without having to rerun `make install`. + // from("$PYTHON_DIR/Lib") + // duplicatesStrategy = DuplicatesStrategy.INCLUDE + + into("site-packages") { + from("$projectDir/src/main/python") + } + } + } + exclude("**/__pycache__") + } + + generateTask(variant, variant.sources.jniLibs!!) { + for ((abi, triplet) in ABIS.entries) { + into(abi) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib") + include("libpython*.*.so") + include("lib*_python.so") + } + } + } +} + + +fun generateTask( + variant: ApplicationVariant, directories: SourceDirectories, + configure: GenerateTask.() -> Unit +) { + val taskName = "generate" + + listOf(variant.name, "Python", directories.name) + .map { it.replaceFirstChar(Char::uppercase) } + .joinToString("") + + directories.addGeneratedSourceDirectory( + tasks.register(taskName) { + into(outputDir) + configure() + }, + GenerateTask::outputDir) +} + + +// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. +abstract class GenerateTask: Sync() { + @get:OutputDirectory + abstract val outputDir: DirectoryProperty +} diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Android/testbed/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000000000..2be8a82d426099 --- /dev/null +++ b/Android/testbed/app/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt new file mode 100644 index 00000000000000..1d5df9a73465b6 --- /dev/null +++ b/Android/testbed/app/src/main/c/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.4.1) +project(testbed) + +set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix) +include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION}) +link_directories(${PREFIX_DIR}/lib) +link_libraries(log python${PYTHON_VERSION}) + +add_library(main_activity SHARED main_activity.c) diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c new file mode 100644 index 00000000000000..73aba4164d000f --- /dev/null +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -0,0 +1,147 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +static void throw_runtime_exception(JNIEnv *env, const char *message) { + (*env)->ThrowNew( + env, + (*env)->FindClass(env, "java/lang/RuntimeException"), + message); +} + + +// --- Stdio redirection ------------------------------------------------------ + +// Most apps won't need this, because the Python-level sys.stdout and sys.stderr +// are redirected to the Android logcat by Python itself. However, in the +// testbed it's useful to redirect the native streams as well, to debug problems +// in the Python startup or redirection process. +// +// Based on +// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp + +typedef struct { + FILE *file; + int fd; + android_LogPriority priority; + char *tag; + int pipe[2]; +} StreamInfo; + +static StreamInfo STREAMS[] = { + {stdout, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}}, + {stderr, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}}, + {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}}, +}; + +// The maximum length of a log message in bytes, including the level marker and +// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +// longer than this will be be truncated by logcat. This limit has already been +// reduced at least once in the history of Android (from 4076 to 4068 between API +// level 23 and 26), so leave some headroom. +static const int MAX_BYTES_PER_WRITE = 4000; + +static void *redirection_thread(void *arg) { + StreamInfo *si = (StreamInfo*)arg; + ssize_t read_size; + char buf[MAX_BYTES_PER_WRITE]; + while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) { + buf[read_size] = '\0'; /* add null-terminator */ + __android_log_write(si->priority, si->tag, buf); + } + return 0; +} + +static char *redirect_stream(StreamInfo *si) { + /* make the FILE unbuffered, to ensure messages are never lost */ + if (setvbuf(si->file, 0, _IONBF, 0)) { + return "setvbuf"; + } + + /* create the pipe and redirect the file descriptor */ + if (pipe(si->pipe)) { + return "pipe"; + } + if (dup2(si->pipe[1], si->fd) == -1) { + return "dup2"; + } + + /* start the logging thread */ + pthread_t thr; + if ((errno = pthread_create(&thr, 0, redirection_thread, si))) { + return "pthread_create"; + } + if ((errno = pthread_detach(thr))) { + return "pthread_detach"; + } + return 0; +} + +JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_redirectStdioToLogcat( + JNIEnv *env, jobject obj +) { + for (StreamInfo *si = STREAMS; si->file; si++) { + char *error_prefix; + if ((error_prefix = redirect_stream(si))) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); + return; + } + } +} + + +// --- Python intialization ---------------------------------------------------- + +static PyStatus set_config_string( + JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value +) { + const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); + PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); + (*env)->ReleaseStringUTFChars(env, value, value_utf8); + return status; +} + +static void throw_status(JNIEnv *env, PyStatus status) { + throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); +} + +JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_runPython( + JNIEnv *env, jobject obj, jstring home, jstring runModule +) { + PyConfig config; + PyStatus status; + PyConfig_InitIsolatedConfig(&config); + + status = set_config_string(env, &config, &config.home, home); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + status = set_config_string(env, &config, &config.run_module, runModule); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. + config.install_signal_handlers = 1; + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + Py_RunMain(); +} diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt new file mode 100644 index 00000000000000..5a590d5d04e954 --- /dev/null +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -0,0 +1,61 @@ +package org.python.testbed + +import android.os.* +import android.system.Os +import android.widget.TextView +import androidx.appcompat.app.* +import java.io.* + +class MainActivity : AppCompatActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + + // Python needs this variable to help it find the temporary directory, + // but Android only sets it on API level 33 and later. + Os.setenv("TMPDIR", cacheDir.toString(), false) + + val pythonHome = extractAssets() + System.loadLibrary("main_activity") + redirectStdioToLogcat() + runPython(pythonHome.toString(), "main") + findViewById(R.id.tvHello).text = "Python complete" + } + + private fun extractAssets() : File { + val pythonHome = File(filesDir, "python") + if (pythonHome.exists() && !pythonHome.deleteRecursively()) { + throw RuntimeException("Failed to delete $pythonHome") + } + extractAssetDir("python", filesDir) + return pythonHome + } + + private fun extractAssetDir(path: String, targetDir: File) { + val names = assets.list(path) + ?: throw RuntimeException("Failed to list $path") + val targetSubdir = File(targetDir, path) + if (!targetSubdir.mkdirs()) { + throw RuntimeException("Failed to create $targetSubdir") + } + + for (name in names) { + val subPath = "$path/$name" + val input: InputStream + try { + input = assets.open(subPath) + } catch (e: FileNotFoundException) { + extractAssetDir(subPath, targetDir) + continue + } + input.use { + File(targetSubdir, name).outputStream().use { output -> + input.copyTo(output) + } + } + } + } + + private external fun redirectStdioToLogcat() + private external fun runPython(home: String, runModule: String) +} \ No newline at end of file diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py new file mode 100644 index 00000000000000..a1b6def34ede81 --- /dev/null +++ b/Android/testbed/app/src/main/python/main.py @@ -0,0 +1,17 @@ +import runpy +import signal +import sys + +# Some tests use SIGUSR1, but that's blocked by default in an Android app in +# order to make it available to `sigwait` in the "Signal Catcher" thread. That +# thread's functionality is only relevant to the JVM ("forcing GC (no HPROF) and +# profile save"), so disabling it should not weaken the tests. +signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) + +# To run specific tests, or pass any other arguments to the test suite, edit +# this command line. +sys.argv[1:] = [ + "--use", "all,-cpu", + "--verbose3", +] +runpy.run_module("test") diff --git a/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 00000000000000..741d6580d60e05 Binary files /dev/null and b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Android/testbed/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000000000..21398609ec9c78 --- /dev/null +++ b/Android/testbed/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/res/values/strings.xml b/Android/testbed/app/src/main/res/values/strings.xml new file mode 100644 index 00000000000000..352d2f9e885a2a --- /dev/null +++ b/Android/testbed/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + Python testbed + \ No newline at end of file diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts new file mode 100644 index 00000000000000..53f4a67287fcc5 --- /dev/null +++ b/Android/testbed/build.gradle.kts @@ -0,0 +1,5 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id("com.android.application") version "8.2.2" apply false + id("org.jetbrains.kotlin.android") version "1.9.22" apply false +} \ No newline at end of file diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties new file mode 100644 index 00000000000000..3c5031eb7d63f7 --- /dev/null +++ b/Android/testbed/gradle.properties @@ -0,0 +1,23 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000000..2dc3339a3ef213 --- /dev/null +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Feb 19 20:29:06 GMT 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/Android/testbed/settings.gradle.kts b/Android/testbed/settings.gradle.kts new file mode 100644 index 00000000000000..5e08773e02450f --- /dev/null +++ b/Android/testbed/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "Python testbed" +include(":app") + \ No newline at end of file diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 6bb1e9b5803b58..82e0980ad753d0 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -120,12 +120,19 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute. + If the frame refers to a function or comprehension, this returns + a write-through proxy object that allows modifying the locals. + In all other cases (classes, modules) it returns the :class:`dict` + representing the frame locals directly. Return a :term:`strong reference`. .. versionadded:: 3.11 + .. versionchanged:: 3.13 + Return a proxy object for functions and comprehensions. + .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 9a8f1507b3f4cc..ba56b03c6ac8e7 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -25,3 +25,4 @@ document the API functions in detail. memory.rst objimpl.rst apiabiversion.rst + monitoring.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 8725ce085145aa..9e118d4f36145f 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1904,6 +1904,58 @@ Python-level trace functions in previous versions. .. versionadded:: 3.12 +Reference tracing +================= + +.. versionadded:: 3.13 + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + +.. versionadded:: 3.13 + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Not that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. The GIL will be held every time + the tracer function is called. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 .. _advanced-debugging: diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 47a8fbb2cd9c97..5195f6cccfe9df 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1250,8 +1250,11 @@ PyConfig If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` for more information. - Set by :option:`-X perf <-X>` command line option and by the - :envvar:`PYTHONPERFSUPPORT` environment variable. + Set by :option:`-X perf <-X>` command-line option and by the + :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support + with stack pointers and :option:`-X perf_jit <-X>` command-line option + and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf + support with DWARF JIT information. Default: ``-1``. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 1eb8f191c3ca32..522c028cfb8d40 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -390,7 +390,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Usage example:: int32_t value; - Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1); + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1); if (bytes < 0) { // Failed. A Python exception was set with the reason. return NULL; @@ -418,7 +418,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. called twice: first to determine the buffer size, then to fill it:: // Ask how much space we need. - Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1); + Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1); if (expected < 0) { // Failed. A Python exception was set with the reason. return NULL; @@ -430,7 +430,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. return NULL; } // Safely get the entire value. - Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1); + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1); if (bytes < 0) { // Exception has been set. free(bignum); return NULL; diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 979b22261efa3b..6fe1ce9e994832 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -411,6 +411,31 @@ The available slot types are: .. versionadded:: 3.12 +.. c:macro:: Py_mod_gil + + Specifies one of the following values: + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :envvar:`PYTHON_GIL` and :option:`-X gil <-X>` for more detail. + + Multiple ``Py_mod_gil`` slots may not be specified in one module definition. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + .. versionadded: 3.13 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions @@ -609,6 +634,19 @@ state: .. versionadded:: 3.9 +.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) + + Indicate that *module* does or does not support running without the global + interpreter lock (GIL), using one of the values from + :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization + function. If this function is not called during module initialization, the + import machinery assumes the module does not support running without the + GIL. This function is only available in Python builds configured with + :option:`--disable-gil`. + Return ``-1`` on error, ``0`` on success. + + .. versionadded:: 3.13 + Module lookup ^^^^^^^^^^^^^ diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst new file mode 100644 index 00000000000000..763ec8ef761e4e --- /dev/null +++ b/Doc/c-api/monitoring.rst @@ -0,0 +1,164 @@ +.. highlight:: c + +.. _monitoring: + +Monitorong C API +================ + +Added in version 3.13. + +An extension may need to interact with the event monitoring system. Subscribing +to events and registering callbacks can be done via the Python API exposed in +:mod:`sys.monitoring`. + +Generating Execution Events +=========================== + +The functions below make it possible for an extension to fire monitoring +events as it emulates the execution of Python code. Each of these functions +accepts a ``PyMonitoringState`` struct which contains concise information +about the activation state of events, as well as the event arguments, which +include a ``PyObject*`` representing the code object, the instruction offset +and sometimes additional, event-specific arguments (see :mod:`sys.monitoring` +for details about the signatures of the different event callbacks). +The ``codelike`` argument should be an instance of :class:`types.CodeType` +or of a type that emulates it. + +The VM disables tracing when firing an event, so there is no need for user +code to do that. + +Monitoring functions should not be called with an exception set, +except those listed below as working with the current exception. + +.. c:type:: PyMonitoringState + + Representation of the state of an event type. It is allocated by the user + while its contents are maintained by the monitoring API functions described below. + + +All of the functions below return 0 on success and -1 (with an exception set) on error. + +See :mod:`sys.monitoring` for descriptions of the events. + +.. c:function:: int PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_START`` event. + + +.. c:function:: int PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_RESUME`` event. + + +.. c:function:: int PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_YIELD`` event. + + +.. c:function:: int PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* callable, PyObject *arg0) + + Fire a ``CALL`` event. + + +.. c:function:: int PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, int lineno) + + Fire a ``LINE`` event. + + +.. c:function:: int PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``JUMP`` event. + + +.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH`` event. + + +.. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) + + Fire a ``C_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_THROW`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``C_RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RERAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire an ``EXCEPTION_HANDLED`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_UNWIND`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``STOP_ITERATION`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +Managing the Monitoring State +----------------------------- + +Monitoring states can be managed with the help of monitoring scopes. A scope +would typically correspond to a python function. + +.. :c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) + + Enter a monitored scope. ``event_types`` is an array of the event IDs for + events that may be fired from the scope. For example, the ID of a ``PY_START`` + event is the value ``PY_MONITORING_EVENT_PY_START``, which is numerically equal + to the base-2 logarithm of ``sys.monitoring.events.PY_START``. + ``state_array`` is an array with a monitoring state entry for each event in + ``event_types``, it is allocated by the user but populated by + ``PyMonitoring_EnterScope`` with information about the activation state of + the event. The size of ``event_types`` (and hence also of ``state_array``) + is given in ``length``. + + The ``version`` argument is a pointer to a value which should be allocated + by the user together with ``state_array`` and initialized to 0, + and then set only by ``PyMonitoring_EnterScope`` itelf. It allows this + function to determine whether event states have changed since the previous call, + and to return quickly if they have not. + + The scopes referred to here are lexical scopes: a function, class or method. + ``PyMonitoring_EnterScope`` should be called whenever the lexical scope is + entered. Scopes can be reentered, reusing the same *state_array* and *version*, + in situations like when emulating a recursive Python function. When a code-like's + execution is paused, such as when emulating a generator, the scope needs to + be exited and re-entered. + + +.. :c:function:: int PyMonitoring_ExitScope(void) + + Exit the last scope that was entered with ``PyMonitoring_EnterScope``. diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst index 7791cdb1781055..5cfdef71b3e191 100644 --- a/Doc/c-api/time.rst +++ b/Doc/c-api/time.rst @@ -72,6 +72,35 @@ with the :term:`GIL` held. See :func:`time.time` for details important on this clock. +Raw Clock Functions +------------------- + +Similar to clock functions, but don't set an exception on error and don't +require the caller to hold the GIL. + +On success, the functions return ``0``. + +On failure, they set ``*result`` to ``0`` and return ``-1``, *without* setting +an exception. To get the cause of the error, acquire the GIL and call the +regular (non-``Raw``) function. Note that the regular function may succeed after +the ``Raw`` one failed. + +.. c:function:: int PyTime_MonotonicRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Monotonic`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_PerfCounterRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_PerfCounter`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_TimeRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Time`, + but don't set an exception on error and don't require holding the GIL. + + Conversion functions -------------------- diff --git a/Doc/conf.py b/Doc/conf.py index 73abe8276f29fe..86371d17ae742a 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -131,6 +131,7 @@ ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), + ('c:type', 'int32_t'), ('c:type', 'int64_t'), ('c:type', 'intmax_t'), ('c:type', 'off_t'), diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 8c8a378f52bd5d..76a035f194d911 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -188,6 +188,9 @@ function,PyEval_EvalFrame,3.2,, function,PyEval_EvalFrameEx,3.2,, function,PyEval_GetBuiltins,3.2,, function,PyEval_GetFrame,3.2,, +function,PyEval_GetFrameBuiltins,3.13,, +function,PyEval_GetFrameGlobals,3.13,, +function,PyEval_GetFrameLocals,3.13,, function,PyEval_GetFuncDesc,3.2,, function,PyEval_GetFuncName,3.2,, function,PyEval_GetGlobals,3.2,, diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 05ac3edb63b65d..2846f77feb112d 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -9,13 +9,14 @@ Glossary .. glossary:: ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. + The default Python prompt of the :term:`interactive` shell. Often + seen for code examples which can be executed interactively in the + interpreter. ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering the + * The default Python prompt of the :term:`interactive` shell when entering the code for an indented code block, when within a pair of matching left and right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. @@ -620,7 +621,8 @@ Glossary execute them and see their results. Just launch ``python`` with no arguments (possibly by selecting it from your computer's main menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). + modules and packages (remember ``help(x)``). For more on interactive + mode, see :ref:`tut-interac`. interpreted Python is an interpreted language, as opposed to a compiled one, @@ -1084,6 +1086,10 @@ Glossary See also :term:`namespace package`. + REPL + An acronym for the "read–eval–print loop", another name for the + :term:`interactive` interpreter shell. + __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index bb1c00e0aa51d5..06459d1b222964 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -162,12 +162,12 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 python script.py + $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: - $ python -X perf script.py + $ perf record -F 9999 -g -o perf.data python -X perf script.py $ perf report -g -i perf.data Example, using the :mod:`sys` APIs in file :file:`example.py`: @@ -184,7 +184,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: ...then:: - $ python ./example.py + $ perf record -F 9999 -g -o perf.data python ./example.py $ perf report -g -i perf.data @@ -205,3 +205,62 @@ You can check if your system has been compiled with this flag by running:: If you don't see any output it means that your interpreter has not been compiled with frame pointers and therefore it may not be able to show Python functions in the output of ``perf``. + + +How to work without frame pointers +---------------------------------- + +If you are working with a Python interpreter that has been compiled without +frame pointers, you can still use the ``perf`` profiler, but the overhead will be +a bit higher because Python needs to generate unwinding information for every +Python function call on the fly. Additionally, ``perf`` will take more time to +process the data because it will need to use the DWARF debugging information to +unwind the stack and this is a slow process. + +To enable this mode, you can use the environment variable +:envvar:`PYTHON_PERF_JIT_SUPPORT` or the :option:`-X perf_jit <-X>` option, +which will enable the JIT mode for the ``perf`` profiler. + +.. note:: + + Due to a bug in the ``perf`` tool, only ``perf`` versions higher than v6.8 + will work with the JIT mode. The fix was also backported to the v6.7.2 + version of the tool. + + Note that when checking the version of the ``perf`` tool (which can be done + by running ``perf version``) you must take into account that some distros + add some custom version numbers including a ``-`` character. This means + that ``perf 6.7-3`` is not necessarily ``perf 6.7.3``. + +When using the perf JIT mode, you need an extra step before you can run ``perf +report``. You need to call the ``perf inject`` command to inject the JIT +information into the ``perf.data`` file.:: + + $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +or using the environment variable:: + + $ PYTHON_PERF_JIT_SUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +``perf inject --jit`` command will read ``perf.data``, +automatically pick up the perf dump file that Python creates (in +``/tmp/perf-$PID.dump``), and then create ``perf.jit.data`` which merges all the +JIT information together. It should also create a lot of ``jitted-XXXX-N.so`` +files in the current directory which are ELF images for all the JIT trampolines +that were created by Python. + +.. warning:: + Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take + snapshots of the stack of the process being profiled and save the + information in the ``perf.data`` file. By default the size of the stack dump + is 8192 bytes but the user can change the size by passing the size after + comma like ``--call-graph dwarf,4096``. The size of the stack dump is + important because if the size is too small ``perf`` will not be able to + unwind the stack and the output will be incomplete. On the other hand, if + the size is too big, then ``perf`` won't be able to sample the process as + frequently as it would like as the overhead will be higher. + diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 7f54a410881514..33a2a7ea89ea07 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -594,5 +594,5 @@ This document was reviewed and revised by John Lee. scripts with a localhost server, I have to prevent urllib from using the proxy. .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. + `_. diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index 2fc1570e654db6..aa2a5c7cda52aa 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -53,7 +53,7 @@ def main(): # Guess the content type based on the file's extension. Encoding # will be ignored, although we should check for simple things like # gzip'd or compressed files. - ctype, encoding = mimetypes.guess_type(path) + ctype, encoding = mimetypes.guess_file_type(path) if ctype is None or encoding is not None: # No guess could be made, or the file is encoded (compressed), so # use a generic bag-of-bits type. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 09f2a404786fb6..02dc7c86082502 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -61,7 +61,7 @@ Node classes .. attribute:: _fields - Each concrete class has an attribute :attr:`_fields` which gives the names + Each concrete class has an attribute :attr:`!_fields` which gives the names of all child nodes. Each instance of a concrete class has one attribute for each child node, @@ -74,6 +74,18 @@ Node classes as Python lists. All possible attributes must be present and have valid values when compiling an AST with :func:`compile`. + .. attribute:: _field_types + + The :attr:`!_field_types` attribute on each concrete class is a dictionary + mapping field names (as also listed in :attr:`_fields`) to their types. + + .. doctest:: + + >>> ast.TypeVar._field_types + {'name': , 'bound': ast.expr | None, 'default_value': ast.expr | None} + + .. versionadded:: 3.13 + .. attribute:: lineno col_offset end_lineno @@ -1748,15 +1760,17 @@ Type parameters :ref:`Type parameters ` can exist on classes, functions, and type aliases. -.. class:: TypeVar(name, bound) +.. class:: TypeVar(name, bound, default_value) - A :class:`typing.TypeVar`. ``name`` is the name of the type variable. - ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. + A :class:`typing.TypeVar`. *name* is the name of the type variable. + *bound* is the bound or constraints, if any. If *bound* is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. *default_value* + is the default value; if the :class:`!TypeVar` has no default, this + attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[T: int] = list[T]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[T: int = bool] = list[T]"), indent=4)) Module( body=[ TypeAlias( @@ -1764,7 +1778,8 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()))], + bound=Name(id='int', ctx=Load()), + default_value=Name(id='bool', ctx=Load()))], value=Subscript( value=Name(id='list', ctx=Load()), slice=Name(id='T', ctx=Load()), @@ -1772,19 +1787,30 @@ aliases. .. versionadded:: 3.12 -.. class:: ParamSpec(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: ParamSpec(name, default_value) - A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + A :class:`typing.ParamSpec`. *name* is the name of the parameter specification. + *default_value* is the default value; if the :class:`!ParamSpec` has no default, + this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[**P] = Callable[P, int]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[**P = (int, str)] = Callable[P, int]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - ParamSpec(name='P')], + ParamSpec( + name='P', + default_value=Tuple( + elts=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + ctx=Load()))], value=Subscript( value=Name(id='Callable', ctx=Load()), slice=Tuple( @@ -1796,19 +1822,26 @@ aliases. .. versionadded:: 3.12 -.. class:: TypeVarTuple(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: TypeVarTuple(name, default_value) - A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + A :class:`typing.TypeVarTuple`. *name* is the name of the type variable tuple. + *default_value* is the default value; if the :class:`!TypeVarTuple` has no + default, this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[*Ts] = tuple[*Ts]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[*Ts = ()] = tuple[*Ts]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple(name='Ts')], + TypeVarTuple( + name='Ts', + default_value=Tuple(ctx=Load()))], value=Subscript( value=Name(id='tuple', ctx=Load()), slice=Tuple( @@ -1821,6 +1854,9 @@ aliases. .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *default_value* parameter. + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 3d300c37419f13..188649a2968df8 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1386,7 +1386,7 @@ Task Object with :meth:`uncancel`. :class:`TaskGroup` context managers use :func:`uncancel` in a similar fashion. - If end-user code is, for some reason, suppresing cancellation by + If end-user code is, for some reason, suppressing cancellation by catching :exc:`CancelledError`, it needs to call this method to remove the cancellation state. diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 7bf4308a96d0f5..504fbd0464515b 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -240,6 +240,9 @@ The :mod:`bdb` module also defines two classes: Called from :meth:`dispatch_call` if a break might stop inside the called function. + *argument_list* is not used anymore and will always be ``None``. + The argument is kept for backwards compatibility. + .. method:: user_line(frame) Called from :meth:`dispatch_line` when either :meth:`stop_here` or @@ -286,6 +289,10 @@ The :mod:`bdb` module also defines two classes: Start debugging from *frame*. If *frame* is not specified, debugging starts from caller's frame. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. + .. method:: set_continue() Stop only at breakpoints or when finished. If there are no breakpoints, diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index b2379befeffcba..5174515ffc23ed 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -36,6 +36,7 @@ The following modules have a command-line interface. * :mod:`pyclbr` * :mod:`pydoc` * :mod:`quopri` +* :ref:`random ` * :mod:`runpy` * :ref:`site ` * :ref:`sqlite3 ` diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index f868799e7f5c10..19b89ad79c7414 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -134,7 +134,7 @@ The class can be used to simulate nested scopes and is useful in templating. :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe - `_ has options to control + `_ has options to control whether writes and other mutations apply only to the first mapping or to any mapping in the chain. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 4ee7820585d3a2..d17468023c6de1 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -156,8 +156,10 @@ The :mod:`csv` module defines the following classes: The *fieldnames* parameter is a :term:`sequence`. If *fieldnames* is omitted, the values in the first row of file *f* will be used as the - fieldnames. Regardless of how the fieldnames are determined, the - dictionary preserves their original ordering. + fieldnames and will be omitted from the results. If + *fieldnames* is provided, they will be used and the first row will be + included in the results. Regardless of how the fieldnames are determined, + the dictionary preserves their original ordering. If a row has more fields than fieldnames, the remaining data is put in a list and stored with the fieldname specified by *restkey* (which defaults diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b1b5e04c74b066..b20b2027cfcc93 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -635,7 +635,7 @@ Notes: .. versionchanged:: 3.13 Comparison between :class:`.datetime` object and an instance of the :class:`date` subclass that is not a :class:`!datetime` subclass - no longer coverts the latter to :class:`!date`, ignoring the time part + no longer converts the latter to :class:`!date`, ignoring the time part and the time zone. The default behavior can be changed by overriding the special comparison methods in subclasses. @@ -1257,7 +1257,7 @@ Supported operations: .. versionchanged:: 3.13 Comparison between :class:`.datetime` object and an instance of the :class:`date` subclass that is not a :class:`!datetime` subclass - no longer coverts the latter to :class:`!date`, ignoring the time part + no longer converts the latter to :class:`!date`, ignoring the time part and the time zone. The default behavior can be changed by overriding the special comparison methods in subclasses. @@ -1735,7 +1735,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``fold in [0, 1]``. If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to 0 except *tzinfo*, which defaults to :const:`None`. + default to 0 except *tzinfo*, which defaults to ``None``. Class attributes: diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index d45e46448207a4..6a34f2e4d7f5e1 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -631,7 +631,7 @@ If you want to know how to change the first sequence into the second, use work. * `Simple version control recipe - `_ for a small application + `_ for a small application built with :class:`SequenceMatcher`. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 21ac2c87a1859e..e255fad55e4a25 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -104,7 +104,7 @@ The following options are accepted: Show offsets of instructions. If :file:`infile` is specified, its disassembled code will be written to stdout. -Otherwise, disassembly is performed on compiled source code recieved from stdin. +Otherwise, disassembly is performed on compiled source code received from stdin. Bytecode analysis ----------------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 00f617e5ffc5e7..21f41b73086c98 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -402,13 +402,15 @@ Data Types in the member assignment will be passed; e.g. >>> from enum import Enum - >>> class MyIntEnum(Enum): - ... SEVENTEEN = '1a', 16 + >>> class MyIntEnum(int, Enum): + ... TWENTYSIX = '1a', 16 - results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + results in the call ``int('1a', 16)`` and a value of ``26`` for the member. - .. note:: When writing a custom ``__new__``, do not use ``super().__new__`` -- - call the appropriate ``__new__`` instead. + .. note:: + + When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. .. method:: Enum.__repr__(self) @@ -859,9 +861,15 @@ Supported ``_sunder_`` names For :class:`Flag` classes the next value chosen will be the next highest power-of-two. +- While ``_sunder_`` names are generally reserved for the further development + of the :class:`Enum` class and can not be used, some are explicitly allowed: + + - ``_repr_*`` (e.g. ``_repr_html_``), as used in `IPython's rich display`_ + .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_``, ``_repr_*`` +.. _`IPython's rich display`: https://ipython.readthedocs.io/en/stable/config/integrating.html#rich-display --------------- diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e598ef423de497..0c7ef67774cd05 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -524,11 +524,11 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, globals=None, locals=None) +.. function:: eval(source, /, globals=None, locals=None) - :param expression: + :param source: A Python expression. - :type expression: :class:`str` | :ref:`code object ` + :type source: :class:`str` | :ref:`code object ` :param globals: The global namespace (default: ``None``). @@ -583,11 +583,15 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + .. index:: pair: built-in function; exec -.. function:: exec(object, globals=None, locals=None, /, *, closure=None) +.. function:: exec(source, /, globals=None, locals=None, *, closure=None) - This function supports dynamic execution of Python code. *object* must be + This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as a suite of Python statements which is then executed (unless a syntax error occurs). [#]_ If it is a code object, it is simply executed. In all cases, @@ -632,14 +636,17 @@ are always available. They are listed here in alphabetical order. .. note:: - The default *locals* act as described for function :func:`locals` below: - modifications to the default *locals* dictionary should not be attempted. + The default *locals* act as described for function :func:`locals` below. Pass an explicit *locals* dictionary if you need to see effects of the code on *locals* after function :func:`exec` returns. .. versionchanged:: 3.11 Added the *closure* parameter. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + .. function:: filter(function, iterable) @@ -1041,14 +1048,44 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Update and return a dictionary representing the current local symbol table. - Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. Note that at the module level, :func:`locals` - and :func:`globals` are the same dictionary. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. + + At module scope, as well as when using ``exec()`` or ``eval()`` with a + single namespace, this function returns the same namespace as ``globals()``. + + At class scope, it returns the namespace that will be passed to the + metaclass constructor. + + When using ``exec()`` or ``eval()`` with separate local and global + namespaces, it returns the local namespace passed in to the function call. + + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as bound, + rebound, or deleted local variables, and binding, rebinding, or deleting + local variables will immediately affect the contents of the returned mapping + object. + + At function scope (including for generators and coroutines), each call to + ``locals()`` instead returns a fresh dictionary containing the current + bindings of the function's local variables and any nonlocal cell references. + In this case, name binding changes made via the returned dict are *not* + written back to the corresponding local variables or nonlocal cell + references, and binding, rebinding, or deleting local variables and nonlocal + cell references does *not* affect the contents of previously returned + dictionaries. + + .. versionchanged:: 3.13 + In previous versions, the semantics of mutating the mapping object + returned from this function were formally undefined. In CPython + specifically, the mapping returned at function scope could be + implicitly refreshed by other operations, such as calling ``locals()`` + again. Obtaining the legacy CPython behaviour now requires explicit + calls to update the initially returned dictionary with the results + of subsequent calls to ``locals()``. - .. note:: - The contents of this dictionary should not be modified; changes may not - affect the values of local and free variables used by the interpreter. .. function:: map(function, iterable, *iterables) @@ -1733,8 +1770,9 @@ are always available. They are listed here in alphabetical order. :ref:`function` for details. A static method can be called either on the class (such as ``C.f()``) or on - an instance (such as ``C().f()``). Moreover, they can be called as regular - functions (such as ``f()``). + an instance (such as ``C().f()``). + Moreover, the static method :term:`descriptor` is also callable, so it can + be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see :func:`classmethod` for a variant that is useful for creating alternate class diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 5039bc6e85ee20..674ce5807fdf11 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -406,6 +406,84 @@ metadata in locations other than the file system, subclass a custom finder, return instances of this derived ``Distribution`` in the ``find_distributions()`` method. +Example +------- + +Consider for example a custom finder that loads Python +modules from a database:: + + class DatabaseImporter(importlib.abc.MetaPathFinder): + def __init__(self, db): + self.db = db + + def find_spec(self, fullname, target=None) -> ModuleSpec: + return self.db.spec_from_name(fullname) + + sys.meta_path.append(DatabaseImporter(connect_db(...))) + +That importer now presumably provides importable modules from a +database, but it provides no metadata or entry points. For this +custom importer to provide metadata, it would also need to implement +``DistributionFinder``:: + + from importlib.metadata import DistributionFinder + + class DatabaseImporter(DistributionFinder): + ... + + def find_distributions(self, context=DistributionFinder.Context()): + query = dict(name=context.name) if context.name else {} + for dist_record in self.db.query_distributions(query): + yield DatabaseDistribution(dist_record) + +In this way, ``query_distributions`` would return records for +each distribution served by the database matching the query. For +example, if ``requests-1.0`` is in the database, ``find_distributions`` +would yield a ``DatabaseDistribution`` for ``Context(name='requests')`` +or ``Context(name=None)``. + +For the sake of simplicity, this example ignores ``context.path``\. The +``path`` attribute defaults to ``sys.path`` and is the set of import paths to +be considered in the search. A ``DatabaseImporter`` could potentially function +without any concern for a search path. Assuming the importer does no +partitioning, the "path" would be irrelevant. In order to illustrate the +purpose of ``path``, the example would need to illustrate a more complex +``DatabaseImporter`` whose behavior varied depending on +``sys.path``/``PYTHONPATH``. In that case, the ``find_distributions`` should +honor the ``context.path`` and only yield ``Distribution``\ s pertinent to that +path. + +``DatabaseDistribution``, then, would look something like:: + + class DatabaseDistribution(importlib.metadata.Distributon): + def __init__(self, record): + self.record = record + + def read_text(self, filename): + """ + Read a file like "METADATA" for the current distribution. + """ + if filename == "METADATA": + return f"""Name: {self.record.name} + Version: {self.record.version} + """ + if filename == "entry_points.txt": + return "\n".join( + f"""[{ep.group}]\n{ep.name}={ep.value}""" + for ep in self.record.entry_points) + + def locate_file(self, path): + raise RuntimeError("This distribution has no file system") + +This basic implementation should provide metadata and entry points for +packages served by the ``DatabaseImporter``, assuming that the +``record`` supplies suitable ``.name``, ``.version``, and +``.entry_points`` attributes. + +The ``DatabaseDistribution`` may also provide other metadata files, like +``RECORD`` (required for ``Distribution.files``) or override the +implementation of ``Distribution.files``. See the source for more inspiration. + .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points .. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9a5cb8be37d349..fb3f33370e3faa 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -504,24 +504,17 @@ loops that truncate the stream. # islice('ABCDEFG', 2, None) → C D E F G # islice('ABCDEFG', 0, None, 2) → A C E G s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + indices = count() if stop is None else range(max(stop, start)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step .. function:: pairwise(iterable) @@ -826,10 +819,7 @@ and :term:`generators ` which incur interpreter overhead. return map(function, count(start)) def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. - - Example: repeatfunc(random.random) - """ + "Repeat calls to func with specified arguments." if times is None: return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) @@ -851,10 +841,8 @@ and :term:`generators ` which incur interpreter overhead. "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: - # feed the entire iterator into a zero-length deque collections.deque(iterator, maxlen=0) else: - # advance to the empty slice starting at position n next(islice(iterator, n, n), None) def nth(iterable, n, default=None): @@ -873,7 +861,7 @@ and :term:`generators ` which incur interpreter overhead. def all_equal(iterable, key=None): "Returns True if all the elements are equal to each other." - # all_equal('4٤໔４৪', key=int) → True + # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 def unique_justseen(iterable, key=None): @@ -903,9 +891,9 @@ and :term:`generators ` which incur interpreter overhead. def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: + iterator = iter(iterable) + window = collections.deque(islice(iterator, n - 1), maxlen=n) + for x in iterator: window.append(x) yield tuple(window) @@ -955,8 +943,8 @@ and :term:`generators ` which incur interpreter overhead. seq_index = getattr(iterable, 'index', None) if seq_index is None: # Path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): + iterator = islice(iterable, start, stop) + for i, element in enumerate(iterator, start): if element is value or element == value: yield i else: @@ -971,10 +959,7 @@ and :term:`generators ` which incur interpreter overhead. pass def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - """ + "Convert a call-until-exception interface to an iterator interface." # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator try: if first is not None: @@ -1074,14 +1059,10 @@ The following recipes have a more mathematical flavor: # sieve(30) → 2 3 5 7 11 13 17 19 23 29 if n > 2: yield 2 - start = 3 data = bytearray((0, 1)) * (n // 2) - limit = math.isqrt(n) + 1 - for p in iter_index(data, 1, start, limit): - yield from iter_index(data, 1, start, p*p) + for p in iter_index(data, 1, start=3, stop=math.isqrt(n) + 1): data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - start = p*p - yield from iter_index(data, 1, start) + yield from iter_index(data, 1, start=3) def factor(n): "Prime factors of n." @@ -1101,8 +1082,8 @@ The following recipes have a more mathematical flavor: "Count of natural numbers up to n that are coprime to n." # https://mathworld.wolfram.com/TotientFunction.html # totient(12) → 4 because len([1, 5, 7, 11]) == 4 - for p in unique_justseen(factor(n)): - n -= n // p + for prime in set(factor(n)): + n -= n // prime return n diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 1475d26486de5f..ab9236cc7cfb81 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -134,7 +134,7 @@ Number-theoretic and representation functions For further discussion and two alternative approaches, see the `ASPN cookbook recipes for accurate floating point summation - `_\. + `_\. .. function:: gcd(*integers) diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index f610032acbe417..a24eab21d57343 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -52,7 +52,22 @@ the information :func:`init` sets up. are also recognized. .. versionchanged:: 3.8 - Added support for url being a :term:`path-like object`. + Added support for *url* being a :term:`path-like object`. + + .. deprecated:: 3.13 + Passing a file path instead of URL is :term:`soft deprecated`. + Use :func:`guess_file_type` for this. + + +.. function:: guess_file_type(path, *, strict=True) + + .. index:: pair: MIME; headers + + Guess the type of a file based on its path, given by *path*. + Similar to the :func:`guess_type` function, but accepts a path instead of URL. + Path can be a string, a bytes object or a :term:`path-like object`. + + .. versionadded:: 3.13 .. function:: guess_all_extensions(type, strict=True) @@ -61,7 +76,7 @@ the information :func:`init` sets up. return value is a list of strings giving all possible filename extensions, including the leading dot (``'.'``). The extensions are not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME - type *type* by :func:`guess_type`. + type *type* by :func:`guess_type` and :func:`guess_file_type`. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -72,8 +87,8 @@ the information :func:`init` sets up. return value is a string giving a filename extension, including the leading dot (``'.'``). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type *type* by - :func:`guess_type`. If no extension can be guessed for *type*, ``None`` is - returned. + :func:`guess_type` and :func:`guess_file_type`. + If no extension can be guessed for *type*, ``None`` is returned. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -238,6 +253,14 @@ than one MIME-type database; it provides an interface similar to the one of the the object. + .. method:: MimeTypes.guess_file_type(path, *, strict=True) + + Similar to the :func:`guess_file_type` function, using the tables stored + as part of the object. + + .. versionadded:: 3.13 + + .. method:: MimeTypes.guess_all_extensions(type, strict=True) Similar to the :func:`guess_all_extensions` function, using the tables stored diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index ac3458c86fd4c4..72da777cd1407e 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -211,7 +211,7 @@ Other Functions After you use :func:`CrtSetReportMode` to specify :const:`CRTDBG_MODE_FILE`, you can specify the file handle to receive the message text. *type* must be - one of the :const:`!CRT_\*` constants listed below. *file* shuld be the file + one of the :const:`!CRT_\*` constants listed below. *file* should be the file handle your want specified. Only available in :ref:`debug build of Python `. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 06ec7da14ebfae..6c92eed9c063f1 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -926,7 +926,7 @@ as internal buffering of data. If *offset_src* is None, then *src* is read from the current position; respectively for *offset_dst*. - In Linux kernel older than 5.3, the files pointed by *src* and *dst* + In Linux kernel older than 5.3, the files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1720,7 +1720,7 @@ or `the MSDN `_ on Windo At least one of the file descriptors must refer to a pipe. If *offset_src* is None, then *src* is read from the current position; respectively for *offset_dst*. The offset associated to the file descriptor that refers to a - pipe must be ``None``. The files pointed by *src* and *dst* must reside in + pipe must be ``None``. The files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -2430,6 +2430,10 @@ features: platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0o700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + This function can also support :ref:`paths relative to directory descriptors `. @@ -2444,6 +2448,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Windows now handles a *mode* of ``0o700``. + .. function:: makedirs(name, mode=0o777, exist_ok=False) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index ac3007f70c3534..7a47a7d5d6754f 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -62,8 +62,8 @@ running without the debugger using the :pdbcmd:`continue` command. The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode:: - > ...(3)double() - -> return x * 2 + > ...(2)double() + -> breakpoint() (Pdb) p x 3 (Pdb) continue @@ -164,6 +164,9 @@ slightly different way: .. versionchanged:: 3.7 The keyword-only argument *header*. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. .. function:: post_mortem(traceback=None) @@ -328,12 +331,16 @@ can be overridden by the local file. .. pdbcommand:: b(reak) [([filename:]lineno | function) [, condition]] - With a *lineno* argument, set a break there in the current file. With a - *function* argument, set a break at the first executable statement within - that function. The line number may be prefixed with a filename and a colon, - to specify a breakpoint in another file (probably one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Note that each breakpoint - is assigned a number to which all the other breakpoint commands refer. + With a *lineno* argument, set a break at line *lineno* in the current file. + The line number may be prefixed with a *filename* and a colon, + to specify a breakpoint in another file (possibly one that hasn't been loaded + yet). The file is searched on :data:`sys.path`. Accepatable forms of *filename* + are ``/abspath/to/file.py``, ``relpath/file.py``, ``module`` and + ``package.module``. + + With a *function* argument, set a break at the first executable statement within + that function. *function* can be any expression that evaluates to a function + in the current namespace. If a second argument is present, it is an expression which must evaluate to true before the breakpoint is honored. @@ -342,6 +349,9 @@ can be overridden by the local file. of times that breakpoint has been hit, the current ignore count, and the associated condition if any. + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + .. pdbcommand:: tbreak [([filename:]lineno | function) [, condition]] Temporary breakpoint, which is removed automatically when it is first hit. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 8fbce18c56f17c..22a799c706b7e2 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -55,10 +55,16 @@ from sources provided by the operating system. `Complementary-Multiply-with-Carry recipe - `_ for a compatible alternative + `_ for a compatible alternative random number generator with a long period and comparatively simple update operations. +.. note:: + The global random number generator and instances of :class:`Random` are thread-safe. + However, in the free-threaded build, concurrent calls to the global generator or + to the same instance of :class:`Random` may encounter contention and poor performance. + Consider using separate instances of :class:`Random` per thread instead. + Bookkeeping functions --------------------- @@ -700,3 +706,83 @@ positive unnormalized float and is equal to ``math.ulp(0.0)``.) `_ a paper by Allen B. Downey describing ways to generate more fine-grained floats than normally generated by :func:`.random`. + +.. _random-cli: + +Command-line usage +------------------ + +.. versionadded:: 3.13 + +The :mod:`!random` module can be executed from the command line. + +.. code-block:: sh + + python -m random [-h] [-c CHOICE [CHOICE ...] | -i N | -f N] [input ...] + +The following options are accepted: + +.. program:: random + +.. option:: -h, --help + + Show the help message and exit. + +.. option:: -c CHOICE [CHOICE ...] + --choice CHOICE [CHOICE ...] + + Print a random choice, using :meth:`choice`. + +.. option:: -i + --integer + + Print a random integer between 1 and N inclusive, using :meth:`randint`. + +.. option:: -f + --float + + Print a random floating point number between 1 and N inclusive, + using :meth:`uniform`. + +If no options are given, the output depends on the input: + +* String or multiple: same as :option:`--choice`. +* Integer: same as :option:`--integer`. +* Float: same as :option:`--float`. + +.. _random-cli-example: + +Command-line example +-------------------- + +Here are some examples of the :mod:`!random` command-line interface: + +.. code-block:: console + + $ # Choose one at random + $ python -m random egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + Lobster Thermidor aux crevettes with a Mornay sauce + + $ # Random integer + $ python -m random 6 + 6 + + $ # Random floating-point number + $ python -m random 1.8 + 1.7080016272295635 + + $ # With explicit arguments + $ python -m random --choice egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + egg + + $ python -m random --integer 6 + 3 + + $ python -m random --float 1.8 + 1.5666339105010318 + + $ python -m random --integer 6 + 5 + + $ python -m random --float 6 + 3.1942323316565915 diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 95c54991887022..1fa614f6584170 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -86,7 +86,7 @@ Two additional methods are supported: .. seealso:: - `Persistent dictionary recipe `_ + `Persistent dictionary recipe `_ with widely supported storage formats and having the speed of native dictionaries. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 8e5828c789e4e2..331acfce3afee3 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -242,7 +242,7 @@ Directory and files operations be copied as far as the platform allows; if false or omitted, the contents and metadata of the linked files are copied to the new tree. - When *symlinks* is false, if the file pointed by the symlink doesn't + When *symlinks* is false, if the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an :exc:`Error` exception at the end of the copy process. You can set the optional *ignore_dangling_symlinks* flag to true if you @@ -447,10 +447,11 @@ Directory and files operations called. If no *cmd* would be called, return ``None``. *mode* is a permission mask passed to :func:`os.access`, by default - determining if the file exists and executable. + determining if the file exists and is executable. - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :data:`os.defpath`. + *path* is a "``PATH`` string" specifying the lookup directory list. When no + *path* is specified, the results of :func:`os.environ` are used, returning + either the "PATH" value or a fallback of :data:`os.defpath`. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index cc72396964342e..d5a316e45ee3e2 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -77,6 +77,7 @@ or sample. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. :func:`kde` Estimate the probability density distribution of the data. +:func:`kde_random` Random sampling from the PDF generated by kde(). :func:`median` Median (middle value) of data. :func:`median_low` Low median of data. :func:`median_high` High median of data. @@ -311,6 +312,30 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.13 +.. function:: kde_random(data, h, kernel='normal', *, seed=None) + + Return a function that makes a random selection from the estimated + probability density function produced by ``kde(data, h, kernel)``. + + Providing a *seed* allows reproducible selections. In the future, the + values may change slightly as more accurate kernel inverse CDF estimates + are implemented. The seed may be an integer, float, str, or bytes. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + Continuing the example for :func:`kde`, we can use + :func:`kde_random` to generate new random selections from an + estimated probability density function: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + .. versionadded:: 3.13 + + .. function:: median(data) Return the median (middle value) of numeric data, using the common "mean of @@ -1148,65 +1173,6 @@ The final prediction goes to the largest posterior. This is known as the 'female' -Sampling from kernel density estimation -*************************************** - -The :func:`kde()` function creates a continuous probability density -function from discrete samples. Some applications need a way to make -random selections from that distribution. - -The technique is to pick a sample from a bandwidth scaled kernel -function and recenter the result around a randomly chosen point from -the input data. This can be done with any kernel that has a known or -accurately approximated inverse cumulative distribution function. - -.. testcode:: - - from random import choice, random, seed - from math import sqrt, log, pi, tan, asin, cos, acos - from statistics import NormalDist - - kernel_invcdfs = { - 'normal': NormalDist().inv_cdf, - 'logistic': lambda p: log(p / (1 - p)), - 'sigmoid': lambda p: log(tan(p * pi/2)), - 'rectangular': lambda p: 2*p - 1, - 'triangular': lambda p: sqrt(2*p) - 1 if p < 0.5 else 1 - sqrt(2 - 2*p), - 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), - 'cosine': lambda p: 2*asin(2*p - 1)/pi, - } - - def kde_random(data, h, kernel='normal'): - 'Return a function that samples from kde() smoothed data.' - kernel_invcdf = kernel_invcdfs[kernel] - def rand(): - return h * kernel_invcdf(random()) + choice(data) - return rand - -For example: - -.. doctest:: - - >>> discrete_samples = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> rand = kde_random(discrete_samples, h=1.5) - >>> seed(8675309) - >>> selections = [rand() for i in range(10)] - >>> [round(x, 1) for x in selections] - [4.7, 7.4, 1.2, 7.8, 6.9, -1.3, 5.8, 0.2, -1.4, 5.7] - -.. testcode:: - :hide: - - from statistics import kde - from math import isclose - - # Verify that cdf / invcdf will round trip - xarr = [i/100 for i in range(-100, 101)] - for kernel, invcdf in kernel_invcdfs.items(): - cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) - for x in xarr: - assert isclose(invcdf(cdf(x)), x, abs_tol=1E-9) - .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 6c13bd015d5691..8f5f9ee519a573 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1496,7 +1496,7 @@ objects that compare equal might have different :attr:`~range.start`, .. seealso:: - * The `linspace recipe `_ + * The `linspace recipe `_ shows how to implement a lazy version of range suitable for floating point applications. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 4980227c60b21e..0e0095e108e9c0 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -255,7 +255,10 @@ No events are active by default. Per code object events '''''''''''''''''''''' -Events can also be controlled on a per code object basis. +Events can also be controlled on a per code object basis. The functions +defined below which accept a :class:`types.CodeType` should be prepared +to accept a look-alike object from functions which are not defined +in Python (see :ref:`monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 19d6856efe5d09..19a396d2300011 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -877,7 +877,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe `_ + See `recursive sizeof recipe `_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. @@ -1199,6 +1199,14 @@ always available. return value of :func:`intern` around to benefit from it. +.. function:: _is_gil_enabled() + + Return :const:`True` if the :term:`GIL` is enabled and :const:`False` if + it is disabled. + + .. versionadded:: 3.13 + + .. function:: is_finalizing() Return :const:`True` if the main Python interpreter is diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index e084d8554c7c09..7e5dee1b562df8 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -979,6 +979,15 @@ of :class:`tkinter.Image`: Either type of image is created through either the ``file`` or the ``data`` option (other options are available as well). +.. versionchanged:: 3.13 + Added the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy()`, + :meth:`!zoom()` and :meth:`!subsample()`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy()`. + The image object can then be used wherever an ``image`` option is supported by some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index d816e6368f40d2..f53080e0610cb1 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -852,14 +852,25 @@ using ``[]``. .. versionadded:: 3.11 .. data:: Never + NoReturn - The `bottom type `_, + :data:`!Never` and :data:`!NoReturn` represent the + `bottom type `_, a type that has no members. - This can be used to define a function that should never be - called, or a function that never returns:: + They can be used to indicate that a function never returns, + such as :func:`sys.exit`:: - from typing import Never + from typing import Never # or NoReturn + + def stop() -> Never: + raise RuntimeError('no way') + + Or to define a function that should never be + called, as there are no valid arguments, such as + :func:`assert_never`:: + + from typing import Never # or NoReturn def never_call_me(arg: Never) -> None: pass @@ -872,31 +883,18 @@ using ``[]``. case str(): print("It's a str") case _: - never_call_me(arg) # OK, arg is of type Never - - .. versionadded:: 3.11 - - On older Python versions, :data:`NoReturn` may be used to express the - same concept. ``Never`` was added to make the intended meaning more explicit. - -.. data:: NoReturn + never_call_me(arg) # OK, arg is of type Never (or NoReturn) - Special type indicating that a function never returns. - - For example:: + :data:`!Never` and :data:`!NoReturn` have the same meaning in the type system + and static type checkers treat both equivalently. - from typing import NoReturn + .. versionadded:: 3.6.2 - def stop() -> NoReturn: - raise RuntimeError('no way') + Added :data:`NoReturn`. - ``NoReturn`` can also be used as a - `bottom type `_, a type that - has no values. Starting in Python 3.11, the :data:`Never` type should - be used for this concept instead. Type checkers should treat the two - equivalently. + .. versionadded:: 3.11 - .. versionadded:: 3.6.2 + Added :data:`Never`. .. data:: Self @@ -1616,7 +1614,7 @@ without the dedicated syntax, as documented below. .. _typevar: -.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False) +.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable. @@ -1754,15 +1752,35 @@ without the dedicated syntax, as documented below. the constraints are evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. attribute:: __default__ + + The default value of the type variable, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the type variable has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionchanged:: 3.12 Type variables can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. The ``infer_variance`` parameter was added. + .. versionchanged:: 3.13 + + Support for default values was added. + .. _typevartuple: -.. class:: TypeVarTuple(name) +.. class:: TypeVarTuple(name, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -1872,6 +1890,22 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __default__ + + The default value of the type variable tuple, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the type variable tuple has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -1879,7 +1913,11 @@ without the dedicated syntax, as documented below. Type variable tuples can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. -.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) + .. versionchanged:: 3.13 + + Support for default values was added. + +.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of :ref:`type variables `. @@ -1948,6 +1986,22 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __default__ + + The default value of the parameter specification, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the parameter specification has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + Parameter specification variables created with ``covariant=True`` or ``contravariant=True`` can be used to declare covariant or contravariant generic types. The ``bound`` argument is also accepted, similar to @@ -1961,6 +2015,10 @@ without the dedicated syntax, as documented below. Parameter specifications can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. + .. versionchanged:: 3.13 + + Support for default values was added. + .. note:: Only parameter specification variables defined in global scope can be pickled. @@ -3173,6 +3231,22 @@ Introspection helpers .. versionadded:: 3.7.4 +.. data:: NoDefault + + A sentinel object used to indicate that a type parameter has no default + value. For example: + + .. doctest:: + + >>> T = TypeVar("T") + >>> T.__default__ is typing.NoDefault + True + >>> S = TypeVar("S", default=None) + >>> S.__default__ is None + True + + .. versionadded:: 3.13 + Constant -------- @@ -3574,8 +3648,14 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` is no ``ReturnType`` type parameter. As with :class:`Generator`, the ``SendType`` behaves contravariantly. - If your generator will only yield values, set the ``SendType`` to - ``None``:: + The ``SendType`` defaults to :const:`!None`:: + + async def infinite_stream(start: int) -> AsyncGenerator[int]: + while True: + yield start + start = await increment(start) + + It is also possible to set this type explicitly:: async def infinite_stream(start: int) -> AsyncGenerator[int, None]: while True: @@ -3597,6 +3677,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + The ``SendType`` parameter now has a default. + .. class:: AsyncIterable(Generic[T_co]) Deprecated alias to :class:`collections.abc.AsyncIterable`. @@ -3680,8 +3763,14 @@ Aliases to other ABCs in :mod:`collections.abc` of :class:`Generator` behaves contravariantly, not covariantly or invariantly. - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: + The ``SendType`` and ``ReturnType`` parameters default to :const:`!None`:: + + def infinite_stream(start: int) -> Generator[int]: + while True: + yield start + start += 1 + + It is also possible to set these types explicitly:: def infinite_stream(start: int) -> Generator[int, None, None]: while True: @@ -3700,6 +3789,9 @@ Aliases to other ABCs in :mod:`collections.abc` :class:`collections.abc.Generator` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Default values for the send and return types were added. + .. class:: Hashable Deprecated alias to :class:`collections.abc.Hashable`. @@ -3727,10 +3819,15 @@ Aliases to other ABCs in :mod:`collections.abc` Aliases to :mod:`contextlib` ABCs """"""""""""""""""""""""""""""""" -.. class:: ContextManager(Generic[T_co]) +.. class:: ContextManager(Generic[T_co, ExitT_co]) Deprecated alias to :class:`contextlib.AbstractContextManager`. + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__enter__` method. The optional second type parameter, ``ExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__exit__` method. + .. versionadded:: 3.5.4 .. deprecated:: 3.9 @@ -3738,10 +3835,18 @@ Aliases to :mod:`contextlib` ABCs now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: AsyncContextManager(Generic[T_co]) + .. versionchanged:: 3.13 + Added the optional second type parameter, ``ExitT_co``. + +.. class:: AsyncContextManager(Generic[T_co, AExitT_co]) Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__aenter__` method. The optional second type parameter, ``AExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__aexit__` method. + .. versionadded:: 3.6.2 .. deprecated:: 3.9 @@ -3749,6 +3854,9 @@ Aliases to :mod:`contextlib` ABCs now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Added the optional second type parameter, ``AExitT_co``. + Deprecation Timeline of Major Features ====================================== diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 3c898c3e826304..59fb14960ba9f6 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -31,6 +31,11 @@ The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in the following sections. +This module's functions use the deprecated term ``netloc`` (or ``net_loc``), +which was introduced in :rfc:`1808`. However, this term has been obsoleted by +:rfc:`3986`, which introduced the term ``authority`` as its replacement. +The use of ``netloc`` is continued for backward compatibility. + URL Parsing ----------- diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index c2b0ba7046967e..7fe84a2de1fceb 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -865,7 +865,7 @@ directory and port number (default: 8000) on the command line:: fn = os.path.join(path, environ["PATH_INFO"][1:]) if "." not in fn.split(os.path.sep)[-1]: fn = os.path.join(fn, "index.html") - mime_type = mimetypes.guess_type(fn)[0] + mime_type = mimetypes.guess_file_type(fn)[0] # Return 200 OK if file exists, otherwise 404 Not Found if os.path.exists(fn): diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 374404bf33abbe..42cca0664df71d 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1620,15 +1620,18 @@ Type parameter lists .. versionadded:: 3.12 +.. versionchanged:: 3.13 + Support for default values was added (see :pep:`696`). + .. index:: single: type parameters .. productionlist:: python-grammar type_params: "[" `type_param` ("," `type_param`)* "]" type_param: `typevar` | `typevartuple` | `paramspec` - typevar: `identifier` (":" `expression`)? - typevartuple: "*" `identifier` - paramspec: "**" `identifier` + typevar: `identifier` (":" `expression`)? ("=" `expression`)? + typevartuple: "*" `identifier` ("=" `expression`)? + paramspec: "**" `identifier` ("=" `expression`)? :ref:`Functions ` (including :ref:`coroutines `), :ref:`classes ` and :ref:`type aliases ` may @@ -1694,19 +1697,31 @@ evaluated in a separate :ref:`annotation scope `. :data:`typing.TypeVarTuple`\ s and :data:`typing.ParamSpec`\ s cannot have bounds or constraints. +All three flavors of type parameters can also have a *default value*, which is used +when the type parameter is not explicitly provided. This is added by appending +a single equals sign (``=``) followed by an expression. Like the bounds and +constraints of type variables, the default value is not evaluated when the +object is created, but only when the type parameter's ``__default__`` attribute +is accessed. To this end, the default value is evaluated in a separate +:ref:`annotation scope `. If no default value is specified +for a type parameter, the ``__default__`` attribute is set to the special +sentinel object :data:`typing.NoDefault`. + The following example indicates the full set of allowed type parameter declarations:: def overly_generic[ SimpleTypeVar, + TypeVarWithDefault = int, TypeVarWithBound: int, TypeVarWithConstraints: (str, bytes), - *SimpleTypeVarTuple, - **SimpleParamSpec, + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), ]( a: SimpleTypeVar, - b: TypeVarWithBound, - c: Callable[SimpleParamSpec, TypeVarWithConstraints], - *d: SimpleTypeVarTuple, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, ): ... .. _generic-functions: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 5e1558362ffaa0..f5e87160732056 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -971,6 +971,7 @@ A class object can be called (see above) to yield a class instance (see below). single: __annotations__ (class attribute) single: __type_params__ (class attribute) single: __static_attributes__ (class attribute) + single: __firstlineno__ (class attribute) Special attributes: @@ -1005,6 +1006,9 @@ Special attributes: A tuple containing names of attributes of this class which are accessed through ``self.X`` from any function in its body. + :attr:`__firstlineno__` + The line number of the first line of the class definition, including decorators. + Class instances --------------- @@ -1341,7 +1345,12 @@ Special read-only attributes * - .. attribute:: frame.f_locals - The dictionary used by the frame to look up - :ref:`local variables ` + :ref:`local variables `. + If the frame refers to a function or comprehension, + this may return a write-through proxy object. + + .. versionchanged:: 3.13 + Return a proxy for functions and comprehensions. * - .. attribute:: frame.f_globals - The dictionary used by the frame to look up diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index ed50faed6c940d..f24e1537af39ed 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -205,7 +205,7 @@ Annotation scopes are used in the following contexts: * Type parameter lists for :ref:`generic classes `. A generic class's base classes and keyword arguments are executed within the annotation scope, but its decorators are not. -* The bounds and constraints for type variables +* The bounds, constraints, and default values for type parameters (:ref:`lazily evaluated `). * The value of type aliases (:ref:`lazily evaluated `). @@ -232,13 +232,17 @@ Annotation scopes differ from function scopes in the following ways: .. versionadded:: 3.12 Annotation scopes were introduced in Python 3.12 as part of :pep:`695`. +.. versionchanged:: 3.13 + Annotation scopes are also used for type parameter defaults, as + introduced by :pep:`696`. + .. _lazy-evaluation: Lazy evaluation --------------- The values of type aliases created through the :keyword:`type` statement are -*lazily evaluated*. The same applies to the bounds and constraints of type +*lazily evaluated*. The same applies to the bounds, constraints, and default values of type variables created through the :ref:`type parameter syntax `. This means that they are not evaluated when the type alias or type variable is created. Instead, they are only evaluated when doing so is necessary to resolve diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index 9a5f4f3676e089..fe45c91501a56a 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -13,16 +13,16 @@ python-docs-theme>=2022.1 # Sphinx 6.2.1 comes from ``needs_sphinx = '6.2.1'`` in ``Doc/conf.py``. alabaster==0.7.16 -Babel==2.14.0 +Babel==2.15.0 certifi==2024.2.2 charset-normalizer==3.3.2 docutils==0.19 idna==3.7 imagesize==1.4.1 -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.5 packaging==24.0 -Pygments==2.17.2 +Pygments==2.18.0 requests==2.31.0 snowballstemmer==2.2.0 Sphinx==6.2.1 diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 44db77af5d24d3..b35bedcb0901f1 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -33,7 +33,7 @@ ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' # Used in conf.py and updated here by python/release-tools/run_release.py -SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.13/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 4bea0d8a49ce20..b8faf756698097 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -10,6 +10,28 @@ Appendix Interactive Mode ================ +There are two variants of the interactive :term:`REPL`. The classic +basic interpreter is supported on all platforms with minimal line +control capabilities. + +On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and +:mod:`readline` support, a new interactive shell is used by default. +This one supports color, multiline editing, history browsing, and +paste mode. To disable color, see :ref:`using-on-controlling-color` for +details. Function keys provide some additional functionality. +:kbd:`F1` enters the interactive help browser :mod:`pydoc`. +:kbd:`F2` allows for browsing command-line history without output nor the +:term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which +makes pasting larger blocks of code easier. Press :kbd:`F3` to return to +the regular prompt. + +When using the new interactive shell, exit the shell by typing :kbd:`exit` +or :kbd:`quit`. Adding call parentheses after those commands is not +required. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + .. _tut-error: Error Handling @@ -40,7 +62,7 @@ Executable Python Scripts On BSD'ish Unix systems, Python scripts can be made directly executable, like shell scripts, by putting the line :: - #!/usr/bin/env python3.5 + #!/usr/bin/env python3 (assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning of the script and giving the file an executable mode. The ``#!`` must be the @@ -107,7 +129,7 @@ of your user site-packages directory. Start Python and run this code:: >>> import site >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' + '/home/user/.local/lib/python3.x/site-packages' Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 299b6c2777adc0..02e7de77322e99 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.13 + python3.14 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.13 - Python 3.13 (default, April 4 2023, 09:25:04) + $ python3.14 + Python 3.14 (default, April 4 2024, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 9def2a5714950b..86c94429339dfc 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python313' + 'C:\\Python314' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 09b6f3d91bcfed..4bc810ce36c71b 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -279,7 +279,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python313/lib/weakref.py", line 46, in __getitem__ + File "C:/python314/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 295e3fb09830ce..2d95fa9474033f 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -42,6 +42,7 @@ additional methods of invocation: * When called with standard input connected to a tty device, it prompts for commands and executes them until an EOF (an end-of-file character, you can produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. + For more on interactive mode, see :ref:`tut-interac`. * When called with a file name argument or with a file as standard input, it reads and executes a script from that file. * When called with a directory name argument, it reads and executes an @@ -586,6 +587,15 @@ Miscellaneous options .. versionadded:: 3.12 + * ``-X perf_jit`` enables support for the Linux ``perf`` profiler with DWARF + support. When this option is provided, the ``perf`` profiler will be able + to report Python calls using DWARF information. This option is only available on + some platforms and will do nothing if is not supported on the current + system. The default value is "off". See also :envvar:`PYTHON_PERF_JIT_SUPPORT` + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. *n* must be greater than or equal to 1. @@ -632,11 +642,11 @@ behavior can be controlled by setting different environment variables. Setting the environment variable ``TERM`` to ``dumb`` will disable color. -If the environment variable ``FORCE_COLOR`` is set, then color will be +If the |FORCE_COLOR|_ environment variable is set, then color will be enabled regardless of the value of TERM. This is useful on CI systems which aren’t terminals but can still display ANSI escape sequences. -If the environment variable ``NO_COLOR`` is set, Python will disable all color +If the |NO_COLOR|_ environment variable is set, Python will disable all color in the output. This takes precedence over ``FORCE_COLOR``. All these environment variables are used also by other tools to control color @@ -645,6 +655,14 @@ output. To control the color output only in the Python interpreter, the precedence over ``NO_COLOR``, which in turn takes precedence over ``FORCE_COLOR``. +.. Apparently this how you hack together a formatted link: + +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ + Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1119,6 +1137,21 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHON_PERF_JIT_SUPPORT + + If this variable is set to a nonzero value, it enables support for + the Linux ``perf`` profiler so Python calls can be detected by it + using DWARF information. + + If set to ``0``, disable Linux ``perf`` profiler support. + + See also the :option:`-X perf_jit <-X>` command-line option + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + + + .. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides the return @@ -1150,6 +1183,15 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_REPL + + If this variable is set to ``1``, the interpreter will not attempt to + load the Python-based :term:`REPL` that requires :mod:`curses` and + :mod:`readline`, and will instead use the traditional parser-based + :term:`REPL`. + + .. versionadded:: 3.13 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e662c0dafdb8de..d30356d2058eff 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -804,11 +804,18 @@ Libraries options .. option:: --with-system-libmpdec - Build the ``_decimal`` extension module using an installed ``mpdec`` - library, see the :mod:`decimal` module (default is no). + Build the ``_decimal`` extension module using an installed ``mpdecimal`` + library, see the :mod:`decimal` module (default is yes). .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Default to using the installed ``mpdecimal`` library. + + .. deprecated-removed:: 3.13 3.15 + A copy of the ``mpdecimal`` library sources will no longer be distributed + with Python 3.15. + .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. .. option:: --with-readline=readline|editline diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 98349a5984bb7e..69264a3672bbbf 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -3,7 +3,7 @@ What's New In Python 3.13 **************************** -:Editor: TBD +:Editor: Thomas Wouters .. Rules for maintenance: @@ -49,6 +49,10 @@ This article explains the new features in Python 3.13, compared to 3.12. For full details, see the :ref:`changelog `. +.. seealso:: + + :pep:`719` -- Python 3.13 Release Schedule + .. note:: Prerelease users should be aware that this document is currently in draft @@ -56,15 +60,68 @@ For full details, see the :ref:`changelog `. so it's worth checking back even after reading earlier versions. -Summary -- Release highlights +Summary -- Release Highlights ============================= .. This section singles out the most important changes in Python 3.13. Brevity is key. +Python 3.13 beta is the pre-release of the next version of the Python +programming language, with a mix of changes to the language, the +implementation and the standard library. The biggest changes to the +implementation include a new interactive interpreter, and experimental +support for dropping the Global Interpreter Lock (:pep:`703`) and a +Just-In-Time compiler (:pep:`744`). The library changes contain removal of +deprecated APIs and modules, as well as the usual improvements in +user-friendliness and correctness. + .. PEP-sized items next. -Important deprecations, removals or restrictions: +Interpreter improvements: + +* A greatly improved :ref:`interactive interpreter + ` and + :ref:`improved error messages `. + +* Color support in the new :ref:`interactive interpreter + `, + as well as in :ref:`tracebacks ` + and :ref:`doctest ` output. This can be disabled through the + :envvar:`PYTHON_COLORS` and |NO_COLOR|_ environment variables. + +* :pep:`744`: A basic :ref:`JIT compiler ` was added. + It is currently disabled by default (though we may turn it on later). + Performance improvements are modest -- we expect to be improving this + over the next few releases. + +* :pep:`667`: :attr:`FrameType.f_locals ` when used in + a function now returns a write-through proxy to the frame's locals, + rather than a ``dict``. See the PEP for corresponding C API changes + and deprecations. + +New typing features: + +* :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, + and :data:`typing.TypeVarTuple`) now support defaults. + +* :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive + type narrowing behavior. + +* :pep:`705`: :data:`typing.ReadOnly` was added, to mark an item of a + :class:`typing.TypedDict` as read-only for type checkers. + +Free-threading: + +* :pep:`703`: CPython 3.13 has experimental support for running with the + :term:`global interpreter lock` disabled when built with ``--disable-gil``. + See :ref:`Free-threaded CPython ` for more details. + +Platform support: + +* :pep:`730`: Apple's iOS is now an officially supported platform. Official + Android support (:pep:`738`) is in the works as well. + +Removed modules: * :ref:`PEP 594 `: The remaining 19 "dead batteries" have been removed from the standard library: @@ -73,6 +130,11 @@ Important deprecations, removals or restrictions: :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. +* Also removed were the :mod:`!tkinter.tix` and :mod:`!lib2to3` modules, and the + ``2to3`` program. + +Release schedule changes: + * :pep:`602` ("Annual Release Cycle for Python") has been updated: * Python 3.9 - 3.12 have one and a half years of full support, @@ -80,30 +142,58 @@ Important deprecations, removals or restrictions: * Python 3.13 and later have two years of full support, followed by three years of security fixes. -Interpreter improvements: +New Features +============ -* :pep:`744`: A basic :ref:`JIT compiler ` was added. - It is currently disabled by default (though we may turn it on later). - Performance improvements are modest -- we expect to be improving this - over the next few releases. +.. _whatsnew313-better-interactive-interpreter: -New typing features: +A Better Interactive Interpreter +-------------------------------- -* :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive - type narrowing behavior. +On Unix-like systems like Linux or macOS, Python now uses a new +:term:`interactive` shell. When the user starts the :term:`REPL` from an +interactive terminal, and both :mod:`curses` and :mod:`readline` are +available, the interactive shell now supports the following new features: -New Features -============ +* Colorized prompts. +* Multiline editing with history preservation. +* Interactive help browsing using :kbd:`F1` with a separate command + history. +* History browsing using :kbd:`F2` that skips output as well as the + :term:`>>>` and :term:`...` prompts. +* "Paste mode" with :kbd:`F3` that makes pasting larger blocks of code + easier (press :kbd:`F3` again to return to the regular prompt). +* The ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit` without the need to use call parentheses after the command + name. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + +For more on interactive mode, see :ref:`tut-interac`. + +(Contributed by Pablo Galindo Salgado, Łukasz Langa, and +Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project.) + +.. _whatsnew313-improved-error-messages: Improved Error Messages ----------------------- * The interpreter now colorizes error messages when displaying tracebacks by default. This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment - variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment + variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. See also :ref:`using-on-controlling-color`. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) +.. Apparently this how you hack together a formatted link: + +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ + * A common mistake is to write a script with the same name as a standard library module. When this results in errors, we now display a more helpful error message: @@ -178,6 +268,105 @@ Support For Mobile Platforms (PEP written and implementation contributed by Russell Keith-Magee in :gh:`114099`.) +.. _whatsnew313-jit-compiler: + +Experimental JIT Compiler +========================= + +When CPython is configured using the ``--enable-experimental-jit`` option, +a just-in-time compiler is added which may speed up some Python programs. + +The internal architecture is roughly as follows. + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 ` for details. + +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + +* The Tier 2 IR uses the same stack-based VM as Tier 1, but the + instruction format is better suited to translation to machine code. + +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. + +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. + The Tier 2 interpreter can be enabled by configuring Python + with ``--enable-experimental-jit=interpreter``. + +* When the JIT is enabled, the optimized + Tier 2 IR is translated to machine code, which is then executed. + +* The machine code translation process uses a technique called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +The ``--enable-experimental-jit`` flag has the following optional values: + +* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline. + +* ``yes`` (default if the flag is present without optional value) + -- Enable the JIT. To disable the JIT at runtime, + pass the environment variable ``PYTHON_JIT=0``. + +* ``yes-off`` -- Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable + ``PYTHON_JIT=1``. + +* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with + ``PYTHON_JIT=0``. + +(On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter.) + +See :pep:`744` for more details. + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + +.. _free-threaded-cpython: + +Free-threaded CPython +===================== + +CPython will run with the :term:`global interpreter lock` (GIL) disabled when +configured using the ``--disable-gil`` option at build time. This is an +experimental feature and therefore isn't used by default. Users need to +either compile their own interpreter, or install one of the experimental +builds that are marked as *free-threaded*. + +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multicore hardware. + +Work is still ongoing: expect some bugs and a substantial single-threaded +performance hit. + +The free-threaded build still supports optionally running with the GIL +enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command line option :option:`-X gil`. + +To check if the current interpreter is configured with ``--disable-gil``, +use ``sysconfig.get_config_var("Py_GIL_DISABLED")``. To check if the :term:`GIL` +is actually disabled in the running process, the :func:`!sys._is_gil_enabled` +function can be used. + +C-API extension modules need to be built specifically for the free-threaded +build. Extensions that support running with the :term:`GIL` disabled should +use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should +use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support +running with the GIL disabled. Importing C extensions that don't use these +mechanisms will cause the GIL to be enabled, unless the GIL was explicitly +disabled with the :envvar:`PYTHON_GIL` environment variable or the +:option:`-X gil=0` option. + +pip 24.1b1 or newer is required to install packages with C extensions in the +free-threaded build. + Other Language Changes ====================== @@ -221,6 +410,11 @@ Other Language Changes equivalent of the :option:`-X frozen_modules <-X>` command-line option. (Contributed by Yilei Yang in :gh:`111374`.) +* Add :ref:`support for the perf profiler ` working without + frame pointers through the new environment variable + :envvar:`PYTHON_PERF_JIT_SUPPORT` and command-line option :option:`-X perf_jit + <-X>` (Contributed by Pablo Galindo in :gh:`118518`.) + * The new :envvar:`PYTHON_HISTORY` environment variable can be used to change the location of a ``.python_history`` file. (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in @@ -280,11 +474,20 @@ Other Language Changes class scopes are not inlined into their parent scope. (Contributed by Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) +* Classes have a new :attr:`!__firstlineno__` attribute, + populated by the compiler, with the line number of the first line + of the class definition. + (Contributed by Serhiy Storchaka in :gh:`118465`.) + +* ``from __future__ import ...`` statements are now just normal + relative imports if dots are present before the module name. + (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) + New Modules =========== -* None yet. +* None. Improved Modules @@ -327,6 +530,12 @@ ast argument that does not map to a field on the AST node is now deprecated, and will raise an exception in Python 3.15. + These changes do not apply to user-defined subclasses of :class:`ast.AST`, + unless the class opts in to the new behavior by setting the attribute + :attr:`ast.AST._field_types`. + + (Contributed by Jelle Zijlstra in :gh:`105858` and :gh:`117486`.) + * :func:`ast.parse` now accepts an optional argument *optimize* which is passed on to the :func:`compile` built-in. This makes it possible to obtain an optimized AST. @@ -431,9 +640,17 @@ dis the ``show_offsets`` parameter. (Contributed by Irit Katriel in :gh:`112137`.) +.. _whatsnew313-doctest: + doctest ------- +* Color is added to the output by default. + This can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment + variables. See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`117225`.) + * The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped tests. Add :attr:`doctest.DocTestRunner.skips` and :attr:`doctest.TestResults.skipped` attributes. @@ -560,6 +777,13 @@ math "fusedMultiplyAdd" operation for special cases. (Contributed by Mark Dickinson and Victor Stinner in :gh:`73468`.) +mimetypes +--------- + +* Add the :func:`~mimetypes.guess_file_type` function which works with file path. + Passing file path instead of URL in :func:`~mimetypes.guess_type` is :term:`soft deprecated`. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + mmap ---- @@ -687,6 +911,15 @@ pdb command line option or :envvar:`PYTHONSAFEPATH` environment variable). (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) +* :mod:`zipapp` is supported as a debugging target. + (Contributed by Tian Gao in :gh:`118501`.) + +* ``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately + rather than on the next line of code to be executed. This change prevents the + debugger from breaking outside of the context when ``breakpoint()`` is positioned + at the end of the context. + (Contributed by Tian Gao in :gh:`118579`.) + queue ----- @@ -694,6 +927,12 @@ queue termination. (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) +random +------ + +* Add a :ref:`command-line interface `. + (Contributed by Hugo van Kemenade in :gh:`54321`.) + re -- * Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. @@ -722,7 +961,8 @@ statistics * Add :func:`statistics.kde` for kernel density estimation. This makes it possible to estimate a continuous probability density function - from a fixed number of discrete samples. + from a fixed number of discrete samples. Also added :func:`statistics.kde_random` + for sampling from the estimated probability density function. (Contributed by Raymond Hettinger in :gh:`115863`.) .. _whatsnew313-subprocess: @@ -796,6 +1036,21 @@ tkinter * Add the :meth:`!after_info` method for Tkinter widgets. (Contributed by Cheryl Sabella in :gh:`77020`.) +* Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy()`, + :meth:`!zoom()` and :meth:`!subsample()`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy()`. + (Contributed by Serhiy Storchaka in :gh:`118225`.) + +* Add the :class:`!PhotoImage` methods :meth:`!read` to read + an image from a file and :meth:`!data` to get the image data. + Add *background* and *grayscale* parameters to :class:`!PhotoImage` method + :meth:`!write`. + (Contributed by Serhiy Storchaka in :gh:`118271`.) + traceback --------- @@ -829,6 +1084,10 @@ typing an item of a :class:`typing.TypedDict` as read-only for type checkers. See :pep:`705` for more details. +* Add :data:`typing.NoDefault`, a sentinel object used to represent the defaults + of some parameters in the :mod:`typing` module. (Contributed by Jelle Zijlstra in + :gh:`116126`.) + unicodedata ----------- @@ -882,119 +1141,331 @@ Optimizations section above for details. (Contributed by Jakub Kulik in :gh:`113117`.) -.. _whatsnew313-jit-compiler: +* Several standard library modules have had their import times significantly + improved. For example, the import time of the :mod:`typing` module has been + reduced by around a third by removing dependencies on :mod:`re` and + :mod:`contextlib`. Other modules to enjoy import-time speedups include + :mod:`importlib.metadata`, :mod:`threading`, :mod:`enum`, :mod:`functools` + and :mod:`email.utils`. + (Contributed by Alex Waygood, Shantanu Jain, Adam Turner, Daniel Hollas and + others in :gh:`109653`.) -Experimental JIT Compiler -========================= +Removed Modules And APIs +======================== -When CPython is configured using the ``--enable-experimental-jit`` option, -a just-in-time compiler is added which can speed up some Python programs. +.. _whatsnew313-pep594: -The internal architecture is roughly as follows. +PEP 594: dead batteries (and other module removals) +--------------------------------------------------- -* We start with specialized *Tier 1 bytecode*. - See :ref:`What's new in 3.11 ` for details. +* :pep:`594` removed 19 modules from the standard library, + deprecated in Python 3.11: -* When the Tier 1 bytecode gets hot enough, it gets translated - to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + * :mod:`!aifc`. + (Contributed by Victor Stinner in :gh:`104773`.) -* The Tier 2 IR uses the same stack-based VM as Tier 1, but the - instruction format is better suited to translation to machine code. + * :mod:`!audioop`. + (Contributed by Victor Stinner in :gh:`104773`.) -* We have several optimization passes for Tier 2 IR, which are applied - before it is interpreted or translated to machine code. + * :mod:`!chunk`. + (Contributed by Victor Stinner in :gh:`104773`.) -* There is a Tier 2 interpreter, but it is mostly intended for debugging - the earlier stages of the optimization pipeline. If the JIT is not - enabled, the Tier 2 interpreter can be invoked by passing Python the - ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment - variable to ``1``. + * :mod:`!cgi` and :mod:`!cgitb`. -* When the ``--enable-experimental-jit`` option is used, the optimized - Tier 2 IR is translated to machine code, which is then executed. - This does not require additional runtime options. + * ``cgi.FieldStorage`` can typically be replaced with + :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, + and the :mod:`email.message` module or :pypi:`multipart` + PyPI project for ``POST`` and ``PUT``. -* The machine code translation process uses an architecture called - *copy-and-patch*. It has no runtime dependencies, but there is a new - build-time dependency on LLVM. + * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` + directly on the desired query string, except for ``multipart/form-data`` + input, which can be handled as described for ``cgi.parse_multipart()``. -See :pep:`744` for more details. + * ``cgi.parse_header()`` can be replaced with the functionality in the + :mod:`email` package, which implements the same MIME RFCs. For example, + with :class:`email.message.EmailMessage`:: -(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. -Tier 2 IR by Mark Shannon and Guido van Rossum. -Tier 2 optimizer by Ken Jin.) + from email.message import EmailMessage + msg = EmailMessage() + msg['content-type'] = 'application/json; charset="utf8"' + main, params = msg.get_content_type(), msg['content-type'].params + * ``cgi.parse_multipart()`` can be replaced with the functionality in the + :mod:`email` package (e.g. :class:`email.message.EmailMessage` and + :class:`email.message.Message`) which implements the same MIME RFCs, or + with the :pypi:`multipart` PyPI project. + (Contributed by Victor Stinner in :gh:`104773`.) -Deprecated -========== + * :mod:`!crypt` module and its private :mod:`!_crypt` extension. + The :mod:`hashlib` module is a potential replacement for certain use cases. + Otherwise, the following PyPI projects can be used: -* :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, - emits :exc:`DeprecationWarning` since 3.13 - and will be removed in Python 3.16. - Use the ``'w'`` format code instead. - (Contributed by Hugo van Kemenade in :gh:`80480`.) + * :pypi:`bcrypt`: + Modern password hashing for your software and your servers. + * :pypi:`passlib`: + Comprehensive password hashing framework supporting over 30 schemes. + * :pypi:`argon2-cffi`: + The secure Argon2 password hashing algorithm. + * :pypi:`legacycrypt`: + :mod:`ctypes` wrapper to the POSIX crypt library call and associated functionality. + * :pypi:`crypt_r`: + Fork of the :mod:`!crypt` module, wrapper to the :manpage:`crypt_r(3)` library + call and associated functionality. -* :mod:`ctypes`: Deprecate undocumented :func:`!ctypes.SetPointerType` - and :func:`!ctypes.ARRAY` functions. - Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. - (Contributed by Victor Stinner in :gh:`105733`.) + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`decimal`: Deprecate non-standard format specifier "N" for - :class:`decimal.Decimal`. - It was not documented and only supported in the C implementation. - (Contributed by Serhiy Storchaka in :gh:`89902`.) + * :mod:`!imghdr`: use the projects :pypi:`filetype`, + :pypi:`puremagic`, or :pypi:`python-magic` instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check - membership in :data:`~dis.hasarg` instead. - (Contributed by Irit Katriel in :gh:`109319`.) + * :mod:`!mailcap`. + The :mod:`mimetypes` module provides an alternative. + (Contributed by Victor Stinner in :gh:`104773`.) -* :ref:`frame-objects`: - Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` - (as has always been the case for an executing frame). - (Contributed by Irit Katriel in :gh:`79932`.) + * :mod:`!msilib`. + (Contributed by Zachary Ware in :gh:`104773`.) -* :mod:`getopt` and :mod:`optparse` modules: They are now - :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. - Previously, the :mod:`optparse` module was already deprecated, its removal - was not scheduled, and no warnings was emitted: so there is no change in - practice. - (Contributed by Victor Stinner in :gh:`106535`.) + * :mod:`!nis`. + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`gettext`: Emit deprecation warning for non-integer numbers in - :mod:`gettext` functions and methods that consider plural forms even if the - translation was not found. - (Contributed by Serhiy Storchaka in :gh:`88434`.) + * :mod:`!nntplib`: + the :pypi:`nntplib` PyPI project can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`glob`: The undocumented :func:`!glob.glob0` and :func:`!glob.glob1` - functions are deprecated. Use :func:`glob.glob` and pass a directory to its - *root_dir* argument instead. - (Contributed by Barney Gale in :gh:`117337`.) + * :mod:`!ossaudiodev`: use the + `pygame project `_ for audio playback. + (Contributed by Victor Stinner in :gh:`104780`.) -* :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a - :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI - HTTP servers have been out of favor for a very long time. This code was - outdated, unmaintained, and rarely used. It has a high potential for both - security and functionality bugs. This includes removal of the ``--cgi`` - flag to the ``python -m http.server`` command line in 3.15. + * :mod:`!pipes`: use the :mod:`subprocess` module instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`pathlib`: - :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. + * :mod:`!sndhdr`: use the projects :pypi:`filetype`, + :pypi:`puremagic`, or :pypi:`python-magic` instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`platform`: - :func:`~platform.java_ver` is deprecated and will be removed in 3.15. - It was largely untested, had a confusing API, - and was only useful for Jython support. - (Contributed by Nikita Sobolev in :gh:`116349`.) + * :mod:`!spwd`: + the :pypi:`python-pam` project can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. - (Contributed by Zackery Spytz in :gh:`64020`.) + * :mod:`!sunau`. + (Contributed by Victor Stinner in :gh:`104773`.) -* :mod:`sqlite3`: Passing more than one positional argument to - :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is - deprecated. The remaining parameters will become keyword-only in Python 3.15. + * :mod:`!telnetlib`, use the projects :pypi:`telnetlib3 ` or + :pypi:`Exscript` instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!uu`: the :mod:`base64` module is a modern alternative. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!xdrlib`. + (Contributed by Victor Stinner in :gh:`104773`.) + +* Remove the ``2to3`` program and the :mod:`!lib2to3` module, + deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) + +* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The + third-party Tix library which the module wrapped is unmaintained. + (Contributed by Zachary Ware in :gh:`75552`.) + +configparser +------------ + +* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, + deprecated in the docstring since Python 3.2, + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104886`.) + +importlib +--------- + +* Remove deprecated :meth:`~object.__getitem__` access for + :class:`!importlib.metadata.EntryPoint` objects. + (Contributed by Jason R. Coombs in :gh:`113175`.) + +locale +------ + +* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: + use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`104783`.) + +logging +------- + +* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and + ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated + since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` + method, :meth:`!logging.LoggerAdapter.warning` method and + :func:`logging.warning` function. + (Contributed by Victor Stinner in :gh:`105376`.) + +pathlib +------- + +* Remove support for using :class:`pathlib.Path` objects as context managers. + This functionality was deprecated and made a no-op in Python 3.9. + +re +-- + +* Remove undocumented, never working, and deprecated ``re.template`` function + and ``re.TEMPLATE`` flag (and ``re.T`` alias). + (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) + + +turtle +------ + +* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, + deprecated in docs since Python 3.1 + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104876`.) + +typing +------ + +* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, + are now removed. The items in those namespaces can be imported directly + from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) + +* Remove support for the keyword-argument method of creating + :class:`typing.TypedDict` types, deprecated in Python 3.11. + (Contributed by Tomas Roun in :gh:`104786`.) + +unittest +-------- + +* Remove the following :mod:`unittest` functions, deprecated in Python 3.11: + + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` + + Use :class:`~unittest.TestLoader` methods instead: + + * :meth:`unittest.TestLoader.loadTestsFromModule` + * :meth:`unittest.TestLoader.loadTestsFromTestCase` + * :meth:`unittest.TestLoader.getTestCaseNames` + + (Contributed by Hugo van Kemenade in :gh:`104835`.) + +* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` + method, deprecated in Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104992`.) + +urllib +------ + +* Remove *cafile*, *capath* and *cadefault* parameters of the + :func:`urllib.request.urlopen` function, deprecated in Python 3.6: pass + the *context* parameter instead. Use + :meth:`ssl.SSLContext.load_cert_chain` to load specific certificates, or + let :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. + (Contributed by Victor Stinner in :gh:`105382`.) + +webbrowser +---------- + +* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, + deprecated in Python 3.11. + Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. + (Contributed by Hugo van Kemenade in :gh:`104804`.) + +* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. + Use :attr:`webbrowser.MacOSXOSAScript.name ` + attribute instead. + (Contributed by Nikita Sobolev in :gh:`105546`.) + +New Deprecations +================ + +* Removed chained :class:`classmethod` descriptors (introduced in + :gh:`63272`). This can no longer be used to wrap other descriptors + such as :class:`property`. The core design of this feature was flawed + and caused a number of downstream problems. To "pass-through" a + :class:`classmethod`, consider using the :attr:`!__wrapped__` + attribute that was added in Python 3.10. (Contributed by Raymond + Hettinger in :gh:`89519`.) + +* :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, + emits :exc:`DeprecationWarning` since 3.13 + and will be removed in Python 3.16. + Use the ``'w'`` format code instead. + (Contributed by Hugo van Kemenade in :gh:`80480`.) + +* :mod:`ctypes`: Deprecate undocumented :func:`!ctypes.SetPointerType` + and :func:`!ctypes.ARRAY` functions. + Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. + (Contributed by Victor Stinner in :gh:`105733`.) + +* :mod:`decimal`: Deprecate non-standard format specifier "N" for + :class:`decimal.Decimal`. + It was not documented and only supported in the C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check + membership in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) + +* :ref:`frame-objects`: + Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` + (as has always been the case for an executing frame). + (Contributed by Irit Katriel in :gh:`79932`.) + +* :mod:`getopt` and :mod:`optparse` modules: They are now + :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. + Previously, the :mod:`optparse` module was already deprecated, its removal + was not scheduled, and no warnings was emitted: so there is no change in + practice. + (Contributed by Victor Stinner in :gh:`106535`.) + +* :mod:`gettext`: Emit deprecation warning for non-integer numbers in + :mod:`gettext` functions and methods that consider plural forms even if the + translation was not found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) + +* :mod:`glob`: The undocumented :func:`!glob.glob0` and :func:`!glob.glob1` + functions are deprecated. Use :func:`glob.glob` and pass a directory to its + *root_dir* argument instead. + (Contributed by Barney Gale in :gh:`117337`.) + +* :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a + :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI + HTTP servers have been out of favor for a very long time. This code was + outdated, unmaintained, and rarely used. It has a high potential for both + security and functionality bugs. This includes removal of the ``--cgi`` + flag to the ``python -m http.server`` command line in 3.15. + +* :mod:`mimetypes`: Passing file path instead of URL in :func:`~mimetypes.guess_type` is + :term:`soft deprecated`. Use :func:`~mimetypes.guess_file_type` instead. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + +* :mod:`re`: Passing optional arguments *maxsplit*, *count* and *flags* in module-level + functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional + arguments is now deprecated. In future Python versions these parameters will be + :ref:`keyword-only `. + (Contributed by Serhiy Storchaka in :gh:`56166`.) + +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`platform`: + :func:`~platform.java_ver` is deprecated and will be removed in 3.15. + It was largely untested, had a confusing API, + and was only useful for Jython support. + (Contributed by Nikita Sobolev in :gh:`116349`.) + +* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) + +* :mod:`sqlite3`: Passing more than one positional argument to + :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is + deprecated. The remaining parameters will become keyword-only in Python 3.15. Deprecate passing name, number of arguments, and the callable as keyword arguments for the following :class:`sqlite3.Connection` APIs: @@ -1174,6 +1645,7 @@ Pending Removal in Python 3.14 :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an exception in Python 3.14. + Pending Removal in Python 3.15 ------------------------------ @@ -1326,391 +1798,67 @@ although there is currently no date scheduled for their removal. instead. * ``ssl.OP_NO_SSL*`` options * ``ssl.OP_NO_TLS*`` options - * ``ssl.PROTOCOL_SSLv3`` - * ``ssl.PROTOCOL_TLS`` - * ``ssl.PROTOCOL_TLSv1`` - * ``ssl.PROTOCOL_TLSv1_1`` - * ``ssl.PROTOCOL_TLSv1_2`` - * ``ssl.TLSVersion.SSLv3`` - * ``ssl.TLSVersion.TLSv1`` - * ``ssl.TLSVersion.TLSv1_1`` - -* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and - ignored. - -* :mod:`threading` methods: - - * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. - * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. - * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: - use :attr:`threading.Thread.daemon` attribute. - * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: - use :attr:`threading.Thread.name` attribute. - * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. - * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. - -* :class:`typing.Text` (:gh:`92332`). - -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not ``None`` from a test case. - -* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead - - * ``splitattr()`` - * ``splithost()`` - * ``splitnport()`` - * ``splitpasswd()`` - * ``splitport()`` - * ``splitquery()`` - * ``splittag()`` - * ``splittype()`` - * ``splituser()`` - * ``splitvalue()`` - * ``to_bytes()`` - -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. - -* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial - writes. - -* :meth:`zipimport.zipimporter.load_module` is deprecated: - use :meth:`~zipimport.zipimporter.exec_module` instead. - - -Removed -======= - -.. _whatsnew313-pep594: - -PEP 594: dead batteries ------------------------ - -* :pep:`594` removed 19 modules from the standard library, - deprecated in Python 3.11: - - * :mod:`!aifc`. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!audioop`. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!chunk`. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!cgi` and :mod:`!cgitb`. - - * ``cgi.FieldStorage`` can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, - and the :mod:`email.message` module or :pypi:`multipart` - PyPI project for ``POST`` and ``PUT``. - - * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` - directly on the desired query string, except for ``multipart/form-data`` - input, which can be handled as described for ``cgi.parse_multipart()``. - - * ``cgi.parse_header()`` can be replaced with the functionality in the - :mod:`email` package, which implements the same MIME RFCs. For example, - with :class:`email.message.EmailMessage`:: - - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params - - * ``cgi.parse_multipart()`` can be replaced with the functionality in the - :mod:`email` package (e.g. :class:`email.message.EmailMessage` and - :class:`email.message.Message`) which implements the same MIME RFCs, or - with the :pypi:`multipart` PyPI project. - - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!crypt` module and its private :mod:`!_crypt` extension. - The :mod:`hashlib` module is a potential replacement for certain use cases. - Otherwise, the following PyPI projects can be used: - - * :pypi:`bcrypt`: - Modern password hashing for your software and your servers. - * :pypi:`passlib`: - Comprehensive password hashing framework supporting over 30 schemes. - * :pypi:`argon2-cffi`: - The secure Argon2 password hashing algorithm. - * :pypi:`legacycrypt`: - Wrapper to the POSIX crypt library call and associated functionality. - - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!imghdr`: use the projects :pypi:`filetype`, - :pypi:`puremagic`, or :pypi:`python-magic` instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!mailcap`. - The :mod:`mimetypes` module provides an alternative. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!msilib`. - (Contributed by Zachary Ware in :gh:`104773`.) - - * :mod:`!nis`. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!nntplib`: - the :pypi:`nntplib` PyPI project can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!ossaudiodev`: use the - `pygame project `_ for audio playback. - (Contributed by Victor Stinner in :gh:`104780`.) - - * :mod:`!pipes`: use the :mod:`subprocess` module instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!sndhdr`: use the projects :pypi:`filetype_, - :pypi:`puremagic`, or :pypi:`python-magic` instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!spwd`: - the :pypi:`python-pam` project can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!sunau`. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!telnetlib`, use the projects :pypi:`telnetlib3 ` or - :pypi:`Exscript` instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!uu`: the :mod:`base64` module is a modern alternative. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!xdrlib`. - (Contributed by Victor Stinner in :gh:`104773`.) - -2to3 ----- - -* Remove the ``2to3`` program and the :mod:`!lib2to3` module, - deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104780`.) - -configparser ------------- - -* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, - deprecated in the docstring since Python 3.2, - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104886`.) - -importlib ---------- - -* Remove deprecated :meth:`~object.__getitem__` access for - :class:`!importlib.metadata.EntryPoint` objects. - (Contributed by Jason R. Coombs in :gh:`113175`.) - -locale ------- - -* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: - use ``locale.setlocale(locale.LC_ALL, "")`` instead. - (Contributed by Victor Stinner in :gh:`104783`.) - -logging -------- - -* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and - ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated - since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` - method, :meth:`!logging.LoggerAdapter.warning` method and - :func:`logging.warning` function. - (Contributed by Victor Stinner in :gh:`105376`.) - -pathlib -------- - -* Remove support for using :class:`pathlib.Path` objects as context managers. - This functionality was deprecated and made a no-op in Python 3.9. - -re --- - -* Remove undocumented, never working, and deprecated ``re.template`` function - and ``re.TEMPLATE`` flag (and ``re.T`` alias). - (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) - -tkinter -------- - -* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The - third-party Tix library which the module wrapped is unmaintained. - (Contributed by Zachary Ware in :gh:`75552`.) - -turtle ------- - -* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, - deprecated in docs since Python 3.1 - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104876`.) - -typing ------- - -* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, - are now removed. The items in those namespaces can be imported directly - from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) - -* Remove support for the keyword-argument method of creating - :class:`typing.TypedDict` types, deprecated in Python 3.11. - (Contributed by Tomas Roun in :gh:`104786`.) - -unittest --------- - -* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: - - * :func:`!unittest.findTestCases` - * :func:`!unittest.makeSuite` - * :func:`!unittest.getTestCaseNames` - - Use :class:`~unittest.TestLoader` methods instead: - - * :meth:`unittest.TestLoader.loadTestsFromModule` - * :meth:`unittest.TestLoader.loadTestsFromTestCase` - * :meth:`unittest.TestLoader.getTestCaseNames` - - (Contributed by Hugo van Kemenade in :gh:`104835`.) - -* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` - method, deprecated in Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104992`.) - -urllib ------- - -* Remove *cafile*, *capath* and *cadefault* parameters of the - :func:`urllib.request.urlopen` function, deprecated in Python 3.6: use the - *context* parameter instead. Please use - :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - (Contributed by Victor Stinner in :gh:`105382`.) - -webbrowser ----------- - -* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, - deprecated in Python 3.11. - Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. - (Contributed by Hugo van Kemenade in :gh:`104804`.) - -* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. - Use :attr:`webbrowser.MacOSXOSAScript.name ` - attribute instead. - (Contributed by Nikita Sobolev in :gh:`105546`.) - -Others ------- - -* None yet - -CPython bytecode changes -======================== - -* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a - yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was - changed to add a bit indicating whether the except-depth is 1, which - is needed to optimize closing of generators. - (Contributed by Irit Katriel in :gh:`111354`.) - -Porting to Python 3.13 -====================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - -Changes in the Python API -------------------------- - -* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, - :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, - :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors which occurred when calling - them, now report them using :func:`sys.unraisablehook`. - You may replace them with other functions as - recommended in the documentation. - (Contributed by Serhiy Storchaka in :gh:`106672`.) - -* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to - retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or - :exc:`KeyError` on Unix platforms where the password database is empty. - -* The :mod:`threading` module now expects the :mod:`!_thread` module to have - an ``_is_main_interpreter`` attribute. It is a function with no - arguments that return ``True`` if the current interpreter is the - main interpreter. - - Any library or application that provides a custom ``_thread`` module - must provide ``_is_main_interpreter()``, just like the module's - other "private" attributes. - (See :gh:`112826`.) - -* :class:`mailbox.Maildir` now ignores files with a leading dot. - (Contributed by Zackery Spytz in :gh:`65559`.) + * ``ssl.PROTOCOL_SSLv3`` + * ``ssl.PROTOCOL_TLS`` + * ``ssl.PROTOCOL_TLSv1`` + * ``ssl.PROTOCOL_TLSv1_1`` + * ``ssl.PROTOCOL_TLSv1_2`` + * ``ssl.TLSVersion.SSLv3`` + * ``ssl.TLSVersion.TLSv1`` + * ``ssl.TLSVersion.TLSv1_1`` -* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both - files and directories if a pattern that ends with "``**``" is given, rather - than directories only. Users may add a trailing slash to match only - directories. +* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and + ignored. -* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` was - changed from integer (``1`` or ``2``) to string (``'rb'`` or ``'wb'``). - The value of the :attr:`!mode` attribute of the readable file-like object - returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. - (Contributed by Serhiy Storchaka in :gh:`115961`.) +* :mod:`threading` methods: -* :c:func:`!PyCode_GetFirstFree` is an unstable API now and has been renamed - to :c:func:`PyUnstable_Code_GetFirstFree`. - (Contributed by Bogdan Romanyuk in :gh:`115781`.) + * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. + * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. + * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: + use :attr:`threading.Thread.daemon` attribute. + * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: + use :attr:`threading.Thread.name` attribute. + * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. + * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. +* :class:`typing.Text` (:gh:`92332`). -Build Changes -============= +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. -* Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate - the :file:`configure` script. - (Contributed by Christian Heimes in :gh:`89886`.) +* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead -* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. - (Contributed by Erlend Aasland in :gh:`105875`.) + * ``splitattr()`` + * ``splithost()`` + * ``splitnport()`` + * ``splitpasswd()`` + * ``splitport()`` + * ``splitquery()`` + * ``splittag()`` + * ``splittype()`` + * ``splituser()`` + * ``splitvalue()`` + * ``to_bytes()`` -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with the Python release build and - :ref:`debug build `. - (Contributed by Victor Stinner in :gh:`108634`.) +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. -* Building CPython now requires a compiler with support for the C11 atomic - library, GCC built-in atomic functions, or MSVC interlocked intrinsics. +* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial + writes. -* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, - ``termios``, ``winsound``, - ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, - ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` - C extensions are now built with the - :ref:`limited C API `. - (Contributed by Victor Stinner in :gh:`85283`.) +* :meth:`zipimport.zipimporter.load_module` is deprecated: + use :meth:`~zipimport.zipimporter.exec_module` instead. -* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. - (Contributed by Brett Cannon in :gh:`115192`.) -* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. - (Contributed by Brett Cannon in :gh:`115192`.) +CPython Bytecode Changes +======================== -* Python now bundles the `mimalloc library `__. - It is licensed under the MIT license, see :ref:`mimalloc license `. - The bundled mimalloc has custom changes, see :gh:`113141` for details. - (Contributed by Dino Viehland in :gh:`109914`.) +* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a + yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was + changed to add a bit indicating whether the except-depth is 1, which + is needed to optimize closing of generators. + (Contributed by Irit Katriel in :gh:`111354`.) C API Changes @@ -1882,9 +2030,15 @@ New Features * :c:type:`PyTime_t` type. * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. - * :c:func:`PyTime_AsSecondsDouble` - :c:func:`PyTime_Monotonic`, :c:func:`PyTime_PerfCounter`, and - :c:func:`PyTime_Time` functions. + * Add functions: + + * :c:func:`PyTime_AsSecondsDouble`. + * :c:func:`PyTime_Monotonic`. + * :c:func:`PyTime_MonotonicRaw`. + * :c:func:`PyTime_PerfCounter`. + * :c:func:`PyTime_PerfCounterRaw`. + * :c:func:`PyTime_Time`. + * :c:func:`PyTime_TimeRaw`. (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) @@ -1917,9 +2071,103 @@ New Features * Add :c:func:`PyType_GetModuleByDef` to the limited C API (Contributed by Victor Stinner in :gh:`116936`.) +* Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and + :c:func:`PyRefTracer_GetTracer`, that allows to track object creation and + destruction the same way the :mod:`tracemalloc` module does. (Contributed + by Pablo Galindo in :gh:`93502`.) + + +Build Changes +============= + +* The :file:`configure` option :option:`--with-system-libmpdec` now defaults + to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python + 3.15. + +* Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate + the :file:`configure` script. + (Contributed by Christian Heimes in :gh:`89886`.) + +* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. + (Contributed by Erlend Aasland in :gh:`105875`.) + +* Python built with :file:`configure` :option:`--with-trace-refs` (tracing + references) is now ABI compatible with the Python release build and + :ref:`debug build `. + (Contributed by Victor Stinner in :gh:`108634`.) + +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. + +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, + ``termios``, ``winsound``, + ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` + C extensions are now built with the :ref:`limited C API `. + (Contributed by Victor Stinner in :gh:`85283`.) + +* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* Python now bundles the `mimalloc library `__. + It is licensed under the MIT license; see :ref:`mimalloc license `. + The bundled mimalloc has custom changes, see :gh:`113141` for details. + (Contributed by Dino Viehland in :gh:`109914`.) + Porting to Python 3.13 ----------------------- +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to + retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or + :exc:`KeyError` on Unix platforms where the password database is empty. + +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + must provide ``_is_main_interpreter()``, just like the module's + other "private" attributes. + (See :gh:`112826`.) + +* :class:`mailbox.Maildir` now ignores files with a leading dot. + (Contributed by Zackery Spytz in :gh:`65559`.) + +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, rather + than directories only. Users may add a trailing slash to match only + directories. + +* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` was + changed from integer (``1`` or ``2``) to string (``'rb'`` or ``'wb'``). + The value of the :attr:`!mode` attribute of the readable file-like object + returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + +* Callbacks registered in the :mod:`tkinter` module now take arguments as + various Python objects (``int``, ``float``, ``bytes``, ``tuple``), + not just ``str``. + To restore the previous behavior set :mod:`!tkinter` module global + :data:`!wantobject` to ``1`` before creating the + :class:`!Tk` object or call the :meth:`!wantobject` + method of the :class:`!Tk` object with argument ``1``. + Calling it with argument ``2`` restores the current default behavior. + (Contributed by Serhiy Storchaka in :gh:`66410`.) + + +Changes in the C API +-------------------- * ``Python.h`` no longer includes the ```` standard header. It was included for the ``finite()`` function which is now provided by the @@ -1974,61 +2222,25 @@ Porting to Python 3.13 } Note that ``Py_TRASHCAN_BEGIN`` has a second argument which - should be the deallocation function it is in. - -Deprecated ----------- - -* Passing optional arguments *maxsplit*, *count* and *flags* in module-level - functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional - arguments is now deprecated. - In future Python versions these parameters will be - :ref:`keyword-only `. - (Contributed by Serhiy Storchaka in :gh:`56166`.) - -* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly - the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and - ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. - (Contributed by Victor Stinner in :gh:`105156`.) - -* Deprecate old Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: - clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. - - Functions scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105145`.) - -* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just - an alias to :c:func:`PyImport_ImportModule` since Python 3.3. - Scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105396`.) + should be the deallocation function it is in. The new macros were + added in Python 3.8 and the old macros were deprecated in Python 3.11. + (Contributed by Irit Katriel in :gh:`105111`.) -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed - reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it - returns a :term:`strong reference`. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyWeakref_GetRef` on Python 3.12 and older. - (Contributed by Victor Stinner in :gh:`105927`.) +* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, + :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, + :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and + :c:func:`PySys_GetObject`, which clear all errors which occurred when calling + them, now report them using :func:`sys.unraisablehook`. + You may replace them with other functions as + recommended in the documentation. + (Contributed by Serhiy Storchaka in :gh:`106672`.) -Removed -------- +* :c:func:`!PyCode_GetFirstFree` is an unstable API now and has been renamed + to :c:func:`PyUnstable_Code_GetFirstFree`. + (Contributed by Bogdan Romanyuk in :gh:`115781`.) -* Removed chained :class:`classmethod` descriptors (introduced in - :gh:`63272`). This can no longer be used to wrap other descriptors - such as :class:`property`. The core design of this feature was flawed - and caused a number of downstream problems. To "pass-through" a - :class:`classmethod`, consider using the :attr:`!__wrapped__` - attribute that was added in Python 3.10. (Contributed by Raymond - Hettinger in :gh:`89519`.) +Removed C APIs +-------------- * Remove many APIs (functions, macros, variables) with names prefixed by ``_Py`` or ``_PY`` (considered as private API). If your project is affected @@ -2108,12 +2320,6 @@ Removed Configuration ` instead (:pep:`587`), added to Python 3.8. (Contributed by Victor Stinner in :gh:`105145`.) -* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and - ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros - ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were - added in Python 3.8 and the old macros were deprecated in Python 3.11. - (Contributed by Irit Katriel in :gh:`105111`.) - * Remove ``PyEval_ThreadsInitialized()`` function, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` always creates the GIL: calling ``PyEval_InitThreads()`` does nothing and @@ -2150,11 +2356,48 @@ Removed be used instead. (Contributed by Serhiy Storchaka in :gh:`86493`.) +Deprecated C APIs +----------------- + +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly + the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and + ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. + (Contributed by Victor Stinner in :gh:`105156`.) + +* Deprecate old Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or + :envvar:`PYTHONHOME` environment variable instead. + + Functions scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105145`.) + +* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just + an alias to :c:func:`PyImport_ImportModule` since Python 3.3. + Scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105396`.) + +* Deprecate the :c:func:`PyWeakref_GetObject` and + :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed + reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it + returns a :term:`strong reference`. The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Victor Stinner in :gh:`105927`.) + Pending Removal in Python 3.14 ------------------------------ * Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. + * Functions to configure the Python initialization, deprecated in Python 3.11: * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. @@ -2196,6 +2439,7 @@ Pending Removal in Python 3.14 Pending Removal in Python 3.15 ------------------------------ +* The bundled copy of ``libmpdecimal``. * :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`. * :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. * :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. diff --git a/Doc/whatsnew/3.14 b/Doc/whatsnew/3.14 new file mode 100644 index 00000000000000..14628f666dd079 --- /dev/null +++ b/Doc/whatsnew/3.14 @@ -0,0 +1,131 @@ + +**************************** + What's New In Python 3.14 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.14, compared to 3.13. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.14 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.14. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + + + +Other Language Changes +====================== + + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + + +Optimizations +============= + + + + +Deprecated +========== + + + +Removed +======= + + + +Porting to Python 3.14 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Build Changes +============= + + +C API Changes +============= + +New Features +------------ + +Porting to Python 3.14 +---------------------- + +Deprecated +---------- + +Removed +------- + diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 52474517f5facc..a6b38207b70c93 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -785,8 +785,8 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - `_\, `recipe 577479 - `_\, :issue:`10586`, and + `_\, `recipe 577479 + `_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute diff --git a/Grammar/python.gram b/Grammar/python.gram index 05d7837e3aa6db..c04bc641779c04 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -269,11 +269,11 @@ function_def[stmt_ty]: function_def_raw[stmt_ty]: | invalid_def_raw - | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) } - | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'async' 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, @@ -641,27 +641,33 @@ type_alias[stmt_ty]: # Type parameter declaration # -------------------------- -type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { +type_params[asdl_type_param_seq*]: + | invalid_type_params + | '[' t=type_param_seq ']' { CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } type_param[type_param_ty] (memo): - | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } + | a=NAME b=[type_param_bound] c=[type_param_default] { _PyAST_TypeVar(a->v.Name.id, b, c, EXTRA) } | '*' a=NAME colon=':' e=expression { RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind ? "cannot use constraints with TypeVarTuple" : "cannot use bound with TypeVarTuple") } - | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) } + | '*' a=NAME b=[type_param_starred_default] { _PyAST_TypeVarTuple(a->v.Name.id, b, EXTRA) } | '**' a=NAME colon=':' e=expression { RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind ? "cannot use constraints with ParamSpec" : "cannot use bound with ParamSpec") } - | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) } + | '**' a=NAME b=[type_param_default] { _PyAST_ParamSpec(a->v.Name.id, b, EXTRA) } type_param_bound[expr_ty]: ':' e=expression { e } +type_param_default[expr_ty]: '=' e=expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } +type_param_starred_default[expr_ty]: '=' e=star_expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } # EXPRESSIONS # ----------- @@ -1388,6 +1394,7 @@ invalid_for_stmt: invalid_def_raw: | ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) } + | ['async'] 'def' NAME [type_params] &&'(' [params] ')' ['->' expression] &&':' [func_type_comment] block invalid_class_def_raw: | 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT { @@ -1431,3 +1438,9 @@ invalid_arithmetic: | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } invalid_factor: | ('+' | '-' | '~') a='not' b=factor { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } + +invalid_type_params: + | '[' token=']' { + RAISE_SYNTAX_ERROR_STARTING_FROM( + token, + "Type parameter list cannot be empty")} diff --git a/Include/Python.h b/Include/Python.h index bb771fb3aec980..e05901b9e52b5a 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -84,6 +84,7 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" +#include "monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" diff --git a/Include/ceval.h b/Include/ceval.h index 8ea9da8d134ee0..1ec746c3708220 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -22,6 +22,10 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameBuiltins(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameGlobals(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameLocals(void); + PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b0e226e0e1971a..ef8f9304ccab56 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -226,7 +226,7 @@ struct PyCodeObject _PyCode_DEF(1); */ #define PY_PARSER_REQUIRES_FUTURE_KEYWORD -#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */ +#define CO_MAXBLOCKS 21 /* Max static block nesting within a function */ PyAPI_DATA(PyTypeObject) PyCode_Type; diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..dbbfbb5105ba7a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -27,3 +27,9 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + +typedef struct { + PyObject_HEAD + PyFrameObject* frame; +} PyFrameLocalsProxyObject; diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h new file mode 100644 index 00000000000000..efb9ec0e587552 --- /dev/null +++ b/Include/cpython/monitoring.h @@ -0,0 +1,250 @@ +#ifndef Py_CPYTHON_MONITORING_H +# error "this header file must not be included directly" +#endif + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 + +#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ + ((ev) < _PY_MONITORING_LOCAL_EVENTS) + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 +#define PY_MONITORING_EVENT_RERAISE 14 + + +/* Ancillary events */ + +#define PY_MONITORING_EVENT_C_RETURN 15 +#define PY_MONITORING_EVENT_C_RAISE 16 + + +typedef struct _PyMonitoringState { + uint8_t active; + uint8_t opaque; +} PyMonitoringState; + + +PyAPI_FUNC(int) +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length); + +PyAPI_FUNC(int) +PyMonitoring_ExitScope(void); + + +PyAPI_FUNC(int) +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0); + +PyAPI_FUNC(int) +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno); + +PyAPI_FUNC(int) +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + + +#define _PYMONITORING_IF_ACTIVE(STATE, X) \ + if ((STATE)->active) { \ + return (X); \ + } \ + else { \ + return 0; \ + } + +static inline int +PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyStartEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyResumeEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0)); +} + +static inline int +PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireLineEvent(state, codelike, offset, lineno)); +} + +static inline int +PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyThrowEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireReraiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyUnwindEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireStopIterationEvent(state, codelike, offset)); +} + +#undef _PYMONITORING_IF_ACTIVE diff --git a/Include/cpython/object.h b/Include/cpython/object.h index a6b93b93ab0f7a..e624326693d8e7 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -275,6 +275,7 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); @@ -510,3 +511,13 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); * assigned, or 0 if a new tag could not be assigned. */ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); + + +typedef enum { + PyRefTracer_CREATE = 0, + PyRefTracer_DESTROY = 1, +} PyRefTracerEvent; + +typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); +PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); +PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 819251a25bb242..5f218d75b346a0 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -24,6 +24,7 @@ typedef struct { uint8_t opcode; uint8_t oparg; uint8_t valid; + uint8_t linked; int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; @@ -101,6 +102,7 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; + void *jit_side_entry; _PyExitData exits[1]; } _PyExecutorObject; @@ -135,13 +137,10 @@ PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); -void _Py_ExecutorClear(_PyExecutorObject *); +void _Py_ExecutorDetach(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); - /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); @@ -149,6 +148,15 @@ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 +#ifdef _Py_TIER2 +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +#else +# define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) +# define _Py_Executors_InvalidateAll(A, B) ((void)0) +#endif + + #ifdef __cplusplus } #endif diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 69083f1d9dd0c2..55a139bb9158db 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -484,6 +484,9 @@ _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index af78a94c736545..c0f3747be45758 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -516,6 +516,10 @@ static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 212cd7817d01c5..f32995c1f578ac 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -989,6 +989,19 @@ _Py_atomic_load_int_acquire(const int *obj) #endif } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint32_release" +#endif +} + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 6a77eae536d8dd..0cdce4e6dd39f0 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -911,6 +911,14 @@ _Py_atomic_load_int_acquire(const int *obj) memory_order_acquire); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index c5adbbe4868f69..eeafbb17a56bad 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -3,8 +3,10 @@ #endif PyAPI_DATA(PyTypeObject) PyFrame_Type; +PyAPI_DATA(PyTypeObject) PyFrameLocalsProxy_Type; #define PyFrame_Check(op) Py_IS_TYPE((op), &PyFrame_Type) +#define PyFrameLocalsProxy_Check(op) Py_IS_TYPE((op), &PyFrameLocalsProxy_Type) PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0611e299403031..2df9ecd6d52084 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -188,6 +188,7 @@ struct _ts { PyObject *previous_executor; + uint64_t dict_global_version; }; #ifdef Py_DEBUG diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h index d8244700d614ce..5c68110aeedb86 100644 --- a/Include/cpython/pytime.h +++ b/Include/cpython/pytime.h @@ -16,6 +16,10 @@ PyAPI_FUNC(int) PyTime_Monotonic(PyTime_t *result); PyAPI_FUNC(int) PyTime_PerfCounter(PyTime_t *result); PyAPI_FUNC(int) PyTime_Time(PyTime_t *result); +PyAPI_FUNC(int) PyTime_MonotonicRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounterRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_TimeRaw(PyTime_t *result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index f222d485e0b54b..f5bf1205a82be9 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -657,14 +657,17 @@ struct _type_param { struct { identifier name; expr_ty bound; + expr_ty default_value; } TypeVar; struct { identifier name; + expr_ty default_value; } ParamSpec; struct { identifier name; + expr_ty default_value; } TypeVarTuple; } v; @@ -892,14 +895,15 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); -type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena - *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, expr_ty + default_value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index f1b1786264803b..09ae95465495c0 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -184,6 +184,7 @@ struct ast_state { PyObject *conversion; PyObject *ctx; PyObject *decorator_list; + PyObject *default_value; PyObject *defaults; PyObject *elt; PyObject *elts; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index decf92bc419c04..90735b202c7a93 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -44,13 +44,18 @@ make_backoff_counter(uint16_t value, uint16_t backoff) { assert(backoff <= 15); assert(value <= 0xFFF); - return (_Py_BackoffCounter){.backoff = backoff, .value = value}; + _Py_BackoffCounter result; + result.value = value; + result.backoff = backoff; + return result; } static inline _Py_BackoffCounter forge_backoff_counter(uint16_t counter) { - return (_Py_BackoffCounter){.as_counter = counter}; + _Py_BackoffCounter result; + result.as_counter = counter; + return result; } static inline _Py_BackoffCounter diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index cfb88c3f4c8e15..48ad0678995904 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -108,6 +108,7 @@ extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; +extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; #endif static inline PyObject* @@ -130,9 +131,54 @@ extern int _PyEval_ThreadsInitialized(void); extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); -extern void _PyEval_AcquireLock(PyThreadState *tstate); +// Acquire the GIL and return 1. In free-threaded builds, this function may +// return 0 to indicate that the GIL was disabled and therefore not acquired. +extern int _PyEval_AcquireLock(PyThreadState *tstate); + extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); +#ifdef Py_GIL_DISABLED +// Returns 0 or 1 if the GIL for the given thread's interpreter is disabled or +// enabled, respectively. +// +// The enabled state of the GIL will not change while one or more threads are +// attached. +static inline int +_PyEval_IsGILEnabled(PyThreadState *tstate) +{ + return tstate->interp->ceval.gil->enabled != 0; +} + +// Enable or disable the GIL used by the interpreter that owns tstate, which +// must be the current thread. This may affect other interpreters, if the GIL +// is shared. All three functions will be no-ops (and return 0) if the +// interpreter's `enable_gil' config is not _PyConfig_GIL_DEFAULT. +// +// Every call to _PyEval_EnableGILTransient() must be paired with exactly one +// call to either _PyEval_EnableGILPermanent() or +// _PyEval_DisableGIL(). _PyEval_EnableGILPermanent() and _PyEval_DisableGIL() +// must only be called while the GIL is enabled from a call to +// _PyEval_EnableGILTransient(). +// +// _PyEval_EnableGILTransient() returns 1 if it enabled the GIL, or 0 if the +// GIL was already enabled, whether transiently or permanently. The caller will +// hold the GIL upon return. +// +// _PyEval_EnableGILPermanent() returns 1 if it permanently enabled the GIL +// (which must already be enabled), or 0 if it was already permanently +// enabled. Once _PyEval_EnableGILPermanent() has been called once, all +// subsequent calls to any of the three functions will be no-ops. +// +// _PyEval_DisableGIL() returns 1 if it disabled the GIL, or 0 if the GIL was +// kept enabled because of another request, whether transient or permanent. +// +// All three functions must be called by an attached thread (this implies that +// if the GIL is enabled, the current thread must hold it). +extern int _PyEval_EnableGILTransient(PyThreadState *tstate); +extern int _PyEval_EnableGILPermanent(PyThreadState *tstate); +extern int _PyEval_DisableGIL(PyThreadState *state); +#endif + extern void _PyEval_DeactivateOpCache(void); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 11f2a100bf531e..009a1ea41eb985 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -41,7 +41,7 @@ struct _pending_call { #define MAXPENDINGCALLSLOOP_MAIN 0 struct _pending_calls { - int busy; + PyThreadState *handling_thread; PyMutex mutex; /* Request for running pending calls. */ int32_t npending; @@ -75,6 +75,7 @@ struct trampoline_api_st { unsigned int code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; + Py_ssize_t code_padding; }; #endif @@ -83,6 +84,7 @@ struct _ceval_runtime_state { struct { #ifdef PY_HAVE_PERF_TRAMPOLINE perf_status_t status; + int perf_trampoline_type; Py_ssize_t extra_code_index; struct code_arena_st *code_arena; struct trampoline_api_st trampoline_api; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 8d832c59e36874..bcbaf60f226c77 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex + // We hide some of the newer PyCodeObject fields behind macros. // This helps with backporting certain changes to 3.12. @@ -16,6 +18,14 @@ extern "C" { #define _PyCode_HAS_INSTRUMENTATION(CODE) \ (CODE->_co_instrumentation_version > 0) +struct _py_code_state { + PyMutex mutex; + // Interned constants from code objects. Used by the free-threaded build. + struct _Py_hashtable_t *constants; +}; + +extern PyStatus _PyCode_Init(PyInterpreterState *interp); +extern void _PyCode_Fini(PyInterpreterState *interp); #define CODE_MAX_WATCHERS 8 diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index a2a7151d50ade7..5e2d5c5ce9d868 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -8,6 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex + +/* Initialize codecs-related state for the given interpreter, including + registering the first codec search function. Must be called before any other + PyCodec-related functions, and while only one thread is active. */ +extern PyStatus _PyCodec_InitRegistry(PyInterpreterState *interp); + +/* Finalize codecs-related state for the given interpreter. No PyCodec-related + functions other than PyCodec_Unregister() may be called after this. */ +extern void _PyCodec_Fini(PyInterpreterState *interp); + extern PyObject* _PyCodec_Lookup(const char *encoding); /* Text codec specific encoding and decoding API. @@ -48,6 +59,26 @@ extern PyObject* _PyCodecInfo_GetIncrementalEncoder( PyObject *codec_info, const char *errors); +// Per-interpreter state used by codecs.c. +struct codecs_state { + // A list of callable objects used to search for codecs. + PyObject *search_path; + + // A dict mapping codec names to codecs returned from a callable in + // search_path. + PyObject *search_cache; + + // A dict mapping error handling strategies to functions to implement them. + PyObject *error_registry; + +#ifdef Py_GIL_DISABLED + // Used to safely delete a specific item from search_path. + PyMutex search_path_mutex; +#endif + + // Whether or not the rest of the state is initialized. + int initialized; +}; #ifdef __cplusplus } diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index f33026dbd6be58..8d8d3748edaea8 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -105,7 +105,10 @@ PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObjec /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); +extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); +extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, @@ -218,8 +221,25 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) #ifdef Py_GIL_DISABLED -#define DICT_NEXT_VERSION(INTERP) \ - (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) + +#define THREAD_LOCAL_DICT_VERSION_COUNT 256 +#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT * DICT_VERSION_INCREMENT + +static inline uint64_t +dict_next_version(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyThreadState_GET(); + uint64_t cur_progress = (tstate->dict_global_version & + (THREAD_LOCAL_DICT_VERSION_BATCH - 1)); + if (cur_progress == 0) { + uint64_t next = _Py_atomic_add_uint64(&interp->dict_state.global_version, + THREAD_LOCAL_DICT_VERSION_BATCH); + tstate->dict_global_version = next; + } + return tstate->dict_global_version += DICT_VERSION_INCREMENT; +} + +#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP) #else #define DICT_NEXT_VERSION(INTERP) \ diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index bc8100b58e8ea3..13f86b01bbfe8f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -326,6 +326,9 @@ extern int _PyFile_Flush(PyObject *); extern int _Py_GetTicksPerSecond(long *ticks_per_second); #endif +// Export for '_testcapi' shared extension +PyAPI_FUNC(int) _Py_IsValidFD(int fd); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index f913928f38bd05..994900c007f4bd 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -25,7 +25,7 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ + PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -227,6 +227,9 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame); + /* Clears all references in the frame. * If take is non-zero, then the _PyInterpreterFrame frame * may be transferred to the frame object it references @@ -242,14 +245,11 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); - -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame); -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear); +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame); static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) @@ -315,6 +315,11 @@ PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) return (PyGenObject *)(((char *)frame) - offset_in_gen); } +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject* const* args, + size_t argcount, PyObject *kwnames); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 24fbb3ddbee602..6d44e933e8a8cb 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -4,6 +4,8 @@ extern "C" { #endif +#include "pycore_lock.h" + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -24,6 +26,11 @@ struct _func_version_cache_item { }; struct _py_func_state { +#ifdef Py_GIL_DISABLED + // Protects next_version + PyMutex mutex; +#endif + uint32_t next_version; // Borrowed references to function and code objects whose // func_version % FUNC_VERSION_CACHE_SIZE diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index d36b4c0db010b2..a2de5077371eba 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -21,8 +21,20 @@ extern "C" { struct _gil_runtime_state { #ifdef Py_GIL_DISABLED - /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime - if, for example, a module that requires the GIL is loaded. */ + /* If this GIL is disabled, enabled == 0. + + If this GIL is enabled transiently (most likely to initialize a module + of unknown safety), enabled indicates the number of active transient + requests. + + If this GIL is enabled permanently, enabled == INT_MAX. + + It must not be modified directly; use _PyEval_EnableGILTransiently(), + _PyEval_EnableGILPermanently(), and _PyEval_DisableGIL() + + It is always read and written atomically, but a thread can assume its + value will be stable as long as that thread is attached or knows that no + other threads are attached (e.g., during a stop-the-world.). */ int enabled; #endif /* microseconds (the Python API uses seconds, though) */ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4a6f40c84088e8..ca7355b2b61aa7 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -624,6 +624,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__eq__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__exit__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__file__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__firstlineno__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__float__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__floordiv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__format__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8332cdf874c0c9..fbb25285f0f282 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -113,6 +113,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__eq__) STRUCT_FOR_ID(__exit__) STRUCT_FOR_ID(__file__) + STRUCT_FOR_ID(__firstlineno__) STRUCT_FOR_ID(__float__) STRUCT_FOR_ID(__floordiv__) STRUCT_FOR_ID(__format__) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index b02769903a6f9b..bd40707fed21a8 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -206,6 +206,19 @@ extern int _PyImport_CheckSubinterpIncompatibleExtensionAllowed( // Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); +#ifdef Py_GIL_DISABLED +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), resolve the transient request depending on the +// state of the module argument: +// - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, +// call _PyEval_DisableGIL(). +// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already +// enabled permanently, issue a warning referencing the module's name. +// +// This function may raise an exception. +extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index adb89167d124eb..e5f222b371a113 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -15,8 +15,20 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; -typedef PyObject *(*PyModInitFunction)(void); - +typedef enum ext_module_kind { + _Py_ext_module_kind_UNKNOWN = 0, + _Py_ext_module_kind_SINGLEPHASE = 1, + _Py_ext_module_kind_MULTIPHASE = 2, + _Py_ext_module_kind_INVALID = 3, +} _Py_ext_module_kind; + +typedef enum ext_module_origin { + _Py_ext_module_origin_CORE = 1, + _Py_ext_module_origin_BUILTIN = 2, + _Py_ext_module_origin_DYNAMIC = 3, +} _Py_ext_module_origin; + +/* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; #ifndef MS_WINDOWS @@ -27,6 +39,7 @@ struct _Py_ext_module_loader_info { /* path is always a borrowed ref of name or filename, * depending on if it's builtin or not. */ PyObject *path; + _Py_ext_module_origin origin; const char *hook_prefix; const char *newcontext; }; @@ -35,7 +48,11 @@ extern void _Py_ext_module_loader_info_clear( extern int _Py_ext_module_loader_info_init( struct _Py_ext_module_loader_info *info, PyObject *name, - PyObject *filename); + PyObject *filename, + _Py_ext_module_origin origin); +extern int _Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); extern int _Py_ext_module_loader_info_init_for_builtin( struct _Py_ext_module_loader_info *p_info, PyObject *name); @@ -43,10 +60,33 @@ extern int _Py_ext_module_loader_info_init_from_spec( struct _Py_ext_module_loader_info *info, PyObject *spec); +/* The result from running an extension module's init function. */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; + _Py_ext_module_kind kind; + struct _Py_ext_module_loader_result_error *err; + struct _Py_ext_module_loader_result_error { + enum _Py_ext_module_loader_result_error_kind { + _Py_ext_module_loader_result_EXCEPTION = 0, + _Py_ext_module_loader_result_ERR_MISSING = 1, + _Py_ext_module_loader_result_ERR_UNREPORTED_EXC = 2, + _Py_ext_module_loader_result_ERR_UNINITIALIZED = 3, + _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE = 4, + _Py_ext_module_loader_result_ERR_NOT_MODULE = 5, + _Py_ext_module_loader_result_ERR_MISSING_DEF = 6, + } kind; + PyObject *exc; + } _err; }; +extern void _Py_ext_module_loader_result_clear( + struct _Py_ext_module_loader_result *res); +extern void _Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name); + +/* The module init function. */ +typedef PyObject *(*PyModInitFunction)(void); extern PyModInitFunction _PyImport_GetModInitFunc( struct _Py_ext_module_loader_info *info, FILE *fp); diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 7f84d4a763bbcf..c98e82c8be5546 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -13,38 +13,6 @@ extern "C" { #define PY_MONITORING_TOOL_IDS 8 -/* Local events. - * These require bytecode instrumentation */ - -#define PY_MONITORING_EVENT_PY_START 0 -#define PY_MONITORING_EVENT_PY_RESUME 1 -#define PY_MONITORING_EVENT_PY_RETURN 2 -#define PY_MONITORING_EVENT_PY_YIELD 3 -#define PY_MONITORING_EVENT_CALL 4 -#define PY_MONITORING_EVENT_LINE 5 -#define PY_MONITORING_EVENT_INSTRUCTION 6 -#define PY_MONITORING_EVENT_JUMP 7 -#define PY_MONITORING_EVENT_BRANCH 8 -#define PY_MONITORING_EVENT_STOP_ITERATION 9 - -#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) - -/* Other events, mainly exceptions */ - -#define PY_MONITORING_EVENT_RAISE 10 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 -#define PY_MONITORING_EVENT_PY_UNWIND 12 -#define PY_MONITORING_EVENT_PY_THROW 13 -#define PY_MONITORING_EVENT_RERAISE 14 - - -/* Ancillary events */ - -#define PY_MONITORING_EVENT_C_RETURN 15 -#define PY_MONITORING_EVENT_C_RAISE 16 - - typedef uint32_t _PyMonitoringEventSet; /* Tool IDs */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d38959e3ce4ec5..86dada5061e7b5 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -14,6 +14,7 @@ extern "C" { #include "pycore_atexit.h" // struct atexit_state #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache +#include "pycore_codecs.h" // struct codecs_state #include "pycore_context.h" // struct _Py_context_state #include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_dict_state.h" // struct _Py_dict_state @@ -182,10 +183,7 @@ struct _is { possible to facilitate out-of-process observability tools. */ - PyObject *codec_search_path; - PyObject *codec_search_cache; - PyObject *codec_error_registry; - int codecs_initialized; + struct codecs_state codecs; PyConfig config; unsigned long feature_flags; @@ -247,6 +245,7 @@ struct _is { struct _Py_long_state long_state; struct _dtoa_state dtoa; struct _py_func_state func_state; + struct _py_code_state code_state; struct _Py_dict_state dict_state; struct _Py_exc_state exc_state; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 8fa88ea3f74caa..39c2a30f6e979d 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -28,8 +28,9 @@ #define INTRINSIC_TYPEVAR_WITH_BOUND 2 #define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 #define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 +#define INTRINSIC_SET_TYPEPARAM_DEFAULT 5 -#define MAX_INTRINSIC_2 4 +#define MAX_INTRINSIC_2 5 typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 2a82912e41d557..73695d10e0c372 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -28,7 +28,11 @@ _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&self->ob_item[len], newitem); +#else PyList_SET_ITEM(self, len, newitem); +#endif Py_SET_SIZE(self, len + 1); return 0; } diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index f993c95ecbf75a..a5b28e4bd4744e 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -150,8 +150,10 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, -// and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns); +// and 0 if the timeout expired or thread was interrupted. If `detach` is +// true, then the thread will detach/release the GIL while waiting. +PyAPI_FUNC(int) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach); // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 5644bbe5e0552b..049677b292e235 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -22,6 +22,9 @@ typedef struct { PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; +#ifdef Py_GIL_DISABLED + void *md_gil; +#endif } PyModuleObject; static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7df8003196d8cc..f15c332a7b0811 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -257,7 +257,7 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); when a memory block is reused from a free list. Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_NewReference(PyObject *op); +extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*); // Fast inlined version of PyType_HasFeature() static inline int @@ -658,6 +658,7 @@ extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); +extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4b1f43cf2af06e..2a237bc6dd8ee5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -95,6 +95,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; + case CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; case CALL_BUILTIN_CLASS: return 2 + oparg; case CALL_BUILTIN_FAST: @@ -125,9 +127,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case CALL_NON_PY_GENERAL: + return 2 + oparg; case CALL_PY_EXACT_ARGS: return 2 + oparg; - case CALL_PY_WITH_DEFAULTS: + case CALL_PY_GENERAL: return 2 + oparg; case CALL_STR_1: return 3; @@ -524,6 +528,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_BOUND_METHOD_EXACT_ARGS: return 0; + case CALL_BOUND_METHOD_GENERAL: + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -554,10 +560,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_METHOD_DESCRIPTOR_O: return 1; + case CALL_NON_PY_GENERAL: + return 1; case CALL_PY_EXACT_ARGS: return 0; - case CALL_PY_WITH_DEFAULTS: - return 1; + case CALL_PY_GENERAL: + return 0; case CALL_STR_1: return 1; case CALL_TUPLE_1: @@ -981,10 +989,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, 0 }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1000,8 +1009,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1029,7 +1039,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1121,7 +1131,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, @@ -1211,6 +1221,7 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1223,7 +1234,9 @@ _PyOpcode_macro_expansion[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, 0, 0 }, { _CALL_NON_PY_GENERAL, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_GENERAL] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, @@ -1347,6 +1360,7 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, 0, 0 } } }, + [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA @@ -1382,6 +1396,7 @@ const char *_PyOpcode_OpName[268] = { [CALL] = "CALL", [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BOUND_METHOD_GENERAL] = "CALL_BOUND_METHOD_GENERAL", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", @@ -1397,8 +1412,9 @@ const char *_PyOpcode_OpName[268] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_NON_PY_GENERAL] = "CALL_NON_PY_GENERAL", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_PY_GENERAL] = "CALL_PY_GENERAL", [CALL_STR_1] = "CALL_STR_1", [CALL_TUPLE_1] = "CALL_TUPLE_1", [CALL_TYPE_1] = "CALL_TYPE_1", @@ -1635,6 +1651,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL] = CALL, [CALL_ALLOC_AND_ENTER_INIT] = CALL, [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BOUND_METHOD_GENERAL] = CALL, [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, @@ -1650,8 +1667,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_METHOD_DESCRIPTOR_NOARGS] = CALL, [CALL_METHOD_DESCRIPTOR_O] = CALL, + [CALL_NON_PY_GENERAL] = CALL, [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, + [CALL_PY_GENERAL] = CALL, [CALL_STR_1] = CALL, [CALL_TUPLE_1] = CALL, [CALL_TYPE_1] = CALL, @@ -1851,8 +1869,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 221: \ - case 222: \ case 223: \ case 224: \ case 225: \ diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index bbfc462a733d0e..bc6aba56cf9fc7 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -39,6 +39,12 @@ extern "C" { _Py_atomic_load_uint8(&value) #define FT_ATOMIC_STORE_UINT8(value, new_value) \ _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ + _Py_atomic_load_uint8_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ + _Py_atomic_load_uint16_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ + _Py_atomic_load_uint32_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -49,6 +55,10 @@ extern "C" { _Py_atomic_store_ssize_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ _Py_atomic_store_uint8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ + _Py_atomic_store_uint16_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ + _Py_atomic_store_uint32_relaxed(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -62,11 +72,16 @@ extern "C" { #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index dc6f6f100f7a92..f58eccf729cb2a 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -132,6 +132,12 @@ typedef struct _Py_DebugOffsets { } unicode_object; } _Py_DebugOffsets; +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* tracer_data; +}; + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. @@ -236,6 +242,7 @@ typedef struct pyruntimestate { struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; + struct _reftracer_runtime_state ref_tracer; // The rwmutex is used to prevent overlapping global and per-interpreter // stop-the-world events. Global stop-the-world events lock the mutex diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 41331df8320a9c..98920dbb7c7a92 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -128,6 +128,10 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ + .ref_tracer = { \ + .tracer_func = NULL, \ + .tracer_data = NULL, \ + }, \ .stoptheworld = { \ .is_global = 1, \ }, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 103279a4cf228b..508da40c53422d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -622,6 +622,7 @@ extern "C" { INIT_ID(__eq__), \ INIT_ID(__exit__), \ INIT_ID(__file__), \ + INIT_ID(__firstlineno__), \ INIT_ID(__float__), \ INIT_ID(__floordiv__), \ INIT_ID(__format__), \ diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 41b351ead25dd1..0494c07fe1869d 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -30,6 +30,9 @@ PyAPI_DATA(PyObject *) _PySet_Dummy; PyAPI_FUNC(int) _PySet_Contains(PySetObject *so, PyObject *key); +// Clears the set without acquiring locks. Used by _PyCode_Fini. +extern void _PySet_ClearInternal(PySetObject *so); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h new file mode 100644 index 00000000000000..93898174789f7b --- /dev/null +++ b/Include/internal/pycore_stackref.h @@ -0,0 +1,195 @@ +#ifndef Py_INTERNAL_STACKREF_H +#define Py_INTERNAL_STACKREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +typedef union { + uintptr_t bits; +} _PyStackRef; + +static const _PyStackRef Py_STACKREF_NULL = { .bits = 0 }; + +#define Py_TAG_DEFERRED (1) + +// Gets a PyObject * from a _PyStackRef +#if defined(Py_GIL_DISABLED) +static inline PyObject * +PyStackRef_Get(_PyStackRef tagged) +{ + PyObject *cleared = ((PyObject *)((tagged).bits & (~Py_TAG_DEFERRED))); + return cleared; +} +#else +# define PyStackRef_Get(tagged) ((PyObject *)((tagged).bits)) +#endif + +// Converts a PyObject * to a PyStackRef, stealing the reference. +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_StealRef(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + return ((_PyStackRef){.bits = ((uintptr_t)(obj))}); +} +# define PyStackRef_StealRef(obj) _PyStackRef_StealRef(_PyObject_CAST(obj)) +#else +# define PyStackRef_StealRef(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) +#endif + +// Converts a PyObject * to a PyStackRef, with a new reference +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_NewRefDeferred(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + assert(obj != NULL); + if (_PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + } + else { + return (_PyStackRef){ .bits = (uintptr_t)Py_NewRef(obj) }; + } +} +# define PyStackRef_NewRefDeferred(obj) _PyStackRef_NewRefDeferred(_PyObject_CAST(obj)) +#else +# define PyStackRef_NewRefDeferred(obj) PyStackRef_NewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) +#endif + +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_XNewRefDeferred(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + if (obj == NULL) { + return Py_STACKREF_NULL; + } + return _PyStackRef_NewRefDeferred(obj); +} +# define PyStackRef_XNewRefDeferred(obj) _PyStackRef_XNewRefDeferred(_PyObject_CAST(obj)) +#else +# define PyStackRef_XNewRefDeferred(obj) PyStackRef_XNewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) +#endif + +// Converts a PyStackRef back to a PyObject *. +#if defined(Py_GIL_DISABLED) +static inline PyObject * +PyStackRef_StealObject(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); + return Py_NewRef(PyStackRef_Get(tagged)); + } + return PyStackRef_Get(tagged); +} +#else +# define PyStackRef_StealObject(tagged) PyStackRef_Get(tagged) +#endif + +static inline void +_Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length) +{ + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_Get(src[i]); + } +} + +static inline void +_Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length) +{ + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_StealObject(src[i]); + } +} + + +#define PyStackRef_XSETREF(dst, src) \ + do { \ + _PyStackRef *_tmp_dst_ptr = &(dst); \ + _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + PyStackRef_XDECREF(_tmp_old_dst); \ + } while (0) + +#define PyStackRef_SETREF(dst, src) \ + do { \ + _PyStackRef *_tmp_dst_ptr = &(dst); \ + _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + PyStackRef_DECREF(_tmp_old_dst); \ + } while (0) + +#define PyStackRef_CLEAR(op) \ + do { \ + _PyStackRef *_tmp_op_ptr = &(op); \ + _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op.bits != Py_STACKREF_NULL.bits) { \ + *_tmp_op_ptr = Py_STACKREF_NULL; \ + PyStackRef_DECREF(_tmp_old_op); \ + } \ + } while (0) + +#if defined(Py_GIL_DISABLED) +static inline void +PyStackRef_DECREF(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + return; + } + Py_DECREF(PyStackRef_Get(tagged)); +} +#else +# define PyStackRef_DECREF(op) Py_DECREF(PyStackRef_Get(op)) +#endif + +#if defined(Py_GIL_DISABLED) +static inline void +PyStackRef_INCREF(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); + return; + } + Py_INCREF(PyStackRef_Get(tagged)); +} +#else +# define PyStackRef_INCREF(op) Py_INCREF(PyStackRef_Get(op)) +#endif + +static inline void +PyStackRef_XDECREF(_PyStackRef op) +{ + if (op.bits != Py_STACKREF_NULL.bits) { + PyStackRef_DECREF(op); + } +} + +static inline _PyStackRef +PyStackRef_NewRef(_PyStackRef obj) +{ + PyStackRef_INCREF(obj); + return obj; +} + +static inline _PyStackRef +PyStackRef_XNewRef(_PyStackRef obj) +{ + if (obj.bits == Py_STACKREF_NULL.bits) { + return obj; + } + return PyStackRef_NewRef(obj); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STACKREF_H */ diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 138d60fdb69806..15806552e0a384 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -249,14 +249,6 @@ typedef struct { double resolution; } _Py_clock_info_t; -// Similar to PyTime_Time() but silently ignore the error and return 0 if the -// internal clock fails. -// -// Use _PyTime_TimeWithInfo() or the public PyTime_Time() to check -// for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_TimeUnchecked(void); - // Get the current time from the system clock. // On success, set *t and *info (if not NULL), and return 0. // On error, raise an exception and return -1. @@ -264,14 +256,6 @@ extern int _PyTime_TimeWithInfo( PyTime_t *t, _Py_clock_info_t *info); -// Similar to PyTime_Monotonic() but silently ignore the error and return 0 if -// the internal clock fails. -// -// Use _PyTime_MonotonicWithInfo() or the public PyTime_Monotonic() -// to check for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_MonotonicUnchecked(void); - // Get the time of a monotonic clock, i.e. a clock that cannot go backwards. // The clock is not affected by system clock updates. The reference point of // the returned value is undefined, so that only the difference between the @@ -296,14 +280,6 @@ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -// Similar to PyTime_PerfCounter() but silently ignore the error and return 0 -// if the internal clock fails. -// -// Use _PyTime_PerfCounterWithInfo() or the public PyTime_PerfCounter() to -// check for failure. -// Export for '_lsprof' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_PerfCounterUnchecked(void); - // Get the performance counter: clock with the highest available resolution to // measure a short duration. @@ -319,12 +295,12 @@ extern int _PyTime_PerfCounterWithInfo( // --- _PyDeadline ----------------------------------------------------------- // Create a deadline. -// Pseudo code: _PyTime_MonotonicUnchecked() + timeout. +// Pseudo code: return PyTime_MonotonicRaw() + timeout // Export for '_ssl' shared extension. PyAPI_FUNC(PyTime_t) _PyDeadline_Init(PyTime_t timeout); // Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_MonotonicUnchecked(). +// Pseudo code: return deadline - PyTime_MonotonicRaw() // Export for '_ssl' shared extension. PyAPI_FUNC(PyTime_t) _PyDeadline_Get(PyTime_t deadline); diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index c9fa97d6820757..80a2daf4efc16a 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -13,10 +13,12 @@ extern PyObject *_Py_make_paramspec(PyThreadState *, PyObject *); extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); +extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); extern PyTypeObject _PyTypeAlias_Type; +extern PyObject _Py_NoDefaultStruct; #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a180054d407b39..cc2fc15ac5cabf 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -180,6 +180,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__file__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__firstlineno__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__float__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 030321ef4fcb23..1e6ef8e54a221a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -51,32 +51,35 @@ extern "C" { #define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 #define _CALL_METHOD_DESCRIPTOR_NOARGS 318 #define _CALL_METHOD_DESCRIPTOR_O 319 -#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_STR_1 320 -#define _CALL_TUPLE_1 321 +#define _CALL_NON_PY_GENERAL 320 +#define _CALL_STR_1 321 +#define _CALL_TUPLE_1 322 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 322 -#define _CHECK_ATTR_METHOD_LAZY_DICT 323 -#define _CHECK_ATTR_MODULE 324 -#define _CHECK_ATTR_WITH_HINT 325 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 326 +#define _CHECK_ATTR_CLASS 323 +#define _CHECK_ATTR_METHOD_LAZY_DICT 324 +#define _CHECK_ATTR_MODULE 325 +#define _CHECK_ATTR_WITH_HINT 326 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 327 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 327 -#define _CHECK_FUNCTION_EXACT_ARGS 328 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 329 -#define _CHECK_PEP_523 330 -#define _CHECK_PERIODIC 331 -#define _CHECK_STACK_SPACE 332 -#define _CHECK_STACK_SPACE_OPERAND 333 -#define _CHECK_VALIDITY 334 -#define _CHECK_VALIDITY_AND_SET_IP 335 -#define _COLD_EXIT 336 -#define _COMPARE_OP 337 -#define _COMPARE_OP_FLOAT 338 -#define _COMPARE_OP_INT 339 -#define _COMPARE_OP_STR 340 -#define _CONTAINS_OP 341 +#define _CHECK_FUNCTION 328 +#define _CHECK_FUNCTION_EXACT_ARGS 329 +#define _CHECK_FUNCTION_VERSION 330 +#define _CHECK_IS_NOT_PY_CALLABLE 331 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 332 +#define _CHECK_METHOD_VERSION 333 +#define _CHECK_PEP_523 334 +#define _CHECK_PERIODIC 335 +#define _CHECK_STACK_SPACE 336 +#define _CHECK_STACK_SPACE_OPERAND 337 +#define _CHECK_VALIDITY 338 +#define _CHECK_VALIDITY_AND_SET_IP 339 +#define _COLD_EXIT 340 +#define _COMPARE_OP 341 +#define _COMPARE_OP_FLOAT 342 +#define _COMPARE_OP_INT 343 +#define _COMPARE_OP_STR 344 +#define _CONTAINS_OP 345 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -88,53 +91,53 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 342 +#define _DEOPT 346 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DYNAMIC_EXIT 343 +#define _DYNAMIC_EXIT 347 #define _END_SEND END_SEND -#define _ERROR_POP_N 344 -#define _EVAL_BREAKER_EXIT 345 +#define _ERROR_POP_N 348 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 346 +#define _EXPAND_METHOD 349 +#define _FATAL_ERROR 350 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 347 -#define _FOR_ITER_GEN_FRAME 348 -#define _FOR_ITER_TIER_TWO 349 +#define _FOR_ITER 351 +#define _FOR_ITER_GEN_FRAME 352 +#define _FOR_ITER_TIER_TWO 353 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 350 -#define _GUARD_BOTH_INT 351 -#define _GUARD_BOTH_UNICODE 352 -#define _GUARD_BUILTINS_VERSION 353 -#define _GUARD_DORV_NO_DICT 354 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 355 -#define _GUARD_GLOBALS_VERSION 356 -#define _GUARD_IS_FALSE_POP 357 -#define _GUARD_IS_NONE_POP 358 -#define _GUARD_IS_NOT_NONE_POP 359 -#define _GUARD_IS_TRUE_POP 360 -#define _GUARD_KEYS_VERSION 361 -#define _GUARD_NOS_FLOAT 362 -#define _GUARD_NOS_INT 363 -#define _GUARD_NOT_EXHAUSTED_LIST 364 -#define _GUARD_NOT_EXHAUSTED_RANGE 365 -#define _GUARD_NOT_EXHAUSTED_TUPLE 366 -#define _GUARD_TOS_FLOAT 367 -#define _GUARD_TOS_INT 368 -#define _GUARD_TYPE_VERSION 369 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 370 -#define _INIT_CALL_PY_EXACT_ARGS 371 -#define _INIT_CALL_PY_EXACT_ARGS_0 372 -#define _INIT_CALL_PY_EXACT_ARGS_1 373 -#define _INIT_CALL_PY_EXACT_ARGS_2 374 -#define _INIT_CALL_PY_EXACT_ARGS_3 375 -#define _INIT_CALL_PY_EXACT_ARGS_4 376 +#define _GUARD_BOTH_FLOAT 354 +#define _GUARD_BOTH_INT 355 +#define _GUARD_BOTH_UNICODE 356 +#define _GUARD_BUILTINS_VERSION 357 +#define _GUARD_DORV_NO_DICT 358 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 359 +#define _GUARD_GLOBALS_VERSION 360 +#define _GUARD_IS_FALSE_POP 361 +#define _GUARD_IS_NONE_POP 362 +#define _GUARD_IS_NOT_NONE_POP 363 +#define _GUARD_IS_TRUE_POP 364 +#define _GUARD_KEYS_VERSION 365 +#define _GUARD_NOS_FLOAT 366 +#define _GUARD_NOS_INT 367 +#define _GUARD_NOT_EXHAUSTED_LIST 368 +#define _GUARD_NOT_EXHAUSTED_RANGE 369 +#define _GUARD_NOT_EXHAUSTED_TUPLE 370 +#define _GUARD_TOS_FLOAT 371 +#define _GUARD_TOS_INT 372 +#define _GUARD_TYPE_VERSION 373 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 374 +#define _INIT_CALL_PY_EXACT_ARGS 375 +#define _INIT_CALL_PY_EXACT_ARGS_0 376 +#define _INIT_CALL_PY_EXACT_ARGS_1 377 +#define _INIT_CALL_PY_EXACT_ARGS_2 378 +#define _INIT_CALL_PY_EXACT_ARGS_3 379 +#define _INIT_CALL_PY_EXACT_ARGS_4 380 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -151,65 +154,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 377 -#define _IS_NONE 378 +#define _INTERNAL_INCREMENT_OPT_COUNTER 381 +#define _IS_NONE 382 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 379 -#define _ITER_CHECK_RANGE 380 -#define _ITER_CHECK_TUPLE 381 -#define _ITER_JUMP_LIST 382 -#define _ITER_JUMP_RANGE 383 -#define _ITER_JUMP_TUPLE 384 -#define _ITER_NEXT_LIST 385 -#define _ITER_NEXT_RANGE 386 -#define _ITER_NEXT_TUPLE 387 -#define _JUMP_TO_TOP 388 +#define _ITER_CHECK_LIST 383 +#define _ITER_CHECK_RANGE 384 +#define _ITER_CHECK_TUPLE 385 +#define _ITER_JUMP_LIST 386 +#define _ITER_JUMP_RANGE 387 +#define _ITER_JUMP_TUPLE 388 +#define _ITER_NEXT_LIST 389 +#define _ITER_NEXT_RANGE 390 +#define _ITER_NEXT_TUPLE 391 +#define _JUMP_TO_TOP 392 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 389 -#define _LOAD_ATTR_CLASS 390 -#define _LOAD_ATTR_CLASS_0 391 -#define _LOAD_ATTR_CLASS_1 392 +#define _LOAD_ATTR 393 +#define _LOAD_ATTR_CLASS 394 +#define _LOAD_ATTR_CLASS_0 395 +#define _LOAD_ATTR_CLASS_1 396 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 393 -#define _LOAD_ATTR_INSTANCE_VALUE_0 394 -#define _LOAD_ATTR_INSTANCE_VALUE_1 395 -#define _LOAD_ATTR_METHOD_LAZY_DICT 396 -#define _LOAD_ATTR_METHOD_NO_DICT 397 -#define _LOAD_ATTR_METHOD_WITH_VALUES 398 -#define _LOAD_ATTR_MODULE 399 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 400 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 401 +#define _LOAD_ATTR_INSTANCE_VALUE 397 +#define _LOAD_ATTR_INSTANCE_VALUE_0 398 +#define _LOAD_ATTR_INSTANCE_VALUE_1 399 +#define _LOAD_ATTR_METHOD_LAZY_DICT 400 +#define _LOAD_ATTR_METHOD_NO_DICT 401 +#define _LOAD_ATTR_METHOD_WITH_VALUES 402 +#define _LOAD_ATTR_MODULE 403 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 404 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 405 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 402 -#define _LOAD_ATTR_SLOT_0 403 -#define _LOAD_ATTR_SLOT_1 404 -#define _LOAD_ATTR_WITH_HINT 405 +#define _LOAD_ATTR_SLOT 406 +#define _LOAD_ATTR_SLOT_0 407 +#define _LOAD_ATTR_SLOT_1 408 +#define _LOAD_ATTR_WITH_HINT 409 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 406 -#define _LOAD_CONST_INLINE_BORROW 407 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 408 -#define _LOAD_CONST_INLINE_WITH_NULL 409 +#define _LOAD_CONST_INLINE 410 +#define _LOAD_CONST_INLINE_BORROW 411 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 412 +#define _LOAD_CONST_INLINE_WITH_NULL 413 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 410 -#define _LOAD_FAST_0 411 -#define _LOAD_FAST_1 412 -#define _LOAD_FAST_2 413 -#define _LOAD_FAST_3 414 -#define _LOAD_FAST_4 415 -#define _LOAD_FAST_5 416 -#define _LOAD_FAST_6 417 -#define _LOAD_FAST_7 418 +#define _LOAD_FAST 414 +#define _LOAD_FAST_0 415 +#define _LOAD_FAST_1 416 +#define _LOAD_FAST_2 417 +#define _LOAD_FAST_3 418 +#define _LOAD_FAST_4 419 +#define _LOAD_FAST_5 420 +#define _LOAD_FAST_6 421 +#define _LOAD_FAST_7 422 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 419 -#define _LOAD_GLOBAL_BUILTINS 420 -#define _LOAD_GLOBAL_MODULE 421 +#define _LOAD_GLOBAL 423 +#define _LOAD_GLOBAL_BUILTINS 424 +#define _LOAD_GLOBAL_MODULE 425 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -223,51 +226,51 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 422 -#define _POP_JUMP_IF_FALSE 423 -#define _POP_JUMP_IF_TRUE 424 +#define _POP_FRAME 426 +#define _POP_JUMP_IF_FALSE 427 +#define _POP_JUMP_IF_TRUE 428 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 425 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 429 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 426 +#define _PUSH_FRAME 430 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 427 +#define _PY_FRAME_GENERAL 431 +#define _REPLACE_WITH_TRUE 432 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR -#define _SAVE_RETURN_OFFSET 428 -#define _SEND 429 +#define _SAVE_RETURN_OFFSET 433 +#define _SEND 434 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 430 -#define _START_EXECUTOR 431 -#define _STORE_ATTR 432 -#define _STORE_ATTR_INSTANCE_VALUE 433 -#define _STORE_ATTR_SLOT 434 +#define _START_EXECUTOR 435 +#define _STORE_ATTR 436 +#define _STORE_ATTR_INSTANCE_VALUE 437 +#define _STORE_ATTR_SLOT 438 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 435 -#define _STORE_FAST_0 436 -#define _STORE_FAST_1 437 -#define _STORE_FAST_2 438 -#define _STORE_FAST_3 439 -#define _STORE_FAST_4 440 -#define _STORE_FAST_5 441 -#define _STORE_FAST_6 442 -#define _STORE_FAST_7 443 +#define _STORE_FAST 439 +#define _STORE_FAST_0 440 +#define _STORE_FAST_1 441 +#define _STORE_FAST_2 442 +#define _STORE_FAST_3 443 +#define _STORE_FAST_4 444 +#define _STORE_FAST_5 445 +#define _STORE_FAST_6 446 +#define _STORE_FAST_7 447 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 444 +#define _STORE_SUBSCR 448 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 445 -#define _TO_BOOL 446 +#define _TIER2_RESUME_CHECK 449 +#define _TO_BOOL 450 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -277,12 +280,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 447 +#define _UNPACK_SEQUENCE 451 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 447 +#define _YIELD_VALUE YIELD_VALUE +#define MAX_UOP_ID 451 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 59e690f3aace35..470e95e2b3b041 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -91,6 +91,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_ASSERTION_ERROR] = 0, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -192,7 +193,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_NON_PY_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -232,11 +239,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, - [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, + [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_EXIT_FLAG, + [_EXIT_TRACE] = 0, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, @@ -248,13 +255,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, - [_FATAL_ERROR] = HAS_ESCAPES_FLAG, + [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, - [_SIDE_EXIT] = 0, [_ERROR_POP_N] = HAS_ARG_FLAG, - [_TIER2_RESUME_CHECK] = HAS_EXIT_FLAG, - [_EVAL_BREAKER_EXIT] = HAS_ESCAPES_FLAG, + [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -296,6 +301,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", @@ -308,7 +314,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", @@ -338,9 +347,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", - [_EVAL_BREAKER_EXIT] = "_EVAL_BREAKER_EXIT", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXPAND_METHOD] = "_EXPAND_METHOD", [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", @@ -451,6 +460,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", + [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", @@ -460,7 +470,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", - [_SIDE_EXIT] = "_SIDE_EXIT", [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", @@ -500,6 +509,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_YIELD_VALUE] = "_YIELD_VALUE", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -648,6 +658,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GET_AWAITABLE: return 1; + case _YIELD_VALUE: + return 1; case _POP_EXCEPT: return 1; case _LOAD_ASSERTION_ERROR: @@ -850,6 +862,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _CHECK_PERIODIC: return 0; + case _PY_FRAME_GENERAL: + return 2 + oparg; + case _CHECK_FUNCTION_VERSION: + return 2 + oparg; + case _CHECK_METHOD_VERSION: + return 2 + oparg; + case _EXPAND_METHOD: + return 2 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE: + return 2 + oparg; + case _CALL_NON_PY_GENERAL: + return 2 + oparg; case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: @@ -968,14 +992,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _DEOPT: return 0; - case _SIDE_EXIT: - return 0; case _ERROR_POP_N: return oparg; case _TIER2_RESUME_CHECK: return 0; - case _EVAL_BREAKER_EXIT: - return 0; default: return -1; } diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 83f8c2030dbb8f..2a17c891dda811 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -76,9 +76,13 @@ struct PyModuleDef_Slot { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 # define Py_mod_multiple_interpreters 3 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_mod_gil 4 +#endif + #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 3 +#define _Py_mod_LAST_SLOT 4 #endif #endif /* New in 3.5 */ @@ -90,6 +94,16 @@ struct PyModuleDef_Slot { # define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) #endif +/* for Py_mod_gil: */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_MOD_GIL_USED ((void *)0) +# define Py_MOD_GIL_NOT_USED ((void *)1) +#endif + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); +#endif + struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; diff --git a/Include/monitoring.h b/Include/monitoring.h new file mode 100644 index 00000000000000..985f7f230e44e3 --- /dev/null +++ b/Include/monitoring.h @@ -0,0 +1,18 @@ +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifdef __cplusplus +extern "C" { +#endif + +// There is currently no limited API for monitoring + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_MONITORING_H +# include "cpython/monitoring.h" +# undef Py_CPYTHON_MONITORING_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_MONITORING_H */ diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 185205c6870edc..647f7c0ecb1ec8 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -144,63 +144,65 @@ extern "C" { #define BINARY_SUBSCR_TUPLE_INT 161 #define CALL_ALLOC_AND_ENTER_INIT 162 #define CALL_BOUND_METHOD_EXACT_ARGS 163 -#define CALL_BUILTIN_CLASS 164 -#define CALL_BUILTIN_FAST 165 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 166 -#define CALL_BUILTIN_O 167 -#define CALL_ISINSTANCE 168 -#define CALL_LEN 169 -#define CALL_LIST_APPEND 170 -#define CALL_METHOD_DESCRIPTOR_FAST 171 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 172 -#define CALL_METHOD_DESCRIPTOR_NOARGS 173 -#define CALL_METHOD_DESCRIPTOR_O 174 -#define CALL_PY_EXACT_ARGS 175 -#define CALL_PY_WITH_DEFAULTS 176 -#define CALL_STR_1 177 -#define CALL_TUPLE_1 178 -#define CALL_TYPE_1 179 -#define COMPARE_OP_FLOAT 180 -#define COMPARE_OP_INT 181 -#define COMPARE_OP_STR 182 -#define CONTAINS_OP_DICT 183 -#define CONTAINS_OP_SET 184 -#define FOR_ITER_GEN 185 -#define FOR_ITER_LIST 186 -#define FOR_ITER_RANGE 187 -#define FOR_ITER_TUPLE 188 -#define LOAD_ATTR_CLASS 189 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 190 -#define LOAD_ATTR_INSTANCE_VALUE 191 -#define LOAD_ATTR_METHOD_LAZY_DICT 192 -#define LOAD_ATTR_METHOD_NO_DICT 193 -#define LOAD_ATTR_METHOD_WITH_VALUES 194 -#define LOAD_ATTR_MODULE 195 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 196 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 197 -#define LOAD_ATTR_PROPERTY 198 -#define LOAD_ATTR_SLOT 199 -#define LOAD_ATTR_WITH_HINT 200 -#define LOAD_GLOBAL_BUILTIN 201 -#define LOAD_GLOBAL_MODULE 202 -#define LOAD_SUPER_ATTR_ATTR 203 -#define LOAD_SUPER_ATTR_METHOD 204 -#define RESUME_CHECK 205 -#define SEND_GEN 206 -#define STORE_ATTR_INSTANCE_VALUE 207 -#define STORE_ATTR_SLOT 208 -#define STORE_ATTR_WITH_HINT 209 -#define STORE_SUBSCR_DICT 210 -#define STORE_SUBSCR_LIST_INT 211 -#define TO_BOOL_ALWAYS_TRUE 212 -#define TO_BOOL_BOOL 213 -#define TO_BOOL_INT 214 -#define TO_BOOL_LIST 215 -#define TO_BOOL_NONE 216 -#define TO_BOOL_STR 217 -#define UNPACK_SEQUENCE_LIST 218 -#define UNPACK_SEQUENCE_TUPLE 219 -#define UNPACK_SEQUENCE_TWO_TUPLE 220 +#define CALL_BOUND_METHOD_GENERAL 164 +#define CALL_BUILTIN_CLASS 165 +#define CALL_BUILTIN_FAST 166 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 +#define CALL_BUILTIN_O 168 +#define CALL_ISINSTANCE 169 +#define CALL_LEN 170 +#define CALL_LIST_APPEND 171 +#define CALL_METHOD_DESCRIPTOR_FAST 172 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 173 +#define CALL_METHOD_DESCRIPTOR_NOARGS 174 +#define CALL_METHOD_DESCRIPTOR_O 175 +#define CALL_NON_PY_GENERAL 176 +#define CALL_PY_EXACT_ARGS 177 +#define CALL_PY_GENERAL 178 +#define CALL_STR_1 179 +#define CALL_TUPLE_1 180 +#define CALL_TYPE_1 181 +#define COMPARE_OP_FLOAT 182 +#define COMPARE_OP_INT 183 +#define COMPARE_OP_STR 184 +#define CONTAINS_OP_DICT 185 +#define CONTAINS_OP_SET 186 +#define FOR_ITER_GEN 187 +#define FOR_ITER_LIST 188 +#define FOR_ITER_RANGE 189 +#define FOR_ITER_TUPLE 190 +#define LOAD_ATTR_CLASS 191 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 192 +#define LOAD_ATTR_INSTANCE_VALUE 193 +#define LOAD_ATTR_METHOD_LAZY_DICT 194 +#define LOAD_ATTR_METHOD_NO_DICT 195 +#define LOAD_ATTR_METHOD_WITH_VALUES 196 +#define LOAD_ATTR_MODULE 197 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 198 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 199 +#define LOAD_ATTR_PROPERTY 200 +#define LOAD_ATTR_SLOT 201 +#define LOAD_ATTR_WITH_HINT 202 +#define LOAD_GLOBAL_BUILTIN 203 +#define LOAD_GLOBAL_MODULE 204 +#define LOAD_SUPER_ATTR_ATTR 205 +#define LOAD_SUPER_ATTR_METHOD 206 +#define RESUME_CHECK 207 +#define SEND_GEN 208 +#define STORE_ATTR_INSTANCE_VALUE 209 +#define STORE_ATTR_SLOT 210 +#define STORE_ATTR_WITH_HINT 211 +#define STORE_SUBSCR_DICT 212 +#define STORE_SUBSCR_LIST_INT 213 +#define TO_BOOL_ALWAYS_TRUE 214 +#define TO_BOOL_BOOL 215 +#define TO_BOOL_INT 216 +#define TO_BOOL_LIST 217 +#define TO_BOOL_NONE 218 +#define TO_BOOL_STR 219 +#define UNPACK_SEQUENCE_LIST 220 +#define UNPACK_SEQUENCE_TUPLE 221 +#define UNPACK_SEQUENCE_TWO_TUPLE 222 #define INSTRUMENTED_RESUME 236 #define INSTRUMENTED_END_FOR 237 #define INSTRUMENTED_END_SEND 238 diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 35c595deaa72c2..d63af11dbd220a 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 13 +#define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 6 +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.13.0a6+" +#define PY_VERSION "3.14.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/py_curses.h b/Include/py_curses.h index e46b08e9cc414e..a51d9980eee401 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -23,10 +23,16 @@ # endif #endif -#if !defined(HAVE_CURSES_IS_PAD) && defined(WINDOW_HAS_FLAGS) -/* The following definition is necessary for ncurses 5.7; without it, - some of [n]curses.h set NCURSES_OPAQUE to 1, and then Python - can't get at the WINDOW flags field. */ +#if defined(WINDOW_HAS_FLAGS) && defined(__APPLE__) +/* gh-109617, gh-115383: we can rely on the default value for NCURSES_OPAQUE on + most platforms, but not on macOS. This is because, starting with Xcode 15, + Apple-provided ncurses.h comes from ncurses 6 (which defaults to opaque + structs) but can still be linked to older versions of ncurses dynamic + libraries which don't provide functions such as is_pad() to deal with opaque + structs. Setting NCURSES_OPAQUE to 0 is harmless in all ncurses releases to + this date (provided that a thread-safe implementation is not required), but + this might change in the future. This fix might become irrelevant once + support for macOS 13 or earlier is dropped. */ #define NCURSES_OPAQUE 0 #endif @@ -39,7 +45,10 @@ #ifdef HAVE_NCURSES_H /* configure was checking , but we will use , which has some or all these features. */ -#if !defined(WINDOW_HAS_FLAGS) && !(NCURSES_OPAQUE+0) +#if !defined(WINDOW_HAS_FLAGS) && \ + (NCURSES_VERSION_PATCH+0 < 20070303 || !(NCURSES_OPAQUE+0)) +/* the WINDOW flags field was always accessible in ncurses prior to 20070303; + after that, it depends on the value of NCURSES_OPAQUE. */ #define WINDOW_HAS_FLAGS 1 #endif #if !defined(HAVE_CURSES_IS_PAD) && NCURSES_VERSION_PATCH+0 >= 20090906 diff --git a/Include/pymacro.h b/Include/pymacro.h index cd6fc4eba9c2ed..b388c2a4a663ce 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -46,24 +46,41 @@ /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) -/* Assert a build-time dependency, as an expression. - - Your compile will fail if the condition isn't true, or can't be evaluated - by the compiler. This can be used in an expression: its value is 0. - - Example: - - #define foo_to_char(foo) \ - ((char *)(foo) \ - + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) - - Written by Rusty Russell, public domain, http://ccodearchive.net/ */ -#define Py_BUILD_ASSERT_EXPR(cond) \ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define Py_BUILD_ASSERT_EXPR(cond) \ + ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \ + 0) +#else + /* Assert a build-time dependency, as an expression. + * + * Your compile will fail if the condition isn't true, or can't be evaluated + * by the compiler. This can be used in an expression: its value is 0. + * + * Example: + * + * #define foo_to_char(foo) \ + * ((char *)(foo) \ + * + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + * + * Written by Rusty Russell, public domain, http://ccodearchive.net/ + */ +# define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#endif -#define Py_BUILD_ASSERT(cond) do { \ - (void)Py_BUILD_ASSERT_EXPR(cond); \ - } while(0) +#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L)) + // Use static_assert() on C11 and newer +# define Py_BUILD_ASSERT(cond) \ + do { \ + static_assert((cond), #cond); \ + } while (0) +#else +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) +#endif /* Get the number of elements in a visible array diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 00000000000000..590e85ea8c2db1 --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,94 @@ +import io +import sys + + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +# longer than this will be be truncated by logcat. This limit has already been +# reduced at least once in the history of Android (from 4076 to 4068 between API +# level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then TextIOWrapper may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + sys.stdout = TextLogStream( + android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors) + sys.stderr = TextLogStream( + android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, android_log_write, prio, tag, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs) + self._CHUNK_SIZE = MAX_BYTES_PER_WRITE + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string before sending it to the superclass. Note that + # "".splitlines() == [], so nothing will be logged for an empty string. + for line in s.splitlines(keepends=True): + while line: + super().write(line[:MAX_CHARS_PER_WRITE]) + line = line[MAX_CHARS_PER_WRITE:] + + return len(s) + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, android_log_write, prio, tag): + self.android_log_write = android_log_write + self.prio = prio + self.tag = tag + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.android_log_write(self.prio, self.tag, + b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/_colorize.py b/Lib/_colorize.py new file mode 100644 index 00000000000000..845fb57a90abb8 --- /dev/null +++ b/Lib/_colorize.py @@ -0,0 +1,64 @@ +import io +import os +import sys + +COLORIZE = True + + +class ANSIColors: + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + GREEN = "\x1b[32m" + GREY = "\x1b[90m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + RESET = "\x1b[0m" + YELLOW = "\x1b[33m" + + +NoColors = ANSIColors() + +for attr in dir(NoColors): + if not attr.startswith("__"): + setattr(NoColors, attr, "") + + +def get_colors(colorize: bool = False) -> ANSIColors: + if colorize or can_colorize(): + return ANSIColors() + else: + return NoColors + + +def can_colorize() -> bool: + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + if not sys.flags.ignore_environment: + if os.environ.get("PYTHON_COLORS") == "0": + return False + if os.environ.get("PYTHON_COLORS") == "1": + return True + if "NO_COLOR" in os.environ: + return False + if not COLORIZE: + return False + if not sys.flags.ignore_environment: + if "FORCE_COLOR" in os.environ: + return True + if os.environ.get("TERM") == "dumb": + return False + + if not hasattr(sys.stderr, "fileno"): + return False + + try: + return os.isatty(sys.stderr.fileno()) + except io.UnsupportedOperation: + return sys.stderr.isatty() diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index b5bafe6302bc9e..b3d7b8103e86c4 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -88,7 +88,6 @@ "CALL": [ "CALL_BOUND_METHOD_EXACT_ARGS", "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", "CALL_TYPE_1", "CALL_STR_1", "CALL_TUPLE_1", @@ -104,6 +103,9 @@ "CALL_METHOD_DESCRIPTOR_NOARGS", "CALL_METHOD_DESCRIPTOR_FAST", "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", ], } @@ -123,63 +125,65 @@ 'BINARY_SUBSCR_TUPLE_INT': 161, 'CALL_ALLOC_AND_ENTER_INIT': 162, 'CALL_BOUND_METHOD_EXACT_ARGS': 163, - 'CALL_BUILTIN_CLASS': 164, - 'CALL_BUILTIN_FAST': 165, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 166, - 'CALL_BUILTIN_O': 167, - 'CALL_ISINSTANCE': 168, - 'CALL_LEN': 169, - 'CALL_LIST_APPEND': 170, - 'CALL_METHOD_DESCRIPTOR_FAST': 171, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 172, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 173, - 'CALL_METHOD_DESCRIPTOR_O': 174, - 'CALL_PY_EXACT_ARGS': 175, - 'CALL_PY_WITH_DEFAULTS': 176, - 'CALL_STR_1': 177, - 'CALL_TUPLE_1': 178, - 'CALL_TYPE_1': 179, - 'COMPARE_OP_FLOAT': 180, - 'COMPARE_OP_INT': 181, - 'COMPARE_OP_STR': 182, - 'CONTAINS_OP_DICT': 183, - 'CONTAINS_OP_SET': 184, - 'FOR_ITER_GEN': 185, - 'FOR_ITER_LIST': 186, - 'FOR_ITER_RANGE': 187, - 'FOR_ITER_TUPLE': 188, - 'LOAD_ATTR_CLASS': 189, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 190, - 'LOAD_ATTR_INSTANCE_VALUE': 191, - 'LOAD_ATTR_METHOD_LAZY_DICT': 192, - 'LOAD_ATTR_METHOD_NO_DICT': 193, - 'LOAD_ATTR_METHOD_WITH_VALUES': 194, - 'LOAD_ATTR_MODULE': 195, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 196, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 197, - 'LOAD_ATTR_PROPERTY': 198, - 'LOAD_ATTR_SLOT': 199, - 'LOAD_ATTR_WITH_HINT': 200, - 'LOAD_GLOBAL_BUILTIN': 201, - 'LOAD_GLOBAL_MODULE': 202, - 'LOAD_SUPER_ATTR_ATTR': 203, - 'LOAD_SUPER_ATTR_METHOD': 204, - 'RESUME_CHECK': 205, - 'SEND_GEN': 206, - 'STORE_ATTR_INSTANCE_VALUE': 207, - 'STORE_ATTR_SLOT': 208, - 'STORE_ATTR_WITH_HINT': 209, - 'STORE_SUBSCR_DICT': 210, - 'STORE_SUBSCR_LIST_INT': 211, - 'TO_BOOL_ALWAYS_TRUE': 212, - 'TO_BOOL_BOOL': 213, - 'TO_BOOL_INT': 214, - 'TO_BOOL_LIST': 215, - 'TO_BOOL_NONE': 216, - 'TO_BOOL_STR': 217, - 'UNPACK_SEQUENCE_LIST': 218, - 'UNPACK_SEQUENCE_TUPLE': 219, - 'UNPACK_SEQUENCE_TWO_TUPLE': 220, + 'CALL_BOUND_METHOD_GENERAL': 164, + 'CALL_BUILTIN_CLASS': 165, + 'CALL_BUILTIN_FAST': 166, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, + 'CALL_BUILTIN_O': 168, + 'CALL_ISINSTANCE': 169, + 'CALL_LEN': 170, + 'CALL_LIST_APPEND': 171, + 'CALL_METHOD_DESCRIPTOR_FAST': 172, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, + 'CALL_METHOD_DESCRIPTOR_O': 175, + 'CALL_NON_PY_GENERAL': 176, + 'CALL_PY_EXACT_ARGS': 177, + 'CALL_PY_GENERAL': 178, + 'CALL_STR_1': 179, + 'CALL_TUPLE_1': 180, + 'CALL_TYPE_1': 181, + 'COMPARE_OP_FLOAT': 182, + 'COMPARE_OP_INT': 183, + 'COMPARE_OP_STR': 184, + 'CONTAINS_OP_DICT': 185, + 'CONTAINS_OP_SET': 186, + 'FOR_ITER_GEN': 187, + 'FOR_ITER_LIST': 188, + 'FOR_ITER_RANGE': 189, + 'FOR_ITER_TUPLE': 190, + 'LOAD_ATTR_CLASS': 191, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, + 'LOAD_ATTR_INSTANCE_VALUE': 193, + 'LOAD_ATTR_METHOD_LAZY_DICT': 194, + 'LOAD_ATTR_METHOD_NO_DICT': 195, + 'LOAD_ATTR_METHOD_WITH_VALUES': 196, + 'LOAD_ATTR_MODULE': 197, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, + 'LOAD_ATTR_PROPERTY': 200, + 'LOAD_ATTR_SLOT': 201, + 'LOAD_ATTR_WITH_HINT': 202, + 'LOAD_GLOBAL_BUILTIN': 203, + 'LOAD_GLOBAL_MODULE': 204, + 'LOAD_SUPER_ATTR_ATTR': 205, + 'LOAD_SUPER_ATTR_METHOD': 206, + 'RESUME_CHECK': 207, + 'SEND_GEN': 208, + 'STORE_ATTR_INSTANCE_VALUE': 209, + 'STORE_ATTR_SLOT': 210, + 'STORE_ATTR_WITH_HINT': 211, + 'STORE_SUBSCR_DICT': 212, + 'STORE_SUBSCR_LIST_INT': 213, + 'TO_BOOL_ALWAYS_TRUE': 214, + 'TO_BOOL_BOOL': 215, + 'TO_BOOL_INT': 216, + 'TO_BOOL_LIST': 217, + 'TO_BOOL_NONE': 218, + 'TO_BOOL_STR': 219, + 'UNPACK_SEQUENCE_LIST': 220, + 'UNPACK_SEQUENCE_TUPLE': 221, + 'UNPACK_SEQUENCE_TWO_TUPLE': 222, } opmap = { diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index de4561a5ee050b..613123ec7b4329 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -2131,10 +2131,16 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2184,13 +2190,18 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 936346e187ff69..4970eb3fa67b2b 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -14,7 +14,91 @@ import re import decimal - +try: + import _decimal +except ImportError: + _decimal = None + +# A number of functions have this form, where `w` is a desired number of +# digits in base `base`: +# +# def inner(...w...): +# if w <= LIMIT: +# return something +# lo = w >> 1 +# hi = w - lo +# something involving base**lo, inner(...lo...), j, and inner(...hi...) +# figure out largest w needed +# result = inner(w) +# +# They all had some on-the-fly scheme to cache `base**lo` results for reuse. +# Power is costly. +# +# This routine aims to compute all amd only the needed powers in advance, as +# efficiently as reasonably possible. This isn't trivial, and all the +# on-the-fly methods did needless work in many cases. The driving code above +# changes to: +# +# figure out largest w needed +# mycache = compute_powers(w, base, LIMIT) +# result = inner(w) +# +# and `mycache[lo]` replaces `base**lo` in the inner function. +# +# While this does give minor speedups (a few percent at best), the primary +# intent is to simplify the functions using this, by eliminating the need for +# them to craft their own ad-hoc caching schemes. +def compute_powers(w, base, more_than, show=False): + seen = set() + need = set() + ws = {w} + while ws: + w = ws.pop() # any element is fine to use next + if w in seen or w <= more_than: + continue + seen.add(w) + lo = w >> 1 + # only _need_ lo here; some other path may, or may not, need hi + need.add(lo) + ws.add(lo) + if w & 1: + ws.add(lo + 1) + + d = {} + if not need: + return d + it = iter(sorted(need)) + first = next(it) + if show: + print("pow at", first) + d[first] = base ** first + for this in it: + if this - 1 in d: + if show: + print("* base at", this) + d[this] = d[this - 1] * base # cheap + else: + lo = this >> 1 + hi = this - lo + assert lo in d + if show: + print("square at", this) + # Multiplying a bigint by itself (same object!) is about twice + # as fast in CPython. + sq = d[lo] * d[lo] + if hi != lo: + assert hi == lo + 1 + if show: + print(" and * base") + sq *= base + d[this] = sq + return d + +_unbounded_dec_context = decimal.getcontext().copy() +_unbounded_dec_context.prec = decimal.MAX_PREC +_unbounded_dec_context.Emax = decimal.MAX_EMAX +_unbounded_dec_context.Emin = decimal.MIN_EMIN +_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check def int_to_decimal(n): """Asymptotically fast conversion of an 'int' to Decimal.""" @@ -29,61 +113,76 @@ def int_to_decimal(n): # "clever" recursive way. If we want a string representation, we # apply str to _that_. - D = decimal.Decimal - D2 = D(2) - - BITLIM = 128 - - mem = {} - - def w2pow(w): - """Return D(2)**w and store the result. Also possibly save some - intermediate results. In context, these are likely to be reused - across various levels of the conversion to Decimal.""" - if (result := mem.get(w)) is None: - if w <= BITLIM: - result = D2**w - elif w - 1 in mem: - result = (t := mem[w - 1]) + t - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w2pow(w2) * w2pow(w - w2) - mem[w] = result - return result + from decimal import Decimal as D + BITLIM = 200 + # Don't bother caching the "lo" mask in this; the time to compute it is + # tiny compared to the multiply. def inner(n, w): if w <= BITLIM: return D(n) w2 = w >> 1 hi = n >> w2 - lo = n - (hi << w2) - return inner(lo, w2) + inner(hi, w - w2) * w2pow(w2) - - with decimal.localcontext() as ctx: - ctx.prec = decimal.MAX_PREC - ctx.Emax = decimal.MAX_EMAX - ctx.Emin = decimal.MIN_EMIN - ctx.traps[decimal.Inexact] = 1 + lo = n & ((1 << w2) - 1) + return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2] + with decimal.localcontext(_unbounded_dec_context): + nbits = n.bit_length() + w2pow = compute_powers(nbits, D(2), BITLIM) if n < 0: negate = True n = -n else: negate = False - result = inner(n, n.bit_length()) + result = inner(n, nbits) if negate: result = -result return result - def int_to_decimal_string(n): """Asymptotically fast conversion of an 'int' to a decimal string.""" - return str(int_to_decimal(n)) - + w = n.bit_length() + if w > 450_000 and _decimal is not None: + # It is only usable with the C decimal implementation. + # _pydecimal.py calls str() on very large integers, which in its + # turn calls int_to_decimal_string(), causing very deep recursion. + return str(int_to_decimal(n)) + + # Fallback algorithm for the case when the C decimal module isn't + # available. This algorithm is asymptotically worse than the algorithm + # using the decimal module, but better than the quadratic time + # implementation in longobject.c. + + DIGLIM = 1000 + def inner(n, w): + if w <= DIGLIM: + return str(n) + w2 = w >> 1 + hi, lo = divmod(n, pow10[w2]) + return inner(hi, w - w2) + inner(lo, w2).zfill(w2) + + # The estimation of the number of decimal digits. + # There is no harm in small error. If we guess too large, there may + # be leading 0's that need to be stripped. If we guess too small, we + # may need to call str() recursively for the remaining highest digits, + # which can still potentially be a large integer. This is manifested + # only if the number has way more than 10**15 digits, that exceeds + # the 52-bit physical address limit in both Intel64 and AMD64. + w = int(w * 0.3010299956639812 + 1) # log10(2) + pow10 = compute_powers(w, 5, DIGLIM) + for k, v in pow10.items(): + pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k + if n < 0: + n = -n + sign = '-' + else: + sign = '' + s = inner(n, w) + if s[0] == '0' and n: + # If our guess of w is too large, there may be leading 0's that + # need to be stripped. + s = s.lstrip('0') + return sign + s def _str_to_int_inner(s): """Asymptotically fast conversion of a 'str' to an 'int'.""" @@ -100,35 +199,15 @@ def _str_to_int_inner(s): DIGLIM = 2048 - mem = {} - - def w5pow(w): - """Return 5**w and store the result. - Also possibly save some intermediate results. In context, these - are likely to be reused across various levels of the conversion - to 'int'. - """ - if (result := mem.get(w)) is None: - if w <= DIGLIM: - result = 5**w - elif w - 1 in mem: - result = mem[w - 1] * 5 - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w5pow(w2) * w5pow(w - w2) - mem[w] = result - return result - def inner(a, b): if b - a <= DIGLIM: return int(s[a:b]) mid = (a + b + 1) >> 1 - return inner(mid, b) + ((inner(a, mid) * w5pow(b - mid)) << (b - mid)) + return (inner(mid, b) + + ((inner(a, mid) * w5pow[b - mid]) + << (b - mid))) + w5pow = compute_powers(len(s), 5, DIGLIM) return inner(0, len(s)) @@ -142,7 +221,6 @@ def int_from_string(s): s = s.rstrip().replace('_', '') return _str_to_int_inner(s) - def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" # FIXME: this doesn't support the full syntax that int() supports. diff --git a/Lib/_pyrepl/__init__.py b/Lib/_pyrepl/__init__.py new file mode 100644 index 00000000000000..1693cbd0b98b74 --- /dev/null +++ b/Lib/_pyrepl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py new file mode 100644 index 00000000000000..c598019e7cd4ad --- /dev/null +++ b/Lib/_pyrepl/__main__.py @@ -0,0 +1,47 @@ +import os +import sys + +CAN_USE_PYREPL = sys.platform != "win32" + + +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + global CAN_USE_PYREPL + if not CAN_USE_PYREPL: + return sys._baserepl() + + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code) + + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + + run_interactive = None + try: + import errno + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) + from .simple_interact import run_multiline_interactive_console + run_interactive = run_multiline_interactive_console + except Exception as e: + from .trace import trace + msg = f"warning: can't use pyrepl: {e}" + trace(msg) + print(msg, file=sys.stderr) + CAN_USE_PYREPL = False + if run_interactive is None: + return sys._baserepl() + return run_interactive(mainmodule) + +if __name__ == "__main__": + interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 00000000000000..0757fb2c664add --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib(): + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if ctypes.cast(result, ctypes.c_void_p).value == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py new file mode 100644 index 00000000000000..456cba0769c952 --- /dev/null +++ b/Lib/_pyrepl/commands.py @@ -0,0 +1,477 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations +import os + +# Categories of actions: +# killing +# yanking +# motion +# editing +# history +# finishing +# [completion] + + +# types +if False: + from .reader import Reader + from .historical_reader import HistoricalReader + from .console import Event + + +class Command: + finish: bool = False + kills_digit_arg: bool = True + + def __init__( + self, reader: HistoricalReader, event_name: str, event: list[str] + ) -> None: + # Reader should really be "any reader" but there's too much usage of + # HistoricalReader methods and fields in the code below for us to + # refactor at the moment. + + self.reader = reader + self.event = event + self.event_name = event_name + + def do(self) -> None: + pass + + +class KillCommand(Command): + def kill_range(self, start: int, end: int) -> None: + if start == end: + return + r = self.reader + b = r.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = True + + +class YankCommand(Command): + pass + + +class MotionCommand(Command): + pass + + +class EditCommand(Command): + pass + + +class FinishCommand(Command): + finish = True + pass + + +def is_kill(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, KillCommand) + + +def is_yank(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, YankCommand) + + +# etc + + +class digit_arg(Command): + kills_digit_arg = False + + def do(self) -> None: + r = self.reader + c = self.event[-1] + if c == "-": + if r.arg is not None: + r.arg = -r.arg + else: + r.arg = -1 + else: + d = int(c) + if r.arg is None: + r.arg = d + else: + if r.arg < 0: + r.arg = 10 * r.arg - d + else: + r.arg = 10 * r.arg + d + r.dirty = True + + +class clear_screen(Command): + def do(self) -> None: + r = self.reader + r.console.clear() + r.dirty = True + + +class refresh(Command): + def do(self) -> None: + self.reader.dirty = True + + +class repaint(Command): + def do(self) -> None: + self.reader.dirty = True + self.reader.console.repaint() + + +class kill_line(KillCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + eol = r.eol() + for c in b[r.pos : eol]: + if not c.isspace(): + self.kill_range(r.pos, eol) + return + else: + self.kill_range(r.pos, eol + 1) + + +class unix_line_discard(KillCommand): + def do(self) -> None: + r = self.reader + self.kill_range(r.bol(), r.pos) + + +class unix_word_rubout(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.pos, r.eow()) + + +class backward_kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class yank(YankCommand): + def do(self) -> None: + r = self.reader + if not r.kill_ring: + r.error("nothing to yank") + return + r.insert(r.kill_ring[-1]) + + +class yank_pop(YankCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if not r.kill_ring: + r.error("nothing to yank") + return + if not is_yank(r.last_command): + r.error("previous command was not a yank") + return + repl = len(r.kill_ring[-1]) + r.kill_ring.insert(0, r.kill_ring.pop()) + t = r.kill_ring[-1] + b[r.pos - repl : r.pos] = t + r.pos = r.pos - repl + len(t) + r.dirty = True + + +class interrupt(FinishCommand): + def do(self) -> None: + import signal + + self.reader.console.finish() + os.kill(os.getpid(), signal.SIGINT) + + +class suspend(Command): + def do(self) -> None: + import signal + + r = self.reader + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + # r.posxy = 0, 0 # XXX this is invalid + r.dirty = True + r.console.screen = [] + + +class up(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y - 1 + + if new_y < 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class down(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y + 1 + + if new_y > r.max_row(): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class left(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + p = r.pos - 1 + if p >= 0: + r.pos = p + else: + self.reader.error("start of buffer") + + +class right(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + p = r.pos + 1 + if p <= len(b): + r.pos = p + else: + self.reader.error("end of buffer") + + +class beginning_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.bol() + + +class end_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.eol() + + +class home(MotionCommand): + def do(self) -> None: + self.reader.pos = 0 + + +class end(MotionCommand): + def do(self) -> None: + self.reader.pos = len(self.reader.buffer) + + +class forward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.eow() + + +class backward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.bow() + + +class self_insert(EditCommand): + def do(self) -> None: + r = self.reader + r.insert(self.event * r.get_arg()) + + +class insert_nl(EditCommand): + def do(self) -> None: + r = self.reader + r.insert("\n" * r.get_arg()) + + +class transpose_characters(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = True + + +class backspace(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +class delete(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + and self.event[-1] == "\004" + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = True + else: + self.reader.error("end of buffer") + + +class accept(FinishCommand): + def do(self) -> None: + pass + + +class help(Command): + def do(self) -> None: + import _sitebuiltins + + with self.reader.suspend(): + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + + +class invalid_key(Command): + def do(self) -> None: + pending = self.reader.console.getpending() + s = "".join(self.event) + pending.data + self.reader.error("`%r' not bound" % s) + + +class invalid_command(Command): + def do(self) -> None: + s = self.event_name + self.reader.error("command `%s' not known" % s) + + +class show_history(Command): + def do(self) -> None: + from .pager import get_pager + from site import gethistoryfile # type: ignore[attr-defined] + + history = os.linesep.join(self.reader.history[:]) + with self.reader.suspend(): + pager = get_pager() + pager(history, gethistoryfile()) + + +class paste_mode(Command): + + def do(self) -> None: + if not self.reader.paste_mode: + self.reader.was_paste_mode_activated = True + self.reader.paste_mode = not self.reader.paste_mode + self.reader.dirty = True + + +class enable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = True + self.reader.was_paste_mode_activated = True + +class disable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = False + self.reader.dirty = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py new file mode 100644 index 00000000000000..19fc06feaf3ced --- /dev/null +++ b/Lib/_pyrepl/completing_reader.py @@ -0,0 +1,287 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from dataclasses import dataclass, field + +import re +from . import commands, console, reader +from .reader import Reader + + +# types +Command = commands.Command +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def prefix(wordlist: list[str], j: int = 0) -> str: + d = {} + i = j + try: + while 1: + for word in wordlist: + d[word[i]] = 1 + if len(d) > 1: + return wordlist[0][j:i] + i += 1 + d = {} + except IndexError: + return wordlist[0][j:i] + return "" + + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + +def stripcolor(s: str) -> str: + return STRIPCOLOR_REGEX.sub('', s) + + +def real_len(s: str) -> int: + return len(stripcolor(s)) + + +def left_align(s: str, maxlen: int) -> str: + stripped = stripcolor(s) + if len(stripped) > maxlen: + # too bad, we remove the color + return stripped[:maxlen] + padding = maxlen - len(stripped) + return s + ' '*padding + + +def build_menu( + cons: console.Console, + wordlist: list[str], + start: int, + use_brackets: bool, + sort_in_column: bool, +) -> tuple[list[str], int]: + if use_brackets: + item = "[ %s ]" + padding = 4 + else: + item = "%s " + padding = 2 + maxlen = min(max(map(real_len, wordlist)), cons.width - padding) + cols = int(cons.width / (maxlen + padding)) + rows = int((len(wordlist) - 1)/cols + 1) + + if sort_in_column: + # sort_in_column=False (default) sort_in_column=True + # A B C A D G + # D E F B E + # G C F + # + # "fill" the table with empty words, so we always have the same amout + # of rows for each column + missing = cols*rows - len(wordlist) + wordlist = wordlist + ['']*missing + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] + wordlist = [wordlist[i] for i in indexes] + menu = [] + i = start + for r in range(rows): + row = [] + for col in range(cols): + row.append(item % left_align(wordlist[i], maxlen)) + i += 1 + if i >= len(wordlist): + break + menu.append(''.join(row)) + if i >= len(wordlist): + i = 0 + break + if r + 5 > cons.height: + menu.append(" %d more... " % (len(wordlist) - i)) + break + return menu, i + +# this gets somewhat user interface-y, and as a result the logic gets +# very convoluted. +# +# To summarise the summary of the summary:- people are a problem. +# -- The Hitch-Hikers Guide to the Galaxy, Episode 12 + +#### Desired behaviour of the completions commands. +# the considerations are: +# (1) how many completions are possible +# (2) whether the last command was a completion +# (3) if we can assume that the completer is going to return the same set of +# completions: this is controlled by the ``assume_immutable_completions`` +# variable on the reader, which is True by default to match the historical +# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match +# more closely readline's semantics (this is needed e.g. by +# fancycompleter) +# +# if there's no possible completion, beep at the user and point this out. +# this is easy. +# +# if there's only one possible completion, stick it in. if the last thing +# user did was a completion, point out that he isn't getting anywhere, but +# only if the ``assume_immutable_completions`` is True. +# +# now it gets complicated. +# +# for the first press of a completion key: +# if there's a common prefix, stick it in. + +# irrespective of whether anything got stuck in, if the word is now +# complete, show the "complete but not unique" message + +# if there's no common prefix and if the word is not now complete, +# beep. + +# common prefix -> yes no +# word complete \/ +# yes "cbnu" "cbnu" +# no - beep + +# for the second bang on the completion key +# there will necessarily be no common prefix +# show a menu of the choices. + +# for subsequent bangs, rotate the menu around (if there are sufficient +# choices). + + +class complete(commands.Command): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions + stem = r.get_stem() + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: + r.error("no matches") + elif len(completions) == 1: + if completions_unchangable and len(completions[0]) == len(stem): + r.msg = "[ sole completion ]" + r.dirty = True + r.insert(completions[0][len(stem):]) + else: + p = prefix(completions, len(stem)) + if p: + r.insert(p) + if last_is_completer: + if not r.cmpltn_menu_vis: + r.cmpltn_menu_vis = 1 + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, r.cmpltn_menu_end, + r.use_brackets, r.sort_in_column) + r.dirty = True + elif stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = True + else: + r.msg = "[ not unique ]" + r.dirty = True + + +class self_insert(commands.self_insert): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + + commands.self_insert.do(self) + + if r.cmpltn_menu_vis: + stem = r.get_stem() + if len(stem) < 1: + r.cmpltn_reset() + else: + completions = [w for w in r.cmpltn_menu_choices + if w.startswith(stem)] + if completions: + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, 0, + r.use_brackets, r.sort_in_column) + else: + r.cmpltn_reset() + + +@dataclass +class CompletingReader(Reader): + """Adds completion support""" + + ### Class variables + # see the comment for the complete command + assume_immutable_completions = True + use_brackets = True # display completions inside [] + sort_in_column = False + + ### Instance variables + cmpltn_menu: list[str] = field(init=False) + cmpltn_menu_vis: int = field(init=False) + cmpltn_menu_end: int = field(init=False) + cmpltn_menu_choices: list[str] = field(init=False) + + def __post_init__(self) -> None: + super().__post_init__() + self.cmpltn_reset() + for c in (complete, self_insert): + self.commands[c.__name__] = c + self.commands[c.__name__.replace('_', '-')] = c + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r'\t', 'complete'),) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if not isinstance(cmd, (complete, self_insert)): + self.cmpltn_reset() + + def calc_screen(self) -> list[str]: + screen = super().calc_screen() + if self.cmpltn_menu_vis: + ly = self.lxy[1] + screen[ly:ly] = self.cmpltn_menu + self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) + self.cxy = self.cxy[0], self.cxy[1] + len(self.cmpltn_menu) + return screen + + def finish(self) -> None: + super().finish() + self.cmpltn_reset() + + def cmpltn_reset(self) -> None: + self.cmpltn_menu = [] + self.cmpltn_menu_vis = 0 + self.cmpltn_menu_end = 0 + self.cmpltn_menu_choices = [] + + def get_stem(self) -> str: + st = self.syntax_table + SW = reader.SYNTAX_WORD + b = self.buffer + p = self.pos - 1 + while p >= 0 and st.get(b[p], SW) == SW: + p -= 1 + return ''.join(b[p+1:self.pos]) + + def get_completions(self, stem: str) -> list[str]: + return [] diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py new file mode 100644 index 00000000000000..d7e86e768671dc --- /dev/null +++ b/Lib/_pyrepl/console.py @@ -0,0 +1,112 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + + +@dataclass +class Event: + evt: str + data: str + raw: bytes = b"" + + +@dataclass +class Console(ABC): + screen: list[str] = field(default_factory=list) + height: int = 25 + width: int = 80 + + @abstractmethod + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + + @abstractmethod + def prepare(self) -> None: ... + + @abstractmethod + def restore(self) -> None: ... + + @abstractmethod + def move_cursor(self, x: int, y: int) -> None: ... + + @abstractmethod + def set_cursor_vis(self, visible: bool) -> None: ... + + @abstractmethod + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + ... + + @abstractmethod + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + ... + + @abstractmethod + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + ... + + @abstractmethod + def beep(self) -> None: ... + + @abstractmethod + def clear(self) -> None: + """Wipe the screen""" + ... + + @abstractmethod + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + ... + + @abstractmethod + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere).""" + ... + + @abstractmethod + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + ... + + @abstractmethod + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + ... + + @abstractmethod + def wait(self) -> None: + """Wait for an event.""" + ... + + @abstractmethod + def repaint(self) -> None: + ... diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 00000000000000..3a624d9f6835d1 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,33 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +try: + import _curses +except ImportError: + try: + import curses as _curses # type: ignore[no-redef] + except ImportError: + from . import _minimal_curses as _curses # type: ignore[no-redef] + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 00000000000000..5b85cb0f52521f --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,74 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 00000000000000..eef7d901b083ef --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,345 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field + +from . import commands, input +from .reader import Reader + + +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class restore_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = True + + +class first_history(commands.Command): + def do(self) -> None: + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self) -> None: + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self) -> None: + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self) -> None: + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = True + + +class forward_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = True + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = True + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = True + + +class isearch_add_character(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = True + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self) -> None: + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = True + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + + +@dataclass +class HistoricalReader(Reader): + """Adds history support (with incremental history searching) to the + Reader class. + """ + + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "last-history"), + (r"\", "first-history"), + ) + + def select_item(self, i: int) -> None: + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i] + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = True + + def get_item(self, i: int) -> str: + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + @contextmanager + def suspend(self) -> SimpleContextManager: + with super().suspend(): + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + + def prepare(self) -> None: + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def isearch_next(self) -> None: + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self) -> None: + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 00000000000000..300e16d1d25441 --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,114 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod +import unicodedata +from collections import deque + + +# types +if False: + from .types import EventTuple + + +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: + pass + + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=0, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self): + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 00000000000000..31a02642ce8ceb --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,215 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +functions for parsing keyspecs + +Support for turning keyspecs into appropriate sequences. + +pyrepl uses it's own bastardized keyspec format, which is meant to be +a strict superset of readline's \"KEYSEQ\" format (which is to say +that if you can come up with a spec readline accepts that this +doesn't, you've found a bug and should tell me about it). + +Note that this is the `\\C-o' style of readline keyspec, not the +`Control-o' sort. + +A keyspec is a string representing a sequence of keypresses that can +be bound to a command. + +All characters other than the backslash represent themselves. In the +traditional manner, a backslash introduces a escape sequence. + +The extension to readline is that the sequence \\ denotes the +sequence of charaters produced by hitting KEY. + +Examples: + +`a' - what you get when you hit the `a' key +`\\EOA' - Escape - O - A (up, on my terminal) +`\\' - the up arrow key +`\\' - ditto (keynames are case insensitive) +`\\C-o', `\\c-o' - control-o +`\\M-.' - meta-period +`\\E.' - ditto (that's how meta works for pyrepl) +`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. Can you think of any more? +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def _parse_key1(key, s): + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) > 1: + raise KeySpecError("\\C- must be followed by a character") + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + if meta: + ret = ["\033", ret] + else: + ret = [ret] + return ret, s + + +def parse_keys(key): + s = 0 + r = [] + while s < len(key): + k, s = _parse_key1(key, s) + r.extend(k) + return r + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 00000000000000..ecd03094dbf538 --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,27 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_any_generics = False +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False + +disable_error_code = return + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 00000000000000..af0409c4523bc2 --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol, Any + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plain_pager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfilepager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + getchar = lambda: sys.stdin.read(1) + has_tty = True + except (ImportError, AttributeError, io.UnsupportedOperation): + getchar = lambda: sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 00000000000000..d15a150180811d --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,666 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field, fields +import unicodedata +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] + + +from . import commands, console, input +from .utils import ANSI_ESCAPE_SEQUENCE, wlen +from .trace import trace + + +# types +Command = commands.Command +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def disp_str(buffer: str) -> tuple[str, list[int]]: + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed represenation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + """ + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) + b.append(wlen(c)) + b.extend([0] * (len(c) - 1)) + return "".join(s), b + + +# syntax classes: + +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) + + +def make_default_syntax_table() -> dict[str, int]: + # XXX perhaps should use some unicodedata here? + st: dict[str, int] = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + (r"\C-z", "suspend"), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + # (r'\M-\n', 'insert-nl'), + ("\\\\", "self-insert"), + (r"\x1b[200~", "enable_bracketed_paste"), + (r"\x1b[201~", "disable_bracketed_paste"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\", "up"), + (r"\", "down"), + (r"\", "left"), + (r"\", "right"), + (r"\", "delete"), + (r"\", "backspace"), + (r"\M-\", "backward-kill-word"), + (r"\", "end-of-line"), # was 'end' + (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), + (r"\", "show-history"), + (r"\", "paste-mode"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + + +@dataclass(slots=True) +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into `buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. + * cxy, lxy: + the position of the insertion point in screen ... + * syntax_table: + Dictionary mapping characters to `syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list[list[str]] = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + was_paste_mode_activated: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + msg_at_bottom: bool = True + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + + def __post_init__(self) -> None: + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # . + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + self.screeninfo = [(0, [0])] + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return default_keymap + + def calc_screen(self) -> list[str]: + """The purpose of this method is to translate changes in + self.buffer into changes in self.screen. Currently it rips + everything down and starts from scratch, which whilst not + especially efficient is certainly simple(r). + """ + lines = self.get_unicode().split("\n") + screen: list[str] = [] + screeninfo: list[tuple[int, list[int]]] = [] + pos = self.pos + for ln, line in enumerate(lines): + ll = len(line) + if 0 <= pos <= ll: + if self.msg and not self.msg_at_bottom: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + self.lxy = pos, ln + prompt = self.get_prompt(ln, ll >= pos >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + screen.append(pre_prompt) + screeninfo.append((0, [])) + pos -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (wlen(l) + lp) // self.console.width + if wrapcount == 0: + screen.append(prompt + l) + screeninfo.append((lp, l2)) + else: + for i in range(wrapcount + 1): + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= self.console.width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + post = "\\" if i != wrapcount else "" + after = [1] if i != wrapcount else [] + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] + self.screeninfo = screeninfo + self.cxy = self.pos2xy() + if self.msg and self.msg_at_bottom: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + return screen + + def process_prompt(self, prompt: str) -> tuple[str, int]: + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + out_prompt = "" + l = wlen(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + + def get_arg(self, default: int = 1) -> int: + """Return any prefix argument that the user has supplied, + returning `default' if there is None. Defaults to 1. + """ + if self.arg is None: + return default + else: + return self.arg + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + """Return what should be in the left-hand margin for line + `lineno'.""" + if self.arg is not None and cursor_on_line: + prompt = "(arg: %s) " % self.arg + elif self.paste_mode: + prompt = "(paste) " + elif "\n" in self.buffer: + if lineno == 0: + prompt = self.ps2 + elif lineno == self.buffer.count("\n"): + prompt = self.ps4 + else: + prompt = self.ps3 + else: + prompt = self.ps1 + + if can_colorize(): + prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + return prompt + + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self) -> None: + self.input_trans = self.input_trans_stack.pop() + + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordinates x, y""" + pos = 0 + i = 0 + while i < y: + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + + def pos2xy(self) -> tuple[int, int]: + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + y = 0 + pos = self.pos + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer): + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: + break + else: + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer + y += 1 + return p + sum(l2[:pos]), y + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = True + + def update_cursor(self) -> None: + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy() + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd: Command) -> None: + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", True): + if self.arg is not None: + self.dirty = True + self.arg = None + + def prepare(self) -> None: + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + + def last_command_is(self, cls: type) -> bool: + if not self.last_command: + return False + return issubclass(cls, self.last_command) + + def restore(self) -> None: + """Clean up after a run.""" + self.console.restore() + + @contextmanager + def suspend(self) -> SimpleContextManager: + """A context manager to delegate to another reader.""" + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} + try: + self.restore() + yield + finally: + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): + setattr(self, arg, prev_state[arg]) + self.prepare() + pass + + def finish(self) -> None: + """Called when a command signals that we're finished.""" + pass + + def error(self, msg: str = "none") -> None: + self.msg = "! " + msg + " " + self.dirty = True + self.console.beep() + + def update_screen(self) -> None: + if self.dirty: + self.refresh() + + def refresh(self) -> None: + """Recalculate and refresh the screen.""" + # this call sets up self.cxy, so call it first. + screen = self.calc_screen() + self.console.refresh(screen, self.cxy) + self.dirty = False + + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" + + trace("received command {cmd}", cmd=cmd) + if isinstance(cmd[0], str): + command_type = self.commands.get(cmd[0], commands.invalid_command) + elif isinstance(cmd[0], type): + command_type = cmd[0] + else: + return # nothing to do + + command = command_type(self, *cmd) # type: ignore[arg-type] + command.do() + + self.after_command(command) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = command_type + + self.finished = bool(command.finish) + if self.finished: + self.console.finish() + self.finish() + + def handle1(self, block: bool = True) -> bool: + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return False if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = True + + while True: + event = self.console.get_event(block) + if not event: # can only happen if we're not blocking + return False + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = [event.evt, event.data] + + if cmd is None: + if block: + continue + else: + return False + + self.do_cmd(cmd) + return True + + def push_char(self, char: int | bytes) -> None: + self.console.push_char(char) + self.handle1(block=False) + + def readline(self, startup_hook: Callback | None = None) -> str: + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + return self.get_unicode() + + finally: + self.restore() + + def bind(self, spec: KeySpec, command: CommandName) -> None: + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_unicode(self) -> str: + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 00000000000000..d28a7f3779f302 --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,502 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import os +import readline +from site import gethistoryfile # type: ignore[attr-defined] +import sys + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .unix_console import UnixConsole, _error + +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + # "read_init_file", + # "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + # "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +@dataclass +class ReadlineConfig: + readline_completer: Completer | None = readline.get_completer() + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +@dataclass(kw_only=True) +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + # Instance fields + config: ReadlineConfig + more_lines: Callable[[str], bool] | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: + pass # don't show error messages by default + + def get_stem(self) -> str: + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem: str) -> list[str]: + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength: int) -> list[str]: + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + # --- simplified support for reading multiline Python statements --- + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\", "backspace-dedent"), + ) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history: bool) -> None: + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer: list[str], pos: int) -> int: + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +class maybe_accept(commands.Command): + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible + # + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + # + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if not self.reader.paste_mode and indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + elif not self.reader.paste_mode: + self.finish = True + else: + r.insert("\n") + + +class backspace_dedent(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +@dataclass(slots=True) +class _ReadlineWrapper: + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = None + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: + if self.reader is None: + console = UnixConsole(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console=console, config=self.config) + return self.reader + + def input(self, prompt: object = "") -> str: + try: + reader = self.get_reader() + except _error: + assert raw_input is not None + return raw_input(prompt) + reader.ps1 = str(prompt) + return reader.readline(startup_hook=self.startup_hook) + + def multiline_input(self, more_lines, ps1, ps2): + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = reader.ps2 = ps1 + reader.ps3 = reader.ps4 = ps2 + return reader.readline(), reader.was_paste_mode_activated + finally: + reader.more_lines = saved + reader.paste_mode = False + reader.was_paste_mode_activated = False + + def parse_and_bind(self, string: str) -> None: + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function: Completer | None = None) -> None: + self.config.readline_completer = function + + def get_completer(self) -> Completer | None: + return self.config.readline_completer + + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) + + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) + + def _histline(self, line: str) -> str: + line = line.rstrip("\n") + return line + + def get_history_length(self) -> int: + return self.saved_history_length + + def set_history_length(self, length: int) -> None: + self.saved_history_length = length + + def get_current_history_length(self) -> int: + return len(self.get_reader().history) + + def read_history_file(self, filename: str = gethistoryfile()) -> None: + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + + with open(os.path.expanduser(filename), 'rb') as f: + lines = [line.decode('utf-8', errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + # Ignore readline history file header + if line.startswith("_HiStOrY_V2_"): + continue + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = "".join(buffer).replace("\r", "") + line + del buffer[:] + if line: + history.append(line) + + def write_history_file(self, filename: str = gethistoryfile()) -> None: + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self) -> None: + del self.get_reader().history[:] + + def get_history_item(self, index: int) -> str | None: + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # like readline.c + + def remove_history_item(self, index: int) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def replace_history_item(self, index: int, line: str) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def add_history(self, line: str) -> None: + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function: Callback | None = None) -> None: + self.startup_hook = function + + def get_line_buffer(self) -> bytes: + buf_str = self.get_reader().get_unicode() + return buf_str.encode(ENCODING) + + def _get_idxs(self) -> tuple[int, int]: + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if str(buf[i]) in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self) -> int: + return self._get_idxs()[0] + + def get_endidx(self) -> int: + return self._get_idxs()[1] + + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.__name__ = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup() -> None: + global raw_input + if raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # this is not really what readline.c does. Better than nothing I guess + import builtins + + raw_input = builtins.input + builtins.input = _wrapper.input + + +raw_input: Callable[[object], str] | None = None diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 00000000000000..31b2097a78a226 --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,158 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +from __future__ import annotations + +import _colorize # type: ignore[import-not-found] +import _sitebuiltins +import linecache +import sys +import code +from types import ModuleType + +from .readline import _get_reader, multiline_input +from .unix_console import _error + + +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" + try: + _get_reader() + except _error as e: + return str(e) or repr(e) or "unknown error" + return "" + + +def _strip_final_indent(text: str) -> str: + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": "help", +} + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showtraceback(self): + super().showtraceback(colorize=self.can_colorize) + + +def run_multiline_interactive_console( + mainmodule: ModuleType | None= None, future_flags: int = 0 +) -> None: + import __main__ + from .readline import _setup + _setup() + + mainmodule = mainmodule or __main__ + console = InteractiveColoredConsole(mainmodule.__dict__, filename="") + if future_flags: + console.compile.compiler.flags |= future_flags + + input_n = 0 + + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + command() + return True + + if isinstance(command, str): + # Internal readline commands require a prepared reader like + # inside multiline_input. + reader.prepare() + reader.refresh() + reader.do_cmd((command, [statement])) + reader.restore() + return True + + return False + + def more_lines(unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + while 1: + try: + try: + sys.stdout.flush() + except Exception: + pass + + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement, contains_pasted_code = multiline_input(more_lines, ps1, ps2) + except EOFError: + break + + if maybe_run_command(statement): + continue + + input_name = f"" + linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] + symbol = "single" if not contains_pasted_code else "exec" + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol=symbol) # type: ignore[call-arg] + if contains_pasted_code and more: + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] + assert not more + input_n += 1 + except KeyboardInterrupt: + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 00000000000000..a8eb2433cd3cce --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +# types +if False: + from typing import IO + + +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): + trace_file = open(trace_filename, "a") + + +def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 00000000000000..f9d48b828c720b --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,8 @@ +from collections.abc import Callable, Iterator + +Callback = Callable[[], object] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 00000000000000..605318c82ae2ea --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,752 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import errno +import os +import re +import select +import signal +import struct +import sys +import termios +import time +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue +from .utils import wlen + + +# types +if False: + from typing import IO + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + +# ------------ start of baudrate definitions ------------ + +# Add (possibly) missing baudrates (check termios man page) to termios + + +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + + +# Check the termios man page (Line speed) to know where these +# values come from. +potential_baudrates = [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +] + +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) + +# Clean up variables to avoid unintended usage +del rate, add_baudrate_if_supported + +# ------------ end of baudrate definitions ------------ + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll: type[select.poll] = select.poll +except AttributeError: + # this is exactly the minumum necessary to support what we + # do with poll objects + class MinimalPoll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r, w, e = select.select([self.fd], [], []) + return r + + poll = MinimalPoll # type: ignore[assignment] + + +class UnixConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ + + self.encoding = encoding or sys.getdefaultencoding() + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + + self.pollob = poll() + self.pollob.register(self.input_fd, select.POLLIN) + curses.setupterm(term or None, self.output_fd) + self.term = term + + def _my_getstr(cap, optional=0): + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) + + self.__setup_movement() + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + def change_encoding(self, encoding: str) -> None: + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ + self.encoding = encoding + + def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.__posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.__posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.__posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + self.flushoutput() + + def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.__posxy = x, y + self.flushoutput() + + def prepare(self): + """ + Prepare the console for input/output operations. + """ + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.lflag &= ~( + termios.ICANON | termios.ECHO | termios.IEXTEN | (termios.ISIG * 1) + ) + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + self.__enable_bracketed_paste() + + def restore(self): + """ + Restore the console to the default state + """ + self.__disable_bracketed_paste() + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block: bool = True) -> Event | None: + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ + while self.event_queue.empty(): + while True: + try: + self.push_char(os.read(self.input_fd, 1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + if not block: + break + return self.event_queue.get() + + def wait(self): + """ + Wait for events on the console. + """ + self.pollob.poll() + + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: + self.__show_cursor() + else: + self.__hide_cursor() + + if TIOCGWINSZ: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + height, width = struct.unpack( + "hhhh", ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + )[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + return 25, 80 + + def forgetinput(self): + """ + Discard any pending input on the console. + """ + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + """ + Flush the output buffer. + """ + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def finish(self): + """ + Finish console operations and flush the output buffer. + """ + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + """ + Emit a beep sound. + """ + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = os.read(self.input_fd, amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = os.read(self.input_fd, amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + """ + Clear the console screen. + """ + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.__posxy = 0, 0 + self.screen = [] + + def __enable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004h") + + def __disable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004l") + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px_coord): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while x_coord < minlen and oldline[x_pos] == newline[x_pos] and newline[x_pos] != "\x1b": + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + if ( + y == self.__posxy[1] + and x_coord > self.__posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + ): + x_pos = px_pos + x_coord = px_coord + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif x_coord < minlen and oldline[x_pos + 1 :] == newline[x_pos + 1 :] and wlen(oldline[x_pos]) == wlen(newline[x_pos]): + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line + elif ( + self.dch1 + and self.ich1 + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.__posxy = self.width - 2, y + self.__write_code(self.dch1) + + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = character_width + 1, y + + else: + self.__hide_cursor() + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): + self.__write_code(self._el) + self.__write(newline[x_pos:]) + self.__posxy = wlen(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x): + if x != self.__posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint(self): + if not self.__gone_tall: + self.__posxy = 0, self.__posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.__posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 00000000000000..70cfade26e23b1 --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,152 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) + +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def get_terminal_keycodes() -> dict[bytes, str]: + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ + keycodes = {} + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODES) + return keycodes + +class EventQueue: + def __init__(self, fd: int, encoding: str) -> None: + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.keymap: + if self.keymap is self.compiled_keymap: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', bytearray(b'\033'))) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 00000000000000..cd1df7c49a216d --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,18 @@ +import re +import unicodedata + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +def str_width(c: str) -> int: + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + length = sum(str_width(i) for i in s) + + # remove lengths of any escape sequences + return length - sum(len(i) for i in ANSI_ESCAPE_SEQUENCE.findall(s)) diff --git a/Lib/argparse.py b/Lib/argparse.py index 0dbdd67a82f391..55bf8cdd875a8d 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -328,17 +328,8 @@ def _format_usage(self, usage, actions, groups, prefix): if len(prefix) + len(usage) > text_width: # break usage into wrappable parts - part_regexp = ( - r'\(.*?\)+(?=\s|$)|' - r'\[.*?\]+(?=\s|$)|' - r'\S+' - ) - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage + opt_parts = self._get_actions_usage_parts(optionals, groups) + pos_parts = self._get_actions_usage_parts(positionals, groups) # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -391,6 +382,9 @@ def get_lines(parts, indent, prefix=None): return '%s%s\n\n' % (prefix, usage) def _format_actions_usage(self, actions, groups): + return ' '.join(self._get_actions_usage_parts(actions, groups)) + + def _get_actions_usage_parts(self, actions, groups): # find group indices and identify actions in groups group_actions = set() inserts = {} @@ -398,58 +392,26 @@ def _format_actions_usage(self, actions, groups): if not group._group_actions: raise ValueError(f'empty group {group}') + if all(action.help is SUPPRESS for action in group._group_actions): + continue + try: start = actions.index(group._group_actions[0]) except ValueError: continue else: - group_action_count = len(group._group_actions) - end = start + group_action_count + end = start + len(group._group_actions) if actions[start:end] == group._group_actions: - - suppressed_actions_count = 0 - for action in group._group_actions: - group_actions.add(action) - if action.help is SUPPRESS: - suppressed_actions_count += 1 - - exposed_actions_count = group_action_count - suppressed_actions_count - if not exposed_actions_count: - continue - - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - if end in inserts: - inserts[end] += ']' - else: - inserts[end] = ']' - elif exposed_actions_count > 1: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - if end in inserts: - inserts[end] += ')' - else: - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' + group_actions.update(group._group_actions) + inserts[start, end] = group # collect all actions format strings parts = [] - for i, action in enumerate(actions): + for action in actions: # suppressed arguments are marked with None - # remove | separators for suppressed arguments if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) + part = None # produce all arg strings elif not action.option_strings: @@ -461,9 +423,6 @@ def _format_actions_usage(self, actions, groups): if part[0] == '[' and part[-1] == ']': part = part[1:-1] - # add the action string to the list - parts.append(part) - # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] @@ -484,26 +443,23 @@ def _format_actions_usage(self, actions, groups): if not action.required and action not in group_actions: part = '[%s]' % part - # add the action string to the list - parts.append(part) + # add the action string to the list + parts.append(part) - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = text.strip() + # group mutually exclusive actions + for start, end in sorted(inserts, reverse=True): + group = inserts[start, end] + group_parts = [item for item in parts[start:end] if item is not None] + if group.required: + open, close = "()" if len(group_parts) > 1 else ("", "") + else: + open, close = "[]" + parts[start] = open + " | ".join(group_parts) + close + for i in range(start + 1, end): + parts[i] = None - # return the text - return text + # return the usage parts + return [item for item in parts if item is not None] def _format_text(self, text): if '%(prog)' in text: diff --git a/Lib/ast.py b/Lib/ast.py index 9f386051659e76..d7e51aba595706 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1124,12 +1124,21 @@ def visit_TypeVar(self, node): if node.bound: self.write(": ") self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeVarTuple(self, node): self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_ParamSpec(self, node): self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeAlias(self, node): self.fill("type ") diff --git a/Lib/bdb.py b/Lib/bdb.py index 1acf7957f0d669..7d63fce6ca63f1 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -32,8 +32,10 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} - self.frame_trace_lines = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None self._load_breaks() @@ -85,6 +87,9 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ + + self.enterframe = frame + if self.quitting: return # None if event == 'line': @@ -101,6 +106,8 @@ def trace_dispatch(self, frame, event, arg): return self.trace_dispatch if event == 'c_return': return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch @@ -187,6 +194,17 @@ def dispatch_exception(self, frame, arg): return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -273,7 +291,21 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -286,6 +318,17 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame. + if self.frame_returning: + caller_frame = self.frame_returning.f_back + if caller_frame and not caller_frame.f_trace: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -300,16 +343,14 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch + self._set_caller_tracefunc() self._set_stopinfo(None, None) + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_caller_tracefunc() + self._set_stopinfo(None, None, opcode=True) + def set_next(self, frame): """Stop on the next line in or below the given frame.""" self._set_stopinfo(frame, None) @@ -329,14 +370,15 @@ def set_trace(self, frame=None): if frame is None: frame = sys._getframe().f_back self.reset() + self.enterframe = frame while frame: frame.f_trace = self.trace_dispatch self.botframe = frame - # We need f_trace_liens == True for the debugger to work - self.frame_trace_lines[frame] = frame.f_trace_lines + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work frame.f_trace_lines = True frame = frame.f_back - self.set_step() + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -353,9 +395,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back - for frame, prev_trace_lines in self.frame_trace_lines.items(): - frame.f_trace_lines = prev_trace_lines - self.frame_trace_lines = {} + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. diff --git a/Lib/code.py b/Lib/code.py index f4aecddeca7813..9d124563f728c2 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -130,7 +130,7 @@ def showsyntaxerror(self, filename=None): # over self.write sys.excepthook(type, value, tb) - def showtraceback(self): + def showtraceback(self, **kwargs): """Display the exception that just occurred. We remove the first stack item because it is our own code. @@ -138,11 +138,12 @@ def showtraceback(self): The output is written by self.write(), below. """ + colorize = kwargs.pop('colorize', False) sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() sys.last_traceback = last_tb sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) + lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next, colorize=colorize) if sys.excepthook is sys.__excepthook__: self.write(''.join(lines)) else: @@ -170,7 +171,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename="", local_exit=False): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -280,7 +281,7 @@ def interact(self, banner=None, exitmsg=None): elif exitmsg != '': self.write('%s\n' % exitmsg) - def push(self, line): + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -296,7 +297,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more diff --git a/Lib/dis.py b/Lib/dis.py index 111d624fc259c5..76934eb00e63f0 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -216,7 +216,7 @@ def _get_code_array(co, adaptive): if op == ENTER_EXECUTOR: try: ex = get_executor(co, i) - except ValueError: + except (ValueError, RuntimeError): ex = None if ex: diff --git a/Lib/doctest.py b/Lib/doctest.py index d8c632e47e7b7a..c531e3ca6a3d5e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,7 +104,8 @@ def _test(): import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple -from traceback import _ANSIColors, _can_colorize +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize class TestResults(namedtuple('TestResults', 'failed attempted')): @@ -1180,8 +1181,8 @@ class DocTestRunner: The `run` method is used to process a single DocTest case. It returns a TestResults instance. - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1234,7 +1235,7 @@ class DocTestRunner: overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1314,7 +1315,7 @@ def report_unexpected_exception(self, out, test, example, exc_info): def _failure_header(self, test, example): red, reset = ( - (_ANSIColors.RED, _ANSIColors.RESET) if _can_colorize() else ("", "") + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") ) out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: @@ -1556,8 +1557,8 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - saved_can_colorize = traceback._can_colorize - traceback._can_colorize = lambda: False + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda: False color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} for key in color_variables: color_variables[key] = os.environ.pop(key, None) @@ -1569,7 +1570,7 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook - traceback._can_colorize = saved_can_colorize + _colorize.can_colorize = saved_can_colorize for key, value in color_variables.items(): if value is not None: os.environ[key] = value @@ -1609,20 +1610,13 @@ def summarize(self, verbose=None): else: failed.append((name, (failures, tries, skips))) - if _can_colorize(): - bold_green = _ANSIColors.BOLD_GREEN - bold_red = _ANSIColors.BOLD_RED - green = _ANSIColors.GREEN - red = _ANSIColors.RED - reset = _ANSIColors.RESET - yellow = _ANSIColors.YELLOW - else: - bold_green = "" - bold_red = "" - green = "" - red = "" - reset = "" - yellow = "" + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW if verbose: if notests: diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c9cbadd2a80c48..2ec54fbabae83c 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -152,7 +152,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): mangle_from_ -- a flag that, when True escapes From_ lines in the body of the message by putting a `>' in front of them. This is used when the message is being - serialized by a generator. Default: True. + serialized by a generator. Default: False. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. diff --git a/Lib/enum.py b/Lib/enum.py index 98a49eafbb9897..c36fc75a24a239 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -365,7 +365,10 @@ def __setitem__(self, key, value): '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', '_add_alias_', '_add_value_alias_', - ): + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) @@ -2035,7 +2038,7 @@ def _test_simple_enum(checked_enum, simple_enum): ) for key in set(checked_keys + simple_keys): if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', - '__static_attributes__'): + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/glob.py b/Lib/glob.py index 72cf22299763f0..6088de00a67a99 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -10,7 +10,8 @@ import stat import sys -__all__ = ["glob", "iglob", "escape"] + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -339,24 +340,27 @@ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): # Low-level methods - lstat = staticmethod(os.lstat) - scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') - concat_path = operator.add + lstat = operator.methodcaller('lstat') + add_slash = operator.methodcaller('joinpath', '') - if os.name == 'nt': - @staticmethod - def add_slash(pathname): - tail = os.path.splitroot(pathname)[2] - if not tail or tail[-1] in '\\/': - return pathname - return f'{pathname}\\' - else: - @staticmethod - def add_slash(pathname): - if not pathname or pathname[-1] == '/': - return pathname - return f'{pathname}/' + @staticmethod + def scandir(path): + """Emulates os.scandir(), which returns an object that can be used as + a context manager. This method is called by walk() and glob(). + """ + return contextlib.nullcontext(path.iterdir()) + + @staticmethod + def concat_path(path, text): + """Appends text to the given path. + """ + return path.with_segments(path._raw_path + text) + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir(). + """ + return entry # High-level methods @@ -555,3 +559,24 @@ def walk(cls, root, top_down, on_error, follow_symlinks): if dirnames: prefix = cls.add_slash(path) paths += [cls.concat_path(prefix, d) for d in reversed(dirnames)] + + +class _StringGlobber(_Globber): + lstat = staticmethod(os.lstat) + scandir = staticmethod(os.scandir) + parse_entry = operator.attrgetter('path') + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/http/server.py b/Lib/http/server.py index ee7a9b6aa55b88..7d0da5052d2d4d 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -897,7 +897,7 @@ def guess_type(self, path): ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] - guess, _ = mimetypes.guess_type(path) + guess, _ = mimetypes.guess_file_type(path) if guess: return guess return 'application/octet-stream' diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 241b1f48e5c1d8..fb07d7b3b3fad8 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,8 @@ Released on 2024-10-xx ========================= +gh-78955: Use user-selected color theme for Help => IDLE Doc. + gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'. gh-72284: Improve the lists of features, editor key bindings, diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index bdf4b2b29f11a2..d8613b2eadd6aa 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -33,6 +33,7 @@ from tkinter import font as tkfont from idlelib.config import idleConf +from idlelib.colorizer import color_config ## About IDLE ## @@ -177,14 +178,16 @@ def __init__(self, parent, filename): normalfont = self.findfont(['TkDefaultFont', 'arial', 'helvetica']) fixedfont = self.findfont(['TkFixedFont', 'monaco', 'courier']) + color_config(self) self['font'] = (normalfont, 12) self.tag_configure('em', font=(normalfont, 12, 'italic')) self.tag_configure('h1', font=(normalfont, 20, 'bold')) self.tag_configure('h2', font=(normalfont, 18, 'bold')) self.tag_configure('h3', font=(normalfont, 15, 'bold')) - self.tag_configure('pre', font=(fixedfont, 12), background='#f6f6ff') + self.tag_configure('pre', font=(fixedfont, 12)) + preback = self['selectbackground'] self.tag_configure('preblock', font=(fixedfont, 10), lmargin1=25, - borderwidth=1, relief='solid', background='#eeffcc') + background=preback) self.tag_configure('l1', lmargin1=25, lmargin2=25) self.tag_configure('l2', lmargin1=50, lmargin2=50) self.tag_configure('l3', lmargin1=75, lmargin2=75) diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 08728956abd900..8e2ba68d3815bf 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -106,6 +106,7 @@ def dispatch(self, operation, *args): to *args to accomplish that. For an example, see colorizer.py. ''' + operation = str(operation) # can be a Tcl_Obj m = self._operations.get(operation) try: if m: diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6d6292f9559253..de5651f0a7fc36 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1134,7 +1134,7 @@ def find_spec(cls, fullname, path=None, target=None): # part of the importer), instead of here (the finder part). # The loader is the usual place to get the data that will # be loaded into the module. (For example, see _LoaderBasics - # in _bootstra_external.py.) Most importantly, this importer + # in _bootstrap_external.py.) Most importantly, this importer # is simpler if we wait to get the data. # However, getting as much data in the finder as possible # to later load the module is okay, and sometimes important. diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 0a11dc9efc252c..db446776901fc3 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -471,6 +471,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3567 (Reimplement line number propagation by the compiler) # Python 3.13a1 3568 (Change semantics of END_FOR) # Python 3.13a5 3569 (Specialize CONTAINS_OP) +# Python 3.13a6 3570 (Add __firstlineno__ class attribute) # Python 3.14 will start with 3600 @@ -487,7 +488,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3569).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3570).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c @@ -1463,7 +1464,7 @@ class PathFinder: @staticmethod def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" + stored in sys.path_importer_cache (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): # Drop entry if finder name is a relative path. The current # working directory may have changed. diff --git a/Lib/inspect.py b/Lib/inspect.py index 1f4216f0389d28..84260b251a4fb8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1035,79 +1035,6 @@ class ClassFoundException(Exception): pass -class _ClassFinder(ast.NodeVisitor): - - def __init__(self, cls, tree, lines, qualname): - self.stack = [] - self.cls = cls - self.tree = tree - self.lines = lines - self.qualname = qualname - self.lineno_found = [] - - def visit_FunctionDef(self, node): - self.stack.append(node.name) - self.stack.append('') - self.generic_visit(node) - self.stack.pop() - self.stack.pop() - - visit_AsyncFunctionDef = visit_FunctionDef - - def visit_ClassDef(self, node): - self.stack.append(node.name) - if self.qualname == '.'.join(self.stack): - # Return the decorator for the class if present - if node.decorator_list: - line_number = node.decorator_list[0].lineno - else: - line_number = node.lineno - - # decrement by one since lines starts with indexing by zero - self.lineno_found.append((line_number - 1, node.end_lineno)) - self.generic_visit(node) - self.stack.pop() - - def get_lineno(self): - self.visit(self.tree) - lineno_found_number = len(self.lineno_found) - if lineno_found_number == 0: - raise OSError('could not find class definition') - elif lineno_found_number == 1: - return self.lineno_found[0][0] - else: - # We have multiple candidates for the class definition. - # Now we have to guess. - - # First, let's see if there are any method definitions - for member in self.cls.__dict__.values(): - if (isinstance(member, types.FunctionType) and - member.__module__ == self.cls.__module__): - for lineno, end_lineno in self.lineno_found: - if lineno <= member.__code__.co_firstlineno <= end_lineno: - return lineno - - class_strings = [(''.join(self.lines[lineno: end_lineno]), lineno) - for lineno, end_lineno in self.lineno_found] - - # Maybe the class has a docstring and it's unique? - if self.cls.__doc__: - ret = None - for candidate, lineno in class_strings: - if self.cls.__doc__.strip() in candidate: - if ret is None: - ret = lineno - else: - break - else: - if ret is not None: - return ret - - # We are out of ideas, just return the last one found, which is - # slightly better than previous ones - return self.lineno_found[-1][0] - - def findsource(object): """Return the entire source file and starting line number for an object. @@ -1140,11 +1067,11 @@ def findsource(object): return lines, 0 if isclass(object): - qualname = object.__qualname__ - source = ''.join(lines) - tree = ast.parse(source) - class_finder = _ClassFinder(object, tree, lines, qualname) - return lines, class_finder.get_lineno() + try: + firstlineno = object.__firstlineno__ + except AttributeError: + raise OSError('source code not available') + return lines, object.__firstlineno__ - 1 if ismethod(object): object = object.__func__ @@ -2179,8 +2106,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): @@ -2283,7 +2212,12 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): module = None module_dict = {} + module_name = getattr(obj, '__module__', None) + if not module_name: + objclass = getattr(obj, '__objclass__', None) + module_name = getattr(objclass, '__module__', None) + if module_name: module = sys.modules.get(module_name, None) if module: diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 597849eca0524a..323332f064edf8 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -244,15 +244,18 @@ def floatstr(o, allow_nan=self.allow_nan, return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): + if self.indent is None or isinstance(self.indent, str): + indent = self.indent + else: + indent = ' ' * self.indent + if _one_shot and c_make_encoder is not None: _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, + markers, self.default, _encoder, indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, + markers, self.default, _encoder, indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) @@ -272,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index dad3813e39dbae..8604000ed77a19 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -40,7 +40,7 @@ __all__ = [ "knownfiles", "inited", "MimeTypes", - "guess_type", "guess_all_extensions", "guess_extension", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] @@ -119,14 +119,14 @@ def guess_type(self, url, strict=True): Optional `strict' argument when False adds a bunch of commonly found, but non-standard types. """ + # TODO: Deprecate accepting file paths (in particular path-like objects). url = os.fspath(url) p = urllib.parse.urlparse(url) if p.scheme and len(p.scheme) > 1: scheme = p.scheme url = p.path else: - scheme = None - url = os.path.splitdrive(url)[1] + return self.guess_file_type(url, strict=strict) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data @@ -146,13 +146,25 @@ def guess_type(self, url, strict=True): if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) while (ext_lower := ext.lower()) in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext_lower]) + base, ext = splitext(base + self.suffix_map[ext_lower]) # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) + base, ext = splitext(base) else: encoding = None ext = ext.lower() @@ -310,6 +322,16 @@ def guess_type(url, strict=True): return _db.guess_type(url, strict) +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) + + def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. @@ -552,6 +574,8 @@ def _default_mime_types(): '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', '.n3' : 'text/n3', '.txt' : 'text/plain', '.bat' : 'text/plain', @@ -565,6 +589,7 @@ def _default_mime_types(): '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', + '.rst' : 'text/x-rst', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 4642707dae2f4e..53b8c492675878 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,3 +1,4 @@ +import atexit import errno import os import selectors @@ -271,6 +272,8 @@ def sigchld_handler(*_unused): selector.close() unused_fds = [alive_r, child_w, sig_r, sig_w] unused_fds.extend(pid_to_fd.values()) + atexit._clear() + atexit.register(util._exit_function) code = _serve_one(child_r, fds, unused_fds, old_handlers) @@ -278,6 +281,7 @@ def sigchld_handler(*_unused): sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: + atexit._run_exitfuncs() os._exit(code) else: # Send pid to client process diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47627c..a57ef6bdad5ccc 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -1,3 +1,4 @@ +import atexit import os import signal @@ -66,10 +67,13 @@ def _launch(self, process_obj): self.pid = os.fork() if self.pid == 0: try: + atexit._clear() + atexit.register(util._exit_function) os.close(parent_r) os.close(parent_w) code = process_obj._bootstrap(parent_sentinel=child_r) finally: + atexit._run_exitfuncs() os._exit(code) else: os.close(child_w) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 271ba3fd325138..b45f7df476f7d8 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -310,11 +310,8 @@ def _bootstrap(self, parent_sentinel=None): # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') - try: - self.run() - exitcode = 0 - finally: - util._exit_function() + self.run() + exitcode = 0 except SystemExit as e: if e.code is None: exitcode = 0 diff --git a/Lib/ntpath.py b/Lib/ntpath.py index e810b655e5ac85..b833e0bad2645f 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -805,6 +805,9 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + if not path: + raise ValueError("no path specified") + if isinstance(path, bytes): sep = b'\\' curdir = b'.' @@ -816,22 +819,20 @@ def relpath(path, start=None): if start is None: start = curdir + else: + start = os.fspath(start) - if not path: - raise ValueError("no path specified") - - start = os.fspath(start) try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) + start_abs = abspath(start) + path_abs = abspath(path) start_drive, _, start_rest = splitroot(start_abs) path_drive, _, path_rest = splitroot(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] + start_list = start_rest.split(sep) if start_rest else [] + path_list = path_rest.split(sep) if path_rest else [] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): @@ -842,7 +843,7 @@ def relpath(path, start=None): rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index f03f317ef6c16a..4b3edf535a61aa 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,866 +5,8 @@ operating systems. """ -import glob -import io -import ntpath -import operator -import os -import posixpath -import sys -import warnings -from itertools import chain -from _collections_abc import Sequence +from ._abc import * +from ._local import * -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -from . import _abc - - -__all__ = [ - "UnsupportedOperation", - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -UnsupportedOperation = _abc.UnsupportedOperation - - -class PurePath(_abc.PurePathBase): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - parser = os.path - _globber = glob._Globber - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.parser is ntpath and self.parser is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - # Avoid calling super().__init__, as an optimisation - self._raw_paths = paths - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.with_segments(self, key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - def __reduce__(self): - return self.__class__, tuple(self._raw_paths) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def __fspath__(self): - return str(self) - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _abc._is_case_sensitive(self.parser): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.parser is other.parser - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) - return self._parts_normcase_cached - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls.parser.sep.join(tail) - elif tail and cls.parser.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls.parser.sep.join(tail) - - def _from_parsed_parts(self, drv, root, tail): - path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - def _from_parsed_string(self, path_str): - path = self.with_segments(path_str) - path._str = path_str or '.' - return path - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls.parser.sep - altsep = cls.parser.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls.parser.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - @property - def _raw_path(self): - """The joined but unnormalized path.""" - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.parser.join(*paths) - return path - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - return self.drive + self.root - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - return self._from_parsed_parts(drv, root, tail[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - def with_name(self, name): - """Return a new path with the file name changed.""" - p = self.parser - if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': - raise ValueError(f"Invalid name {name!r}") - tail = self._tail.copy() - if not tail: - raise ValueError(f"{self!r} has an empty name") - tail[-1] = name - return self._from_parsed_parts(self.drive, self.root, tail) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - for step, path in enumerate(chain([other], other.parents)): - if path == self or path in self.parents: - break - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self._from_parsed_parts('', '', parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - return other == self or other in self.parents - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if self.parser is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - return self.parser.isabs(self) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " - "for removal in Python 3.15. Use os.path.isreserved() to " - "detect reserved paths on Windows.") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - if self.parser is ntpath: - return self.parser.isreserved(self) - return False - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - # The string representation of an empty path is a single dot ('.'). Empty - # paths shouldn't match wildcards, so we change it to the empty string. - path_str = str(self) - return '' if path_str == '.' else path_str - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - parser = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - parser = ntpath - __slots__ = () - - -class Path(_abc.PathBase, PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - as_uri = PurePath.as_uri - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported on this system" - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_text(self, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, read it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.read_text(self, encoding, errors, newline) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.write_text(self, data, encoding, errors, newline) - - _remove_leading_dot = operator.itemgetter(slice(2, None)) - _remove_trailing_slash = operator.itemgetter(slice(-1)) - - def _filter_trailing_slash(self, paths): - sep = self.parser.sep - anchor_len = len(self.anchor) - for path_str in paths: - if len(path_str) > anchor_len and path_str[-1] == sep: - path_str = path_str[:-1] - yield path_str - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - root_dir = str(self) - with os.scandir(root_dir) as scandir_it: - paths = [entry.path for entry in scandir_it] - if root_dir == '.': - paths = map(self._remove_leading_dot, paths) - return map(self._from_parsed_string, paths) - - def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - if pattern.anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - parts = pattern._tail.copy() - if not parts: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - raw = pattern._raw_path - if raw[-1] in (self.parser.sep, self.parser.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - parts.append('') - select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) - root = str(self) - paths = select(root) - - # Normalize results - if root == '.': - paths = map(self._remove_leading_dot, paths) - if parts[-1] == '': - paths = map(self._remove_trailing_slash, paths) - elif parts[-1] == '**': - paths = self._filter_trailing_slash(paths) - paths = map(self._from_parsed_string, paths) - return paths - - def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - pattern = '**' / pattern - return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - root_dir = str(self) - results = self._globber.walk(root_dir, top_down, on_error, follow_symlinks) - for path_str, dirnames, filenames in results: - if root_dir == '.': - path_str = path_str[2:] - yield self._from_parsed_string(path_str), dirnames, filenames - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - if self.root: - drive = os.path.splitroot(os.getcwd())[0] - return self._from_parsed_parts(drive, self.root, self._tail) - if self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - if not self._tail: - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - return self._from_parsed_string(cwd) - drive, root, rel = os.path.splitroot(cwd) - if not rel: - return self._from_parsed_parts(drive, root, self._tail) - tail = rel.split(self.parser.sep) - tail.extend(self._tail) - return self._from_parsed_parts(drive, root, tail) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - uid = self.stat(follow_symlinks=follow_symlinks).st_uid - return pwd.getpwuid(uid).pw_name - - if grp: - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - gid = self.stat(follow_symlinks=follow_symlinks).st_gid - return grp.getgrgid(gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") +__all__ = (_abc.__all__ + + _local.__all__) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 05698d5de24afb..06c10e8e4612ca 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -12,11 +12,13 @@ """ import functools -import glob -import operator +from glob import _Globber, _no_recurse_symlinks from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO + +__all__ = ["UnsupportedOperation"] + # # Internals # @@ -43,30 +45,6 @@ def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' -class Globber(glob._Globber): - lstat = operator.methodcaller('lstat') - add_slash = operator.methodcaller('joinpath', '') - - @staticmethod - def scandir(path): - # Emulate os.scandir(), which returns an object that can be used as a - # context manager. This method is called by walk() and glob(). - from contextlib import nullcontext - return nullcontext(path.iterdir()) - - @staticmethod - def concat_path(path, text): - """Appends text to the given path. - """ - return path.with_segments(path._raw_path + text) - - @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir(). - """ - return entry - - class UnsupportedOperation(NotImplementedError): """An exception that is raised when an unsupported operation is called on a path object. @@ -140,7 +118,7 @@ class PurePathBase: '_resolving', ) parser = ParserBase() - _globber = Globber + _globber = _Globber def __init__(self, path, *paths): self._raw_path = self.parser.join(path, *paths) if paths else path @@ -637,7 +615,7 @@ def samefile(self, other_path): def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ - Open the file pointed by this path and return a file object, as + Open the file pointed to by this path and return a file object, as the built-in open() function does. """ raise UnsupportedOperation(self._unsupported_msg('open()')) @@ -692,7 +670,7 @@ def _glob_selector(self, parts, case_sensitive, recurse_symlinks): # know the case sensitivity of the underlying filesystem, so we # must use scandir() for everything, including non-wildcard parts. case_pedantic = True - recursive = True if recurse_symlinks else glob._no_recurse_symlinks + recursive = True if recurse_symlinks else _no_recurse_symlinks globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) return globber.selector(parts) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 00000000000000..b1e678aceb9ce8 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,859 @@ +import io +import ntpath +import operator +import os +import posixpath +import sys +import warnings +from glob import _StringGlobber +from itertools import chain +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from ._abc import UnsupportedOperation, PurePathBase, PathBase + + +__all__ = [ + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath(PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unnormalized string paths. This is set + # in the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + _globber = _StringGlobber + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is ntpath and self.parser is posixpath: + # GH-103631: Convert separators for backwards compatibility. + paths.extend(path.replace('\\', '/') for path in arg._raw_paths) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + # Avoid calling super().__init__, as an optimisation + self._raw_paths = paths + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] + return drv, root, parsed + + @property + def _raw_path(self): + """The joined but unnormalized path.""" + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self.parser.join(*paths) + return path + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def relative_to(self, other, /, *_deprecated, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other, /, *_deprecated): + """Return True if the path is relative to another path or False. + """ + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path_str = str(self) + return '' if path_str == '.' else path_str + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.read_text(self, encoding, errors, newline) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.write_text(self, data, encoding, errors, newline) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + paths = [entry.path for entry in scandir_it] + if root_dir == '.': + paths = map(self._remove_leading_dot, paths) + return map(self._from_parsed_string, paths) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + if pattern.anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + parts = pattern._tail.copy() + if not parts: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + raw = pattern._raw_path + if raw[-1] in (self.parser.sep, self.parser.altsep): + # GH-65238: pathlib doesn't preserve trailing slash. Add it back. + parts.append('') + select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) + root = str(self) + paths = select(root) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + results = self._globber.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pdb.py b/Lib/pdb.py index d4138b95d3c332..e507a9bb896611 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -77,13 +77,16 @@ import code import glob import token +import types import codeop import pprint import signal import inspect +import textwrap import tokenize import traceback import linecache +import _colorize from contextlib import contextmanager from rlcompleter import Completer @@ -120,7 +123,10 @@ def find_function(funcname, filename): try: fp = tokenize.open(filename) except OSError: - return None + lines = linecache.getlines(filename) + if not lines: + return None + fp = io.StringIO(''.join(lines)) funcdef = "" funcstart = None # consumer of this info expects the first line to be 1 @@ -237,6 +243,44 @@ def namespace(self): ) +class _ZipTarget(_ExecutableTarget): + def __init__(self, target): + import runpy + + self._target = os.path.realpath(target) + sys.path.insert(0, self._target) + try: + _, self._spec, self._code = runpy._get_main_module_details() + except ImportError as e: + print(f"ImportError: {e}") + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + def __repr__(self): + return self._target + + @property + def filename(self): + return self._code.co_filename + + @property + def code(self): + return self._code + + @property + def namespace(self): + return dict( + __name__='__main__', + __file__=os.path.normcase(os.path.abspath(self.filename)), + __package__=self._spec.parent, + __loader__=self._spec.loader, + __spec__=self._spec, + __builtins__=__builtins__, + ) + + class _PdbInteractiveConsole(code.InteractiveConsole): def __init__(self, ns, message): self._message = message @@ -389,6 +433,8 @@ def user_line(self, frame): if self.bp_commands(frame): self.interaction(frame, None) + user_opcode = user_line + def bp_commands(self, frame): """Call every command that was set for the current active breakpoint (if there is one). @@ -580,11 +626,96 @@ def _disable_command_completion(self): self.completenames = completenames return + def _exec_in_closure(self, source, globals, locals): + """ Run source code in closure so code object created within source + can find variables in locals correctly + + returns True if the source is executed, False otherwise + """ + + # Determine if the source should be executed in closure. Only when the + # source compiled to multiple code objects, we should use this feature. + # Otherwise, we can just raise an exception and normal exec will be used. + + code = compile(source, "", "exec") + if not any(isinstance(const, CodeType) for const in code.co_consts): + return False + + # locals could be a proxy which does not support pop + # copy it first to avoid modifying the original locals + locals_copy = dict(locals) + + locals_copy["__pdb_eval__"] = { + "result": None, + "write_back": {} + } + + # If the source is an expression, we need to print its value + try: + compile(source, "", "eval") + except SyntaxError: + pass + else: + source = "__pdb_eval__['result'] = " + source + + # Add write-back to update the locals + source = ("try:\n" + + textwrap.indent(source, " ") + "\n" + + "finally:\n" + + " __pdb_eval__['write_back'] = locals()") + + # Build a closure source code with freevars from locals like: + # def __pdb_outer(): + # var = None + # def __pdb_scope(): # This is the code object we want to execute + # nonlocal var + # + # return __pdb_scope.__code__ + source_with_closure = ("def __pdb_outer():\n" + + "\n".join(f" {var} = None" for var in locals_copy) + "\n" + + " def __pdb_scope():\n" + + "\n".join(f" nonlocal {var}" for var in locals_copy) + "\n" + + textwrap.indent(source, " ") + "\n" + + " return __pdb_scope.__code__" + ) + + # Get the code object of __pdb_scope() + # The exec fills locals_copy with the __pdb_outer() function and we can call + # that to get the code object of __pdb_scope() + ns = {} + try: + exec(source_with_closure, {}, ns) + except Exception: + return False + code = ns["__pdb_outer"]() + + cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) + + try: + exec(code, globals, locals_copy, closure=cells) + except Exception: + return False + + # get the data we need from the statement + pdb_eval = locals_copy["__pdb_eval__"] + + # __pdb_eval__ should not be updated back to locals + pdb_eval["write_back"].pop("__pdb_eval__") + + # Write all local variables back to locals + locals.update(pdb_eval["write_back"]) + eval_result = pdb_eval["result"] + if eval_result is not None: + print(repr(eval_result)) + + return True + def default(self, line): if line[:1] == '!': line = line[1:].strip() locals = self.curframe_locals globals = self.curframe.f_globals try: + buffer = line if (code := codeop.compile_command(line + '\n', '', 'single')) is None: # Multi-line mode with self._disable_command_completion(): @@ -617,7 +748,8 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - exec(code, globals, locals) + if not self._exec_in_closure(buffer, globals, locals): + exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin @@ -1076,7 +1208,7 @@ def lineinfo(self, identifier): if f: fname = f item = parts[1] - answer = find_function(item, fname) + answer = find_function(item, self.canonic(fname)) return answer or failed def checkline(self, filename, lineno): @@ -2007,17 +2139,23 @@ def lookupmodule(self, filename): lookupmodule() translates (possibly incomplete) file or module name into an absolute file name. + + filename could be in format of: + * an absolute path like '/path/to/file.py' + * a relative path like 'file.py' or 'dir/file.py' + * a module name like 'module' or 'package.module' + + files and modules will be searched in sys.path. """ - if os.path.isabs(filename) and os.path.exists(filename): - return filename - f = os.path.join(sys.path[0], filename) - if os.path.exists(f) and self.canonic(f) == self.mainpyfile: - return f - root, ext = os.path.splitext(filename) - if ext == '': - filename = filename + '.py' + if not filename.endswith('.py'): + # A module is passed in so convert it to equivalent file + filename = filename.replace('.', os.sep) + '.py' + if os.path.isabs(filename): - return filename + if os.path.exists(filename): + return filename + return None + for dirname in sys.path: while os.path.islink(dirname): dirname = os.readlink(dirname) @@ -2276,7 +2414,10 @@ def main(): if not opts.args: parser.error("no module or script to run") file = opts.args.pop(0) - target = _ScriptTarget(file) + if file.endswith('.pyz'): + target = _ZipTarget(file) + else: + target = _ScriptTarget(file) sys.argv[:] = [file] + opts.args # Hide "pdb.py" and pdb options from argument list @@ -2297,7 +2438,7 @@ def main(): print("The program exited via sys.exit(). Exit status:", end=' ') print(e) except BaseException as e: - traceback.print_exc() + traceback.print_exception(e, colorize=_colorize.can_colorize()) print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") pdb.interaction(None, e) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 56b7915826daf4..b4547d7893b1cd 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -298,11 +298,8 @@ def expanduser(path): return path if isinstance(path, bytes): userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root + userhome = userhome.rstrip(sep) + return (userhome + path[i:]) or sep # Expand paths containing shell variable substitutions. @@ -532,15 +529,17 @@ def relpath(path, start=None): start = os.fspath(start) try: - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] + start_tail = abspath(start).lstrip(sep) + path_tail = abspath(path).lstrip(sep) + start_list = start_tail.split(sep) if start_tail else [] + path_list = path_tail.split(sep) if path_tail else [] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 02af672a628f5a..55ccf2152c26cb 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -76,6 +76,18 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, plain, escape_less, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -314,7 +326,7 @@ def visiblename(name, all=None, obj=None): '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', '__path__', '__qualname__', '__slots__', '__version__', - '__static_attributes__'}: + '__static_attributes__', '__firstlineno__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 @@ -1640,153 +1652,9 @@ def bold(self, text): def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() + pager = get_pager() pager(text, title) -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - if sys.platform == "emscripten": - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text, title='': tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text, title='': pipepager(plain(text), use_pager, title) - else: - return lambda text, title='': pipepager(text, use_pager, title) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text, title='': tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text, title='': pipepager(text, 'less', title) - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text, title='': pipepager(text, 'more', title) - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def escape_less(s): - return re.sub(r'([?:.%\\])', r'\\\1', s) - -def pipepager(text, cmd, title=''): - """Page through text by feeding it to another program.""" - import subprocess - env = os.environ.copy() - if title: - title += ' ' - esc_title = escape_less(title) - prompt_string = ( - f' {esc_title}' + - '?ltline %lt?L/%L.' - ':byte %bB?s/%s.' - '.' - '?e (END):?pB %pB\\%..' - ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace', env=env) - try: - with proc.stdin as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd, title=''): - """Page through text by invoking a program on a temporary file.""" - import tempfile - with tempfile.TemporaryDirectory() as tempdir: - filename = os.path.join(tempdir, 'pydoc.out') - with open(filename, 'w', errors='backslashreplace', - encoding=os.device_encoding(0) if - sys.platform == 'win32' else None - ) as file: - file.write(text) - os.system(cmd + ' "' + filename + '"') - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text, title=''): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text, title=''): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) - def describe(thing): """Produce a short description of the given thing.""" if inspect.ismodule(thing): diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 26fc498ac95bd7..4643df80e44aaf 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Apr 9 11:53:07 2024 +# Autogenerated by Sphinx on Wed May 8 11:11:17 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -419,7 +419,7 @@ 'async': 'Coroutines\n' '**********\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' '\n' 'Coroutine function definition\n' @@ -792,7 +792,7 @@ 'Changed in version 3.5: "__class__" module attribute is ' 'now writable.\n' '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'Added in version 3.7: "__getattr__" and "__dir__" module ' 'attributes.\n' '\n' 'See also:\n' @@ -1206,7 +1206,7 @@ '\n' ' await_expr ::= "await" primary\n' '\n' - 'New in version 3.5.\n', + 'Added in version 3.5.\n', 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' @@ -1239,7 +1239,7 @@ 'The "@" (at) operator is intended to be used for matrix\n' 'multiplication. No builtin Python types implement this operator.\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' 'The "/" (division) and "//" (floor division) operators yield the\n' 'quotient of their arguments. The numeric arguments are first\n' @@ -2765,7 +2765,7 @@ 'The "match" statement\n' '=====================\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'The match statement is used for pattern matching. Syntax:\n' '\n' @@ -3849,7 +3849,7 @@ 'Coroutines\n' '==========\n' '\n' - 'New in version 3.5.\n' + 'Added in version 3.5.\n' '\n' '\n' 'Coroutine function definition\n' @@ -3976,13 +3976,18 @@ 'Type parameter lists\n' '====================\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' + '\n' + 'Changed in version 3.13: Support for default values was added ' + '(see\n' + '**PEP 696**).\n' '\n' ' type_params ::= "[" type_param ("," type_param)* "]"\n' ' type_param ::= typevar | typevartuple | paramspec\n' - ' typevar ::= identifier (":" expression)?\n' - ' typevartuple ::= "*" identifier\n' - ' paramspec ::= "**" identifier\n' + ' typevar ::= identifier (":" expression)? ("=" ' + 'expression)?\n' + ' typevartuple ::= "*" identifier ("=" expression)?\n' + ' paramspec ::= "**" identifier ("=" expression)?\n' '\n' 'Functions (including coroutines), classes and type aliases may ' 'contain\n' @@ -4081,21 +4086,41 @@ 'bounds or\n' 'constraints.\n' '\n' + 'All three flavors of type parameters can also have a *default ' + 'value*,\n' + 'which is used when the type parameter is not explicitly ' + 'provided. This\n' + 'is added by appending a single equals sign ("=") followed by an\n' + 'expression. Like the bounds and constraints of type variables, ' + 'the\n' + 'default value is not evaluated when the object is created, but ' + 'only\n' + 'when the type parameter’s "__default__" attribute is accessed. ' + 'To this\n' + 'end, the default value is evaluated in a separate annotation ' + 'scope. If\n' + 'no default value is specified for a type parameter, the ' + '"__default__"\n' + 'attribute is set to the special sentinel object ' + '"typing.NoDefault".\n' + '\n' 'The following example indicates the full set of allowed type ' 'parameter\n' 'declarations:\n' '\n' ' def overly_generic[\n' ' SimpleTypeVar,\n' + ' TypeVarWithDefault = int,\n' ' TypeVarWithBound: int,\n' ' TypeVarWithConstraints: (str, bytes),\n' - ' *SimpleTypeVarTuple,\n' - ' **SimpleParamSpec,\n' + ' *SimpleTypeVarTuple = (int, float),\n' + ' **SimpleParamSpec = (str, bytearray),\n' ' ](\n' ' a: SimpleTypeVar,\n' - ' b: TypeVarWithBound,\n' - ' c: Callable[SimpleParamSpec, TypeVarWithConstraints],\n' - ' *d: SimpleTypeVarTuple,\n' + ' b: TypeVarWithDefault,\n' + ' c: TypeVarWithBound,\n' + ' d: Callable[SimpleParamSpec, TypeVarWithConstraints],\n' + ' *e: SimpleTypeVarTuple,\n' ' ): ...\n' '\n' '\n' @@ -4940,8 +4965,8 @@ 'you are\n' 'in debug mode:\n' '\n' - ' > ...(3)double()\n' - ' -> return x * 2\n' + ' > ...(2)double()\n' + ' -> breakpoint()\n' ' (Pdb) p x\n' ' 3\n' ' (Pdb) continue\n' @@ -5063,6 +5088,11 @@ '\n' ' Changed in version 3.7: The keyword-only argument *header*.\n' '\n' + ' Changed in version 3.13: "set_trace()" will enter the ' + 'debugger\n' + ' immediately, rather than on the next line of code to be ' + 'executed.\n' + '\n' 'pdb.post_mortem(traceback=None)\n' '\n' ' Enter post-mortem debugging of the given *traceback* object. ' @@ -5215,7 +5245,7 @@ '* "$_exception": the exception if the frame is raising an ' 'exception\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' 'If a file ".pdbrc" exists in the user’s home directory or in ' 'the\n' @@ -5271,19 +5301,22 @@ '\n' 'b(reak) [([filename:]lineno | function) [, condition]]\n' '\n' - ' With a *lineno* argument, set a break there in the current ' - 'file.\n' + ' With a *lineno* argument, set a break at line *lineno* in ' + 'the\n' + ' current file. The line number may be prefixed with a ' + '*filename* and\n' + ' a colon, to specify a breakpoint in another file (possibly ' + 'one that\n' + ' hasn’t been loaded yet). The file is searched on ' + '"sys.path".\n' + ' Accepatable forms of *filename* are "/abspath/to/file.py",\n' + ' "relpath/file.py", "module" and "package.module".\n' + '\n' ' With a *function* argument, set a break at the first ' 'executable\n' - ' statement within that function. The line number may be ' - 'prefixed\n' - ' with a filename and a colon, to specify a breakpoint in ' - 'another\n' - ' file (probably one that hasn’t been loaded yet). The file ' - 'is\n' - ' searched on "sys.path". Note that each breakpoint is ' - 'assigned a\n' - ' number to which all the other breakpoint commands refer.\n' + ' statement within that function. *function* can be any ' + 'expression\n' + ' that evaluates to a function in the current namespace.\n' '\n' ' If a second argument is present, it is an expression which ' 'must\n' @@ -5295,6 +5328,9 @@ 'current\n' ' ignore count, and the associated condition if any.\n' '\n' + ' Each breakpoint is assigned a number to which all the other\n' + ' breakpoint commands refer.\n' + '\n' 'tbreak [([filename:]lineno | function) [, condition]]\n' '\n' ' Temporary breakpoint, which is removed automatically when it ' @@ -5479,7 +5515,7 @@ ' List all source code for the current function or frame.\n' ' Interesting lines are marked as for "list".\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'a(rgs)\n' '\n' @@ -5512,7 +5548,7 @@ '\n' ' Try to get source code of *expression* and display it.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'display [expression]\n' '\n' @@ -5571,7 +5607,7 @@ ' display lst[:]: [1] [old: []]\n' ' (Pdb)\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'undisplay [expression]\n' '\n' @@ -5580,7 +5616,7 @@ ' *expression*, clear all display expressions for the current ' 'frame.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'interact\n' '\n' @@ -5603,7 +5639,7 @@ ' the mutable objects will be reflected in the original ' 'namespaces.\n' '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' ' Changed in version 3.13: "exit()" and "quit()" can be used to ' 'exit\n' @@ -5748,7 +5784,7 @@ ' > example.py(10)middle()\n' ' -> return inner(0)\n' '\n' - ' New in version 3.13.\n' + ' Added in version 3.13.\n' '\n' '-[ Footnotes ]-\n' '\n' @@ -5813,7 +5849,8 @@ 'dict\n' 'items and earlier dictionary unpackings.\n' '\n' - 'New in version 3.5: Unpacking into dictionary displays, originally\n' + 'Added in version 3.5: Unpacking into dictionary displays, ' + 'originally\n' 'proposed by **PEP 448**.\n' '\n' 'A dict comprehension, in contrast to list and set comprehensions,\n' @@ -6121,9 +6158,12 @@ 'of the module "builtins". The global namespace is searched ' 'first. If\n' 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed ' - 'names.\n' + 'searched\n' + 'next. If the names are also not found in the builtins ' + 'namespace, new\n' + 'variables are created in the global namespace. The global ' + 'statement\n' + 'must precede all uses of the listed names.\n' '\n' 'The "global" statement has the same scope as a name binding ' 'operation\n' @@ -6211,8 +6251,9 @@ 'annotation\n' ' scope, but its decorators are not.\n' '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' + '* The bounds, constraints, and default values for type ' + 'parameters\n' + ' (lazily evaluated).\n' '\n' '* The value of type aliases (lazily evaluated).\n' '\n' @@ -6255,9 +6296,13 @@ 'object were\n' ' defined in the enclosing scope.\n' '\n' - 'New in version 3.12: Annotation scopes were introduced in ' - 'Python 3.12\n' - 'as part of **PEP 695**.\n' + 'Added in version 3.12: Annotation scopes were introduced in ' + 'Python\n' + '3.12 as part of **PEP 695**.\n' + '\n' + 'Changed in version 3.13: Annotation scopes are also used for ' + 'type\n' + 'parameter defaults, as introduced by **PEP 696**.\n' '\n' '\n' 'Lazy evaluation\n' @@ -6265,15 +6310,15 @@ '\n' 'The values of type aliases created through the "type" statement ' 'are\n' - '*lazily evaluated*. The same applies to the bounds and ' - 'constraints of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type ' - 'variable is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' + '*lazily evaluated*. The same applies to the bounds, ' + 'constraints, and\n' + 'default values of type variables created through the type ' + 'parameter\n' + 'syntax. This means that they are not evaluated when the type ' + 'alias or\n' + 'type variable is created. Instead, they are only evaluated when ' + 'doing\n' + 'so is necessary to resolve an attribute access.\n' '\n' 'Example:\n' '\n' @@ -6317,7 +6362,7 @@ 'looked up\n' 'as if they were used in the immediately enclosing scope.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' '\n' 'Builtins and restricted execution\n' @@ -6472,9 +6517,8 @@ 'the\n' 'unpacking.\n' '\n' - 'New in version 3.5: Iterable unpacking in expression lists, ' - 'originally\n' - 'proposed by **PEP 448**.\n' + 'Added in version 3.5: Iterable unpacking in expression lists,\n' + 'originally proposed by **PEP 448**.\n' '\n' 'A trailing comma is required only to create a one-item tuple, ' 'such as\n' @@ -7769,7 +7813,7 @@ 'Soft Keywords\n' '=============\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'Some identifiers are only reserved under specific contexts. ' 'These are\n' @@ -8397,9 +8441,12 @@ 'namespace\n' 'of the module "builtins". The global namespace is searched ' 'first. If\n' - 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed names.\n' + 'the names are not found there, the builtins namespace is searched\n' + 'next. If the names are also not found in the builtins namespace, ' + 'new\n' + 'variables are created in the global namespace. The global ' + 'statement\n' + 'must precede all uses of the listed names.\n' '\n' 'The "global" statement has the same scope as a name binding ' 'operation\n' @@ -8482,8 +8529,8 @@ 'annotation\n' ' scope, but its decorators are not.\n' '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' + '* The bounds, constraints, and default values for type parameters\n' + ' (lazily evaluated).\n' '\n' '* The value of type aliases (lazily evaluated).\n' '\n' @@ -8523,9 +8570,12 @@ 'were\n' ' defined in the enclosing scope.\n' '\n' - 'New in version 3.12: Annotation scopes were introduced in Python ' - '3.12\n' - 'as part of **PEP 695**.\n' + 'Added in version 3.12: Annotation scopes were introduced in ' + 'Python\n' + '3.12 as part of **PEP 695**.\n' + '\n' + 'Changed in version 3.13: Annotation scopes are also used for type\n' + 'parameter defaults, as introduced by **PEP 696**.\n' '\n' '\n' 'Lazy evaluation\n' @@ -8533,15 +8583,15 @@ '\n' 'The values of type aliases created through the "type" statement ' 'are\n' - '*lazily evaluated*. The same applies to the bounds and constraints ' - 'of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type variable ' - 'is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' + '*lazily evaluated*. The same applies to the bounds, constraints, ' + 'and\n' + 'default values of type variables created through the type ' + 'parameter\n' + 'syntax. This means that they are not evaluated when the type alias ' + 'or\n' + 'type variable is created. Instead, they are only evaluated when ' + 'doing\n' + 'so is necessary to resolve an attribute access.\n' '\n' 'Example:\n' '\n' @@ -8584,7 +8634,7 @@ 'looked up\n' 'as if they were used in the immediately enclosing scope.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' '\n' 'Builtins and restricted execution\n' @@ -9503,7 +9553,7 @@ 'for\n' ' correctness.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' 'Note:\n' '\n' @@ -9754,7 +9804,7 @@ 'descriptor, or\n' ' generator instance.\n' '\n' - ' New in version 3.3.\n' + ' Added in version 3.3.\n' '\n' 'definition.__type_params__\n' '\n' @@ -9762,7 +9812,7 @@ 'type\n' ' aliases.\n' '\n' - ' New in version 3.12.\n' + ' Added in version 3.12.\n' '\n' 'class.__mro__\n' '\n' @@ -9788,7 +9838,15 @@ '\n' ' >>> int.__subclasses__()\n' " [, , , " - "]\n", + "]\n" + '\n' + 'class.__static_attributes__\n' + '\n' + ' A tuple containing names of attributes of this class ' + 'which are\n' + ' accessed through "self.X" from any function in its body.\n' + '\n' + ' Added in version 3.13.\n', 'specialnames': 'Special method names\n' '********************\n' '\n' @@ -10471,7 +10529,7 @@ 'Changed in version 3.5: "__class__" module attribute is now ' 'writable.\n' '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'Added in version 3.7: "__getattr__" and "__dir__" module ' 'attributes.\n' '\n' 'See also:\n' @@ -10840,7 +10898,7 @@ 'explicit\n' ' hint) can be accessed as "type(cls)".\n' '\n' - ' New in version 3.6.\n' + ' Added in version 3.6.\n' '\n' 'When a class is created, "type.__new__()" scans the class ' 'variables\n' @@ -10872,7 +10930,7 @@ '\n' ' See Creating the class object for more details.\n' '\n' - ' New in version 3.6.\n' + ' Added in version 3.6.\n' '\n' '\n' 'Metaclasses\n' @@ -11474,7 +11532,7 @@ 'for\n' ' correctness.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' 'Note:\n' '\n' @@ -11923,7 +11981,7 @@ 'will\n' 'raise a "TypeError".\n' '\n' - 'New in version 3.10.\n' + 'Added in version 3.10.\n' '\n' 'See also:\n' '\n' @@ -11972,7 +12030,7 @@ 'to\n' ' implement this method.\n' '\n' - 'New in version 3.12.\n' + 'Added in version 3.12.\n' '\n' 'See also:\n' '\n' @@ -12136,7 +12194,7 @@ '‘Default\n' ' Case Folding’ of the Unicode Standard.\n' '\n' - ' New in version 3.3.\n' + ' Added in version 3.3.\n' '\n' 'str.center(width[, fillchar])\n' '\n' @@ -12320,7 +12378,7 @@ "{country}'.format_map(Default(name='Guido'))\n" " 'Guido was born in country'\n" '\n' - ' New in version 3.2.\n' + ' Added in version 3.2.\n' '\n' 'str.index(sub[, start[, end]])\n' '\n' @@ -12364,7 +12422,7 @@ 'have code\n' ' points in the range U+0000-U+007F.\n' '\n' - ' New in version 3.7.\n' + ' Added in version 3.7.\n' '\n' 'str.isdecimal()\n' '\n' @@ -12604,7 +12662,7 @@ " >>> 'BaseTestCase'.removeprefix('Test')\n" " 'BaseTestCase'\n" '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' 'str.removesuffix(suffix, /)\n' '\n' @@ -12619,7 +12677,7 @@ " >>> 'TmpDirMixin'.removesuffix('Tests')\n" " 'TmpDirMixin'\n" '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' 'str.replace(old, new, count=-1)\n' '\n' @@ -13098,8 +13156,8 @@ 'than\n' 'Python 3.x’s the "\'ur\'" syntax is not supported.\n' '\n' - 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' - 'been\n' + 'Added in version 3.3: The "\'rb\'" prefix of raw bytes literals ' + 'has been\n' 'added as a synonym of "\'br\'".Support for the unicode legacy ' 'literal\n' '("u\'value\'") was reintroduced to simplify the maintenance of ' @@ -14071,7 +14129,7 @@ '| function.__qualname__ | The ' 'function’s *qualified name*. See also: |\n' '| | ' - '"__qualname__ attributes". New in version 3.3. |\n' + '"__qualname__ attributes". Added in version 3.3. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| function.__module__ | The name of ' 'the module the function was defined |\n' @@ -14114,7 +14172,7 @@ '| function.__type_params__ | A "tuple" ' 'containing the type parameters of a |\n' '| | generic ' - 'function. New in version 3.12. |\n' + 'function. Added in version 3.12. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '\n' 'Function objects also support getting and setting arbitrary\n' @@ -14421,8 +14479,7 @@ 'to\n' 'a common ancestor. Additional details on the C3 MRO used by Python ' 'can\n' - 'be found in the documentation accompanying the 2.3 release at\n' - 'https://docs.python.org/3/howto/mro.html.\n' + 'be found at The Python 2.3 Method Resolution Order.\n' '\n' 'When a class attribute reference (for class "C", say) would yield ' 'a\n' @@ -14470,6 +14527,15 @@ ' "__type_params__"\n' ' A tuple containing the type parameters of a generic class.\n' '\n' + ' "__static_attributes__"\n' + ' A tuple containing names of attributes of this class which ' + 'are\n' + ' accessed through "self.X" from any function in its body.\n' + '\n' + ' "__firstlineno__"\n' + ' The line number of the first line of the class definition,\n' + ' including decorators.\n' + '\n' '\n' 'Class instances\n' '===============\n' @@ -14566,9 +14632,9 @@ 'name |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_qualname | The fully ' - 'qualified function name New in version |\n' - '| | ' - '3.11. |\n' + 'qualified function name Added in |\n' + '| | version ' + '3.11. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| codeobject.co_argcount | The total ' 'number of positional *parameters* |\n' @@ -14722,7 +14788,7 @@ ' When this occurs, some or all of the tuple elements can be ' '"None".\n' '\n' - ' New in version 3.11.\n' + ' Added in version 3.11.\n' '\n' ' Note:\n' '\n' @@ -14780,7 +14846,7 @@ 'but\n' ' have been eliminated by the *bytecode* compiler.\n' '\n' - ' New in version 3.10.\n' + ' Added in version 3.10.\n' '\n' ' See also:\n' '\n' @@ -14797,7 +14863,7 @@ ' Code objects are also supported by the generic function\n' ' "copy.replace()".\n' '\n' - ' New in version 3.8.\n' + ' Added in version 3.8.\n' '\n' '\n' 'Frame objects\n' @@ -14830,8 +14896,14 @@ '+----------------------------------------------------+----------------------------------------------------+\n' '| frame.f_locals | The ' 'dictionary used by the frame to look up local |\n' + '| | variables. ' + 'If the frame refers to a function or |\n' '| | ' - 'variables |\n' + 'comprehension, this may return a write- through |\n' + '| | proxy ' + 'object. Changed in version 3.13: Return a |\n' + '| | proxy for ' + 'functions and comprehensions. |\n' '+----------------------------------------------------+----------------------------------------------------+\n' '| frame.f_globals | The ' 'dictionary used by the frame to look up global |\n' @@ -14913,7 +14985,7 @@ ' "RuntimeError" is raised if the frame is currently executing or\n' ' suspended.\n' '\n' - ' New in version 3.4.\n' + ' Added in version 3.4.\n' '\n' ' Changed in version 3.13: Attempting to clear a suspended frame\n' ' raises "RuntimeError" (as has always been the case for ' @@ -15306,7 +15378,7 @@ 'dictionary. This\n' ' is a shortcut for "reversed(d.keys())".\n' '\n' - ' New in version 3.8.\n' + ' Added in version 3.8.\n' '\n' ' setdefault(key[, default])\n' '\n' @@ -15357,7 +15429,7 @@ ' *other* take priority when *d* and *other* share ' 'keys.\n' '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' ' d |= other\n' '\n' @@ -15369,7 +15441,7 @@ 'and *other*\n' ' share keys.\n' '\n' - ' New in version 3.9.\n' + ' Added in version 3.9.\n' '\n' ' Dictionaries compare equal if and only if they have the ' 'same "(key,\n' @@ -15491,7 +15563,7 @@ 'original\n' ' dictionary to which the view refers.\n' '\n' - ' New in version 3.10.\n' + ' Added in version 3.10.\n' '\n' 'Keys views are set-like since their entries are unique and ' '*hashable*.\n' @@ -16031,7 +16103,7 @@ 'mutable\n' ' sequence classes provide it.\n' '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' + ' Added in version 3.3: "clear()" and "copy()" methods.\n' '\n' '6. The value *n* is an integer, or an object implementing\n' ' "__index__()". Zero and negative values of *n* clear the ' @@ -16499,7 +16571,8 @@ 'concrete mutable\n' ' sequence classes provide it.\n' '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' + ' Added in version 3.3: "clear()" and "copy()" ' + 'methods.\n' '\n' '6. The value *n* is an integer, or an object ' 'implementing\n' diff --git a/Lib/random.py b/Lib/random.py index 875beb2f8cf41c..bcc11c7cd3c208 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -996,5 +996,75 @@ def _test(N=10_000): _os.register_at_fork(after_in_child=_inst.seed) +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating point number between 1 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(1, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(1, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + if __name__ == '__main__': - _test() + print(main()) diff --git a/Lib/shutil.py b/Lib/shutil.py index 910d6b6c63ac08..c9b4da34b1e19b 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -556,7 +556,7 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. diff --git a/Lib/site.py b/Lib/site.py index 93af9c453ac7bb..b63447d6673f68 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -485,6 +485,8 @@ def register_readline(): try: import readline import rlcompleter + import _pyrepl.readline + import _pyrepl.unix_console except ImportError: return @@ -513,13 +515,19 @@ def register_readline(): # http://bugs.python.org/issue5845#msg198636 history = gethistoryfile() try: - readline.read_history_file(history) - except OSError: + if os.getenv("PYTHON_BASIC_REPL"): + readline.read_history_file(history) + else: + _pyrepl.readline.read_history_file(history) + except (OSError,* _pyrepl.unix_console._error): pass def write_history(): try: - readline.write_history_file(history) + if os.getenv("PYTHON_BASIC_REPL"): + readline.write_history_file(history) + else: + _pyrepl.readline.write_history_file(history) except (FileNotFoundError, PermissionError): # home directory does not exist or is not writable # https://bugs.python.org/issue19891 diff --git a/Lib/statistics.py b/Lib/statistics.py index fc00891b083dc3..c2f4fe8e054d3d 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -113,6 +113,7 @@ 'geometric_mean', 'harmonic_mean', 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -138,12 +139,13 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf, pi, cos, sin, cosh, atan +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict _SQRT2 = sqrt(2.0) +_random = random # === Exceptions === @@ -978,11 +980,9 @@ def pdf(x): return sum(K((x - x_i) / h) for x_i in data) / (n * h) def cdf(x): - n = len(data) return sum(W((x - x_i) / h) for x_i in data) / n - else: sample = sorted(data) @@ -1078,6 +1078,7 @@ def quantiles(data, *, n=4, method='exclusive'): if ld == 1: return data * (n - 1) raise StatisticsError('must have at least one data point') + if method == 'inclusive': m = ld - 1 result = [] @@ -1086,6 +1087,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n result.append(interpolated) return result + if method == 'exclusive': m = ld + 1 result = [] @@ -1096,6 +1098,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n result.append(interpolated) return result + raise ValueError(f'Unknown method: {method!r}') @@ -1709,3 +1712,96 @@ def __getstate__(self): def __setstate__(self, state): self._mu, self._sigma = state + + +## kde_random() ############################################################## + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p >= 0.004 < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +_quartic_invcdf = _newton_raphson( + f_inv_estimate = _quartic_invcdf_estimate, + f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, + f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) + +def _triweight_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + return x * sign + +_triweight_invcdf = _newton_raphson( + f_inv_estimate = _triweight_invcdf_estimate, + f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, + f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) + +_kernel_invcdfs = { + 'normal': NormalDist().inv_cdf, + 'logistic': lambda p: log(p / (1 - p)), + 'sigmoid': lambda p: log(tan(p * pi/2)), + 'rectangular': lambda p: 2*p - 1, + 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), + 'quartic': _quartic_invcdf, + 'triweight': _triweight_invcdf, + 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), + 'cosine': lambda p: 2 * asin(2*p - 1) / pi, +} +_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] +_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] +_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] +_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_invcdf = _kernel_invcdfs.get(kernel) + if kernel_invcdf is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * kernel_invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 70bdecf2138fd9..98a14e5d3a3187 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -601,10 +601,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a74b61013c4848..46afdfca331a23 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2812,8 +2812,8 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs - gc.collect() # For PyPy or other GCs. time.sleep(DELTA) # let threaded cleanup code run + support.gc_collect() # For PyPy or other GCs. self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. @@ -6161,6 +6161,29 @@ def submain(): pass self.assertFalse(err, msg=err.decode('utf-8')) +class _TestAtExit(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _write_file_at_exit(self, output_path): + import atexit + def exit_handler(): + with open(output_path, 'w') as f: + f.write("deadbeef") + atexit.register(exit_handler) + + def test_atexit(self): + # gh-83856 + with os_helper.temp_dir() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + p = self.Process(target=self._write_file_at_exit, args=(output_path,)) + p.start() + p.join() + with open(output_path) as f: + self.assertEqual(f.read(), 'deadbeef') + + class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in not_exported are excluded diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 15f654302b1793..e2a854663ddb7d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -535,6 +535,15 @@ def suppress_immortalization(suppress=True): finally: _testinternalcapi.set_immortalize_deferred(*old_values) +def skip_if_suppress_immortalization(): + try: + import _testinternalcapi + except ImportError: + return + return unittest.skipUnless(_testinternalcapi.get_immortalize_deferred(), + "requires immortalization of deferred objects") + + MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. @@ -2539,17 +2548,17 @@ def exceeds_recursion_limit(): # Decorator to disable optimizer while a function run def without_optimizer(func): try: - import _testinternalcapi + from _testinternalcapi import get_optimizer, set_optimizer except ImportError: return func @functools.wraps(func) def wrapper(*args, **kwargs): - save_opt = _testinternalcapi.get_optimizer() + save_opt = get_optimizer() try: - _testinternalcapi.set_optimizer(None) + set_optimizer(None) return func(*args, **kwargs) finally: - _testinternalcapi.set_optimizer(save_opt) + set_optimizer(save_opt) return wrapper @@ -2579,20 +2588,21 @@ def copy_python_src_ignore(path, names): } return ignored + def force_not_colorized(func): """Force the terminal not to be colorized.""" @functools.wraps(func) def wrapper(*args, **kwargs): - import traceback - original_fn = traceback._can_colorize + import _colorize + original_fn = _colorize.can_colorize variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} try: for key in variables: variables[key] = os.environ.pop(key, None) - traceback._can_colorize = lambda: False + _colorize.can_colorize = lambda: False return func(*args, **kwargs) finally: - traceback._can_colorize = original_fn + _colorize.can_colorize = original_fn for key, value in variables.items(): if value is not None: os.environ[key] = value diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 8071c248b9b67e..891405943b78c5 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -632,7 +632,8 @@ def fd_count(): if hasattr(os, 'sysconf'): try: MAXFD = os.sysconf("SC_OPEN_MAX") - except OSError: + except (OSError, ValueError): + # gh-118201: ValueError is raised intermittently on iOS pass old_modes = None diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py new file mode 100644 index 00000000000000..d55b97ade68cef --- /dev/null +++ b/Lib/test/test__colorize.py @@ -0,0 +1,59 @@ +import contextlib +import sys +import unittest +import unittest.mock +import _colorize +from test.support import force_not_colorized + +ORIGINAL_CAN_COLORIZE = _colorize.can_colorize + + +def setUpModule(): + _colorize.can_colorize = lambda: False + + +def tearDownModule(): + _colorize.can_colorize = ORIGINAL_CAN_COLORIZE + + +class TestColorizeFunction(unittest.TestCase): + @force_not_colorized + def test_colorized_detection_checks_for_environment_variables(self): + if sys.platform == "win32": + virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", + return_value=True) + else: + virtual_patching = contextlib.nullcontext() + with virtual_patching: + + flags = unittest.mock.MagicMock(ignore_environment=False) + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.flags", flags), + unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE)): + isatty_mock.return_value = True + with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): + self.assertEqual(_colorize.can_colorize(), False) + isatty_mock.return_value = False + with unittest.mock.patch("os.environ", {}): + self.assertEqual(_colorize.can_colorize(), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 00000000000000..115882a4c281f6 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,332 @@ +import platform +import queue +import re +import subprocess +import sys +import unittest +from array import array +from contextlib import contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + + +# Test redirection of stdout and stderr to the Android log. +@unittest.skipIf( + api_level < 23 and platform.machine() == "aarch64", + "SELinux blocks reading logs on older ARM64 emulators" +) +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + Thread(target=logcat_thread).start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + def test_str(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + self.assertEqual("UTF-8", stream.encoding) + self.assertTrue(stream.line_buffering) + self.assertFalse(stream.write_through) + + # stderr is backslashreplace by default; stdout is configured + # that way by libregrtest.main. + self.assertEqual("backslashreplace", stream.errors) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", + [("foobar" * 666) + "foob", # 4000 bytes + "ar" + ("foobar" * 33)]) # 200 bytes + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", + [s * 100, # 3000 bytes + s * 50]) # 1500 bytes + + s = "0123456789" + write(s * 200, []) + write(s * 150, []) + write(s * 51, [s * 350]) # 3500 bytes + write("\n", [s * 51]) # 510 bytes + + def test_bytes(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 617b1721f3dbb1..02b499145f6c43 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4255,6 +4255,140 @@ class TestHelpUsagePositionalsOnlyWrap(HelpTestCase): version = '' +class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): + # https://github.com/python/cpython/issues/62549 + # https://github.com/python/cpython/issues/89743 + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('-n1', metavar='()', help='n1'), + Sig('-o1', metavar='(1, 2)', help='o1'), + Sig('-u1', metavar=' (uu) ', help='u1'), + Sig('-v1', metavar='( vv )', help='v1'), + Sig('-w1', metavar='(w)w', help='w1'), + Sig('-x1', metavar='x(x)', help='x1'), + Sig('-y1', metavar='yy)', help='y1'), + Sig('-z1', metavar='(zz', help='z1'), + Sig('-n2', metavar='[]', help='n2'), + Sig('-o2', metavar='[1, 2]', help='o2'), + Sig('-u2', metavar=' [uu] ', help='u2'), + Sig('-v2', metavar='[ vv ]', help='v2'), + Sig('-w2', metavar='[w]w', help='w2'), + Sig('-x2', metavar='x[x]', help='x2'), + Sig('-y2', metavar='yy]', help='y2'), + Sig('-z2', metavar='[zz', help='z2'), + ] + + usage = '''\ + usage: PROG [-h] [-n1 ()] [-o1 (1, 2)] [-u1 (uu) ] [-v1 ( vv )] [-w1 (w)w] + [-x1 x(x)] [-y1 yy)] [-z1 (zz] [-n2 []] [-o2 [1, 2]] [-u2 [uu] ] + [-v2 [ vv ]] [-w2 [w]w] [-x2 x[x]] [-y2 yy]] [-z2 [zz] + ''' + help = usage + '''\ + + options: + -h, --help show this help message and exit + -n1 () n1 + -o1 (1, 2) o1 + -u1 (uu) u1 + -v1 ( vv ) v1 + -w1 (w)w w1 + -x1 x(x) x1 + -y1 yy) y1 + -z1 (zz z1 + -n2 [] n2 + -o2 [1, 2] o2 + -u2 [uu] u2 + -v2 [ vv ] v2 + -w2 [w]w w2 + -x2 x[x] x2 + -y2 yy] y2 + -z2 [zz z2 + ''' + version = '' + + +class TestHelpUsageNoWhitespaceCrash(TestCase): + + def test_all_suppressed_mutex_followed_by_long_arg(self): + # https://github.com/python/cpython/issues/62090 + # https://github.com/python/cpython/issues/96310 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument('--spam', help=argparse.SUPPRESS) + parser.add_argument('--eggs-eggs-eggs-eggs-eggs-eggs') + usage = textwrap.dedent('''\ + usage: PROG [-h] + [--eggs-eggs-eggs-eggs-eggs-eggs EGGS_EGGS_EGGS_EGGS_EGGS_EGGS] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_newline_in_metavar(self): + # https://github.com/python/cpython/issues/77048 + mapping = ['123456', '12345', '12345', '123'] + parser = argparse.ArgumentParser('11111111111111') + parser.add_argument('-v', '--verbose', + help='verbose mode', action='store_true') + parser.add_argument('targets', + help='installation targets', + nargs='+', + metavar='\n'.join(mapping)) + usage = textwrap.dedent('''\ + usage: 11111111111111 [-h] [-v] + 123456 + 12345 + 12345 + 123 [123456 + 12345 + 12345 + 123 ...] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_empty_metavar_required_arg(self): + # https://github.com/python/cpython/issues/82091 + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('--nil', metavar='', required=True) + parser.add_argument('--a', metavar='A' * 70) + usage = ( + 'usage: PROG [-h] --nil \n' + ' [--a AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAAAAAAAAAAAAAAA]\n' + ) + self.assertEqual(parser.format_usage(), usage) + + def test_all_suppressed_mutex_with_optional_nargs(self): + # https://github.com/python/cpython/issues/98666 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument( + '--param1', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + mutex.add_argument( + '--param2', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + usage = 'usage: PROG [-h]\n' + self.assertEqual(parser.format_usage(), usage) + + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + gg = g.add_mutually_exclusive_group() + gg.add_argument("--hax") + gg.add_argument("--hox", help=argparse.SUPPRESS) + gg.add_argument("--hex") + g.add_argument("--eggs") + parser.add_argument("--num") + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] + [--num NUM] + ''') + self.assertEqual(parser.format_usage(), usage) + + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 68024304dfb746..f6e22d44406d9e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -195,16 +195,19 @@ def to_tuple(t): "type X[T, *Ts, **P] = (T, Ts, P)", "type X[T: int, *Ts, **P] = (T, Ts, P)", "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", # Generic classes "class X[T]: pass", "class X[T, *Ts, **P]: pass", "class X[T: int, *Ts, **P]: pass", "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", # Generic functions "def f[T](): pass", "def f[T, *Ts, **P](): pass", "def f[T: int, *Ts, **P](): pass", "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", ] # These are compiled through "single" @@ -1108,6 +1111,18 @@ def test_type_params_feature_version(self): with self.assertRaises(SyntaxError): ast.parse(sample, feature_version=(3, 11)) + def test_type_params_default_feature_version(self): + samples = [ + "type X[*Ts=int] = int", + "class X[T=int]: pass", + "def f[**P=int](): pass", + ] + for sample in samples: + with self.subTest(sample): + ast.parse(sample) + with self.assertRaises(SyntaxError): + ast.parse(sample, feature_version=(3, 12)) + def test_invalid_major_feature_version(self): with self.assertRaises(ValueError): ast.parse('pass', feature_version=(2, 7)) @@ -3021,7 +3036,7 @@ def test_FunctionDef(self): self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) - def test_custom_subclass(self): + def test_custom_subclass_with_no_fields(self): class NoInit(ast.AST): pass @@ -3029,17 +3044,17 @@ class NoInit(ast.AST): self.assertIsInstance(obj, NoInit) self.assertEqual(obj.__dict__, {}) + def test_fields_but_no_field_types(self): class Fields(ast.AST): _fields = ('a',) - with self.assertWarnsRegex(DeprecationWarning, - r"Fields provides _fields but not _field_types."): - obj = Fields() + obj = Fields() with self.assertRaises(AttributeError): obj.a obj = Fields(a=1) self.assertEqual(obj.a, 1) + def test_fields_and_types(self): class FieldsAndTypes(ast.AST): _fields = ('a',) _field_types = {'a': int | None} @@ -3050,6 +3065,7 @@ class FieldsAndTypes(ast.AST): obj = FieldsAndTypes(a=1) self.assertEqual(obj.a, 1) + def test_fields_and_types_no_default(self): class FieldsAndTypesNoDefault(ast.AST): _fields = ('a',) _field_types = {'a': int} @@ -3062,6 +3078,38 @@ class FieldsAndTypesNoDefault(ast.AST): obj = FieldsAndTypesNoDefault(a=1) self.assertEqual(obj.a, 1) + def test_incomplete_field_types(self): + class MoreFieldsThanTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None} + a: int | None = None + b: int | None = None + + with self.assertWarnsRegex( + DeprecationWarning, + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" + ): + obj = MoreFieldsThanTypes() + self.assertIs(obj.a, None) + self.assertIs(obj.b, None) + + obj = MoreFieldsThanTypes(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + def test_complete_field_types(self): + class _AllFieldTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} + # This must be set explicitly + a: int | None = None + # This will add an implicit empty list default + b: list[str] + + obj = _AllFieldTypes() + self.assertIs(obj.a, None) + self.assertEqual(obj.b, []) + @support.cpython_only class ModuleStateTests(unittest.TestCase): @@ -3261,18 +3309,21 @@ def main(): ('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), ('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts'), ('ParamSpec', (1, 15, 1, 18), 'P')], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',))), ('TypeVarTuple', (1, 15, 1, 18), 'Ts'), ('ParamSpec', (1, 20, 1, 23), 'P')], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 22, 1, 25), 'Ts'), ('ParamSpec', (1, 27, 1, 30), 'P')], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts'), ('ParamSpec', (1, 16, 1, 19), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',))), ('TypeVarTuple', (1, 16, 1, 19), 'Ts'), ('ParamSpec', (1, 21, 1, 24), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 23, 1, 26), 'Ts'), ('ParamSpec', (1, 28, 1, 31), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts'), ('ParamSpec', (1, 14, 1, 17), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',))), ('TypeVarTuple', (1, 14, 1, 17), 'Ts'), ('ParamSpec', (1, 19, 1, 22), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 21, 1, 24), 'Ts'), ('ParamSpec', (1, 26, 1, 29), 'P')])], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), ] single_results = [ ('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index a1e9e1b89c6a2c..4f2278bb263681 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -393,6 +393,151 @@ async def gen(): r'anext\(\): asynchronous generator is already running'): an.__next__() + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + def test_async_gen_3_arg_deprecation_warning(self): async def gen(): yield 123 @@ -426,6 +571,54 @@ async def gen(): self.assertTrue(inspect.isawaitable(aclose)) aclose.close() + def test_async_gen_asend_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + class AsyncGenAsyncioTest(unittest.TestCase): @@ -1571,6 +1764,8 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1624,6 +1819,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1809,9 +2005,56 @@ class MyException(Exception): g = gen() with self.assertRaises(MyException): g.aclose().throw(MyException) - del g - gc_collect() + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 568c88e326c087..ed1a63daea1186 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -610,6 +614,15 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 82dea8a6d731ea..1f3b6746ce4a62 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -139,13 +139,21 @@ def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + # Test excess data exceptions assertExcessData(b'ab==a', b'i') assertExcessData(b'ab===', b'i') + assertExcessData(b'ab====', b'i') assertExcessData(b'ab==:', b'i') assertExcessData(b'abc=a', b'i\xb7') assertExcessData(b'abc=:', b'i\xb7') assertExcessData(b'ab==\n', b'i') + assertExcessData(b'abc==', b'i\xb7') + assertExcessData(b'abc===', b'i\xb7') + assertExcessData(b'abc====', b'i\xb7') + assertExcessData(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -157,8 +165,15 @@ def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') + assertLeadingPadding(b'====', b'') + assertLeadingPadding(b'=====', b'') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') def test_base64errors(self): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9a0bf524e3943f..230789f29ff788 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -46,6 +46,8 @@ x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) +# used as proof of globals being used +A_GLOBAL_VALUE = 123 class Squares: @@ -684,6 +686,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -777,6 +784,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index d3f4d6c29c5536..7ea27929138da3 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -46,11 +46,16 @@ def test_frames_are_popped_after_failed_calls(self): # recovering from failed calls: def f(): pass - for _ in range(1000): - try: - f(None) - except TypeError: + class C: + def m(self): pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass # BOOM! diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index c798b343626677..0491ff9b84d486 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -34,6 +34,8 @@ def clear_executors(func): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") class TestOptimizerAPI(unittest.TestCase): def test_new_counter_optimizer_dealloc(self): @@ -136,6 +138,8 @@ def get_opnames(ex): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): def setUp(self): @@ -215,6 +219,8 @@ def f(): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): @@ -579,6 +585,8 @@ def testfunc(n): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): @@ -1313,5 +1321,18 @@ def testfunc(n): self.assertIsNotNone(ex) self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) + def test_modified_local_is_seen_by_optimized_code(self): + l = sys._getframe().f_locals + a = 1 + s = 0 + for j in range(1 << 10): + a + a + l["xa"[j >> 9]] = 1.0 + s += a + self.assertIs(type(a), float) + self.assertIs(type(s), float) + self.assertEqual(s, 1024.0) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py index 3a390273ec21ad..894f66b437a39c 100644 --- a/Lib/test/test_capi/test_run.py +++ b/Lib/test/test_capi/test_run.py @@ -11,6 +11,10 @@ Py_eval_input = _testcapi.Py_eval_input +class DictSubclass(dict): + pass + + class CAPITest(unittest.TestCase): # TODO: Test the following functions: # @@ -50,15 +54,19 @@ def run(s, *args): self.assertRaises(TypeError, run, b'a\n', dict(a=1), []) self.assertRaises(TypeError, run, b'a\n', dict(a=1), 1) + self.assertIsNone(run(b'a\n', DictSubclass(a=1))) + self.assertIsNone(run(b'a\n', DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, b'a\n', DictSubclass()) + self.assertIsNone(run(b'\xc3\xa4\n', {'\xe4': 1})) self.assertRaises(SyntaxError, run, b'\xe4\n', {}) - # CRASHES run(b'a\n', NULL) - # CRASHES run(b'a\n', NULL, {}) - # CRASHES run(b'a\n', NULL, dict(a=1)) - # CRASHES run(b'a\n', UserDict()) - # CRASHES run(b'a\n', UserDict(), {}) - # CRASHES run(b'a\n', UserDict(), dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', NULL) + self.assertRaises(SystemError, run, b'a\n', NULL, {}) + self.assertRaises(SystemError, run, b'a\n', NULL, dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', UserDict()) + self.assertRaises(SystemError, run, b'a\n', UserDict(), {}) + self.assertRaises(SystemError, run, b'a\n', UserDict(), dict(a=1)) # CRASHES run(NULL, {}) @@ -82,12 +90,16 @@ def run(*args): self.assertRaises(TypeError, run, dict(a=1), []) self.assertRaises(TypeError, run, dict(a=1), 1) - # CRASHES run(NULL) - # CRASHES run(NULL, {}) - # CRASHES run(NULL, dict(a=1)) - # CRASHES run(UserDict()) - # CRASHES run(UserDict(), {}) - # CRASHES run(UserDict(), dict(a=1)) + self.assertIsNone(run(DictSubclass(a=1))) + self.assertIsNone(run(DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, DictSubclass()) + + self.assertRaises(SystemError, run, NULL) + self.assertRaises(SystemError, run, NULL, {}) + self.assertRaises(SystemError, run, NULL, dict(a=1)) + self.assertRaises(SystemError, run, UserDict()) + self.assertRaises(SystemError, run, UserDict(), {}) + self.assertRaises(SystemError, run, UserDict(), dict(a=1)) @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') @unittest.skipIf(os.name == 'nt', 'does not work on Windows') diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py index 10b7fbf2c372a3..17ebd7c1962d48 100644 --- a/Lib/test/test_capi/test_time.py +++ b/Lib/test/test_capi/test_time.py @@ -18,11 +18,6 @@ def test_min_max(self): self.assertEqual(PyTime_MIN, -2**63) self.assertEqual(PyTime_MAX, 2**63 - 1) - def check_clock(self, c_func, py_func): - t1 = c_func() - t2 = py_func() - self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) - def test_assecondsdouble(self): # Test PyTime_AsSecondsDouble() def ns_to_sec(ns): @@ -58,14 +53,22 @@ def ns_to_sec(ns): self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), ns_to_sec(ns)) + def check_clock(self, c_func, py_func): + t1 = c_func() + t2 = py_func() + self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) + def test_monotonic(self): - # Test PyTime_Monotonic() + # Test PyTime_Monotonic() and PyTime_MonotonicRaw() self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) + self.check_clock(_testcapi.PyTime_MonotonicRaw, time.monotonic) def test_perf_counter(self): - # Test PyTime_PerfCounter() + # Test PyTime_PerfCounter() and PyTime_PerfCounterRaw() self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) + self.check_clock(_testcapi.PyTime_PerfCounterRaw, time.perf_counter) def test_time(self): - # Test PyTime_time() + # Test PyTime_Time() and PyTime_TimeRaw() self.check_clock(_testcapi.PyTime_Time, time.time) + self.check_clock(_testcapi.PyTime_TimeRaw, time.time) diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 662abd4005e8d2..eb23dbe20353ba 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -44,6 +44,11 @@ _testcext_exec(PyObject *module) return -1; } #endif + + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 5885db84b66b01..655d53b8d5bb6a 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -882,6 +882,25 @@ class Foo: f.a = 3 self.assertEqual(f.a, 3) + def test_store_attr_type_cache(self): + """Verifies that the type cache doesn't provide a value which is + inconsistent from the dict.""" + class X: + def __del__(inner_self): + v = C.a + self.assertEqual(v, C.__dict__['a']) + + class C: + a = X() + + # prime the cache + C.a + C.a + + # destructor shouldn't be able to see inconsisent state + C.a = X() + C.a = X() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index aa793f56225393..059fab1e2ecc08 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -142,7 +142,8 @@ from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, gc_collect, Py_GIL_DISABLED, - suppress_immortalization) + suppress_immortalization, + skip_if_suppress_immortalization) from test.support.script_helper import assert_python_ok from test.support import threading_helper, import_helper from test.support.bytecode_helper import instructions_with_positions @@ -570,11 +571,31 @@ def f(a='str_value'): self.assertIsInterned(f()) @cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "free-threaded build interns all string constants") def test_interned_string_with_null(self): co = compile(r'res = "str\0value!"', '?', 'exec') v = self.find_const(co.co_consts, 'str\0value!') self.assertIsNotInterned(v) + @cpython_only + @unittest.skipUnless(Py_GIL_DISABLED, "does not intern all constants") + @skip_if_suppress_immortalization() + def test_interned_constants(self): + # compile separately to avoid compile time de-duping + + globals = {} + exec(textwrap.dedent(""" + def func1(): + return (0.0, (1, 2, "hello")) + """), globals) + + exec(textwrap.dedent(""" + def func2(): + return (0.0, (1, 2, "hello")) + """), globals) + + self.assertTrue(globals["func1"]() is globals["func2"]()) + class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 484d72e63411a8..1f4368b15f473c 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1958,7 +1958,10 @@ def test_column_offset_deduplication(self): def test_load_super_attr(self): source = "class C:\n def __init__(self):\n super().__init__()" - code = compile(source, "", "exec").co_consts[0].co_consts[1] + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break self.assertOpcodeSourcePositionIs( code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 ) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 14c2af19e3eb28..bf0fd051672db4 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -502,19 +502,25 @@ def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') with open(self.source_path, 'w', encoding='utf-8') as file: - file.write('# -*- coding: utf-8 -*-\n') - file.write('print u"\u20ac"\n') + # Intentional syntax error: bytes can only contain + # ASCII literal characters. + file.write('b"\u20ac"') def tearDown(self): shutil.rmtree(self.directory) def test_error(self): - try: - orig_stdout = sys.stdout - sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii') - compileall.compile_dir(self.directory) - finally: - sys.stdout = orig_stdout + buffer = io.TextIOWrapper(io.BytesIO(), encoding='ascii') + with contextlib.redirect_stdout(buffer): + compiled = compileall.compile_dir(self.directory) + self.assertFalse(compiled) # should not be successful + buffer.seek(0) + res = buffer.read() + self.assertIn( + 'SyntaxError: bytes can only contain ASCII literal characters', + res, + ) + self.assertNotIn('UnicodeEncodeError', res) class CommandLineTestsBase: diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index a569c2251d1ad7..ab485b629b7788 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -225,6 +225,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } diff --git a/Lib/test/test_ctypes/test_internals.py b/Lib/test/test_ctypes/test_internals.py index 94c9a86c2d06df..778da6573da975 100644 --- a/Lib/test/test_ctypes/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -28,7 +28,7 @@ def test_ints(self): self.assertEqual(ci._objects, None) def test_c_char_p(self): - s = b"Hello, World" + s = "Hello, World".encode("ascii") refcnt = sys.getrefcount(s) cs = c_char_p(s) self.assertEqual(refcnt + 1, sys.getrefcount(s)) diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 77da35855928a4..1072a109833261 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -47,7 +47,7 @@ def test_PyLong_Long(self): @support.refcount_test def test_PyObj_FromPtr(self): - s = "abc def ghi jkl" + s = object() ref = sys.getrefcount(s) # id(python-object) is the address pyobj = _ctypes.PyObj_FromPtr(id(s)) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 7010c34792e093..e927e24b582a5d 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4716,9 +4716,33 @@ def test_py_exact_power(self): c.prec = 1 x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('3e-6')) + c.prec = 2 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.6e-6')) + c.prec = 3 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 28 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 201 x = Decimal(2**578) ** Decimal("-0.5") + # See https://github.com/python/cpython/issues/118027 + # Testing for an exact power could appear to hang, in the Python + # version, as it attempted to compute 10**(MAX_EMAX + 1). + # Fixed via https://github.com/python/cpython/pull/118503. + c.prec = P.MAX_PREC + c.Emax = P.MAX_EMAX + c.Emin = P.MIN_EMIN + c.traps[P.Inexact] = 1 + D2 = Decimal(2) + # If the bug is still present, the next statement won't complete. + res = D2 ** 117 + self.assertEqual(res, 1 << 117) + def test_py_immutability_operations(self): # Do operations and check that it didn't change internal objects. Decimal = P.Decimal @@ -5705,7 +5729,6 @@ def test_format_fallback_rounding(self): with C.localcontext(rounding=C.ROUND_DOWN): self.assertEqual(format(y, '#.1f'), '6.0') - @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 93f66a721e8108..18144c8cbb2f0a 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5088,7 +5088,8 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', '__static_attributes__', '__weakref__', 'meth']) @@ -5099,7 +5100,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 6) + self.assertEqual(len(values), 7) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5109,7 +5110,8 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', '__static_attributes__', '__weakref__', 'meth']) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 747a73829fa705..b68ed3baadc652 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -840,7 +840,7 @@ def loop_test(): %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) - CALL_PY_WITH_DEFAULTS 1 + CALL_PY_GENERAL 1 POP_TOP JUMP_BACKWARD 16 (to L1) diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 0f1e584e22a888..286c3ecfbc9239 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -16,7 +16,7 @@ import tempfile import types import contextlib -import traceback +import _colorize def doctest_skip_if(condition): @@ -893,8 +893,8 @@ def basics(): r""" DocTestRunner is used to run DocTest test cases, and to accumulate statistics. Here's a simple DocTest case we can use: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... ''' @@ -951,7 +951,7 @@ def basics(): r""" ok TestResults(failed=1, attempted=3) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def verbose_flag(): r""" The `verbose` flag makes the test runner generate more detailed @@ -1027,8 +1027,8 @@ def exceptions(): r""" lines between the first line and the type/value may be omitted or replaced with any other string: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... ''' @@ -1261,7 +1261,7 @@ def exceptions(): r""" ZeroDivisionError: integer division or modulo by zero TestResults(failed=1, attempted=1) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def displayhook(): r""" Test that changing sys.displayhook doesn't matter for doctest. @@ -1303,8 +1303,8 @@ def optionflags(): r""" The DONT_ACCEPT_TRUE_FOR_1 flag disables matches between True/False and 1/0: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... '>>> True\n1\n' @@ -1725,7 +1725,7 @@ def optionflags(): r""" Clean up. >>> del doctest.OPTIONFLAGS_BY_NAME[unlikely] - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ @@ -1736,8 +1736,8 @@ def option_directives(): r""" single example. To turn an option on for an example, follow that example with a comment of the form ``# doctest: +OPTION``: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): r''' ... >>> print(list(range(10))) # should fail: no ellipsis @@ -1947,7 +1947,7 @@ def option_directives(): r""" Traceback (most recent call last): ValueError: line 0 of the doctest for s has an option directive on a line with no example: '# doctest: +ELLIPSIS' - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def test_testsource(): r""" @@ -2031,8 +2031,8 @@ def test_pdb_set_trace(): with a version that restores stdout. This is necessary for you to see debugger output. - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> doc = ''' ... >>> x = 42 @@ -2056,8 +2056,7 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin - --Return-- - > (1)()->None + > (1)() -> import pdb; pdb.set_trace() (Pdb) print(x) 42 @@ -2087,8 +2086,7 @@ def test_pdb_set_trace(): ... runner.run(test) ... finally: ... sys.stdin = real_stdin - --Return-- - > (3)calls_set_trace()->None + > (3)calls_set_trace() -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2114,6 +2112,7 @@ def test_pdb_set_trace(): >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', # return event of g ... 'list', # list source from example 2 ... 'next', # return from g() ... 'list', # list source from example 1 @@ -2124,6 +2123,9 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin ... # doctest: +NORMALIZE_WHITESPACE + > (3)g() + -> import pdb; pdb.set_trace() + (Pdb) step --Return-- > (3)g()->None -> import pdb; pdb.set_trace() @@ -2157,7 +2159,7 @@ def test_pdb_set_trace(): 9 TestResults(failed=1, attempted=3) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def test_pdb_set_trace_nested(): @@ -2188,6 +2190,7 @@ def test_pdb_set_trace_nested(): >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', ... 'print(y)', # print data defined in the function ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', ... 'up', 'print(x)', @@ -2201,6 +2204,9 @@ def test_pdb_set_trace_nested(): ... finally: ... sys.stdin = real_stdin ... # doctest: +REPORT_NDIFF + > (4)calls_set_trace() + -> import pdb; pdb.set_trace() + (Pdb) step > (5)calls_set_trace() -> self.f1() (Pdb) print(y) @@ -2694,8 +2700,8 @@ def test_testfile(): r""" We don't want color or `-v` in sys.argv for these tests. - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> save_argv = sys.argv >>> if '-v' in sys.argv: @@ -2863,7 +2869,7 @@ def test_testfile(): r""" TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. >>> sys.argv = save_argv - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): @@ -3001,8 +3007,8 @@ def test_testmod(): r""" def test_unicode(): """ Check doctest with a non-ascii filename: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> doc = ''' ... >>> raise Exception('clé') @@ -3030,7 +3036,7 @@ def test_unicode(): """ Exception: clé TestResults(failed=1, attempted=1) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ @@ -3325,8 +3331,8 @@ def test_run_doctestsuite_multiple_times(): def test_exception_with_note(note): """ - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> test_exception_with_note('Note') Traceback (most recent call last): @@ -3378,7 +3384,7 @@ def test_exception_with_note(note): note TestResults(failed=1, attempted=...) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ exc = ValueError('Text') exc.add_note(note) @@ -3459,8 +3465,8 @@ def test_syntax_error_subclass_from_stdlib(): def test_syntax_error_with_incorrect_expected_note(): """ - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... r''' @@ -3491,7 +3497,7 @@ def test_syntax_error_with_incorrect_expected_note(): note2 TestResults(failed=1, attempted=...) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 0aa3be6a1bde6a..21bece26b893c6 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import swap_item, swap_attr +from test.support import swap_item, swap_attr, is_wasi, Py_DEBUG class RebindBuiltinsTests(unittest.TestCase): @@ -134,6 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 8e744a1223e86f..212255374bddd1 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,3 +1,4 @@ +import copy import gc import operator import re @@ -13,7 +14,7 @@ _testcapi = None from test import support -from test.support import threading_helper, Py_GIL_DISABLED +from test.support import import_helper, threading_helper, Py_GIL_DISABLED from test.support.script_helper import assert_python_ok @@ -198,14 +199,6 @@ def inner(): tb = tb.tb_next return frames - def test_locals(self): - f, outer, inner = self.make_frames() - outer_locals = outer.f_locals - self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) - self.assertEqual(outer_locals, {'x': 5, 'y': 6}) - inner_locals = inner.f_locals - self.assertEqual(inner_locals, {'x': 5, 'z': 7}) - def test_clear_locals(self): # Test f_locals after clear() (issue #21897) f, outer, inner = self.make_frames() @@ -217,8 +210,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) @@ -269,6 +262,192 @@ def inner(): r"^$" % (file_repr, offset + 5)) +class TestFrameLocals(unittest.TestCase): + def test_scope(self): + class A: + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + + self.assertEqual(A.x, 2) + self.assertEqual(A.y, 2) + + def f(): + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + self.assertEqual(x, 2) + self.assertEqual(locals()['y'], 2) + f() + + def test_closure(self): + x = 1 + y = 2 + + def f(): + z = x + y + d = sys._getframe().f_locals + self.assertEqual(d['x'], 1) + self.assertEqual(d['y'], 2) + d['x'] = 2 + d['y'] = 3 + + f() + self.assertEqual(x, 2) + self.assertEqual(y, 3) + + def test_as_dict(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + # self, x, y, d + self.assertEqual(len(d), 4) + self.assertIs(d['d'], d) + self.assertEqual(set(d.keys()), set(['x', 'y', 'd', 'self'])) + self.assertEqual(len(d.values()), 4) + self.assertIn(1, d.values()) + self.assertEqual(len(d.items()), 4) + self.assertIn(('x', 1), d.items()) + self.assertEqual(d.__getitem__('x'), 1) + d.__setitem__('x', 2) + self.assertEqual(d['x'], 2) + self.assertEqual(d.get('x'), 2) + self.assertIs(d.get('non_exist', None), None) + self.assertEqual(d.__len__(), 4) + self.assertEqual(set([key for key in d]), set(['x', 'y', 'd', 'self'])) + self.assertIn('x', d) + self.assertTrue(d.__contains__('x')) + + self.assertEqual(reversed(d), list(reversed(d.keys()))) + + d.update({'x': 3, 'z': 4}) + self.assertEqual(d['x'], 3) + self.assertEqual(d['z'], 4) + + with self.assertRaises(TypeError): + d.update([1, 2]) + + self.assertEqual(d.setdefault('x', 5), 3) + self.assertEqual(d.setdefault('new', 5), 5) + self.assertEqual(d['new'], 5) + + with self.assertRaises(KeyError): + d['non_exist'] + + def test_as_number(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + self.assertIn('z', d | {'z': 3}) + d |= {'z': 3} + self.assertEqual(d['z'], 3) + d |= {'y': 3} + self.assertEqual(d['y'], 3) + with self.assertRaises(TypeError): + d |= 3 + with self.assertRaises(TypeError): + _ = d | [3] + + def test_non_string_key(self): + d = sys._getframe().f_locals + d[1] = 2 + self.assertEqual(d[1], 2) + + def test_write_with_hidden(self): + def f(): + f_locals = [sys._getframe().f_locals for b in [0]][0] + f_locals['b'] = 2 + f_locals['c'] = 3 + self.assertEqual(b, 2) + self.assertEqual(c, 3) + b = 0 + c = 0 + f() + + def test_local_objects(self): + o = object() + k = '.'.join(['a', 'b', 'c']) + f_locals = sys._getframe().f_locals + f_locals['o'] = f_locals['k'] + self.assertEqual(o, 'a.b.c') + + def test_update_with_self(self): + def f(): + f_locals = sys._getframe().f_locals + f_locals.update(f_locals) + f_locals.update(f_locals) + f_locals.update(f_locals) + f() + + def test_repr(self): + x = 1 + # Introduce a reference cycle + frame = sys._getframe() + self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) + + def test_delete(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + del d['x'] + + with self.assertRaises(AttributeError): + d.clear() + + with self.assertRaises(AttributeError): + d.pop('x') + + @support.cpython_only + def test_sizeof(self): + proxy = sys._getframe().f_locals + support.check_sizeof(self, proxy, support.calcobjsize("P")) + + def test_unsupport(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(AttributeError): + d.copy() + + with self.assertRaises(TypeError): + copy.copy(d) + + with self.assertRaises(TypeError): + copy.deepcopy(d) + + +class TestFrameCApi(unittest.TestCase): + def test_basic(self): + x = 1 + ctypes = import_helper.import_module('ctypes') + PyEval_GetFrameLocals = ctypes.pythonapi.PyEval_GetFrameLocals + PyEval_GetFrameLocals.restype = ctypes.py_object + frame_locals = PyEval_GetFrameLocals() + self.assertTrue(type(frame_locals), dict) + self.assertEqual(frame_locals['x'], 1) + frame_locals['x'] = 2 + self.assertEqual(x, 1) + + PyEval_GetFrameGlobals = ctypes.pythonapi.PyEval_GetFrameGlobals + PyEval_GetFrameGlobals.restype = ctypes.py_object + frame_globals = PyEval_GetFrameGlobals() + self.assertTrue(type(frame_globals), dict) + self.assertIs(frame_globals, globals()) + + PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins + PyEval_GetFrameBuiltins.restype = ctypes.py_object + frame_builtins = PyEval_GetFrameBuiltins() + self.assertEqual(frame_builtins, __builtins__) + + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocals.argtypes = [ctypes.py_object] + PyFrame_GetLocals.restype = ctypes.py_object + frame = sys._getframe() + f_locals = PyFrame_GetLocals(frame) + self.assertTrue(f_locals['x'], 1) + f_locals['x'] = 2 + self.assertEqual(x, 2) + + class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py new file mode 100644 index 00000000000000..f877582e6b565c --- /dev/null +++ b/Lib/test/test_free_threading/test_dict.py @@ -0,0 +1,177 @@ +import gc +import time +import unittest +import weakref + +from ast import Or +from functools import partial +from threading import Thread +from unittest import TestCase + +from _testcapi import dict_version + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestDict(TestCase): + def test_racing_creation_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with shared keys""" + class C(int): + pass + + self.racing_creation(C) + + def test_racing_creation_no_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with an ordinary dict""" + self.racing_creation(Or) + + def test_racing_creation_inline_values_invalid(self): + """Verify that re-creating a dict after we have invalid inline values + is thread safe""" + class C: + pass + + def make_obj(): + a = C() + # Make object, make inline values invalid, and then delete dict + a.__dict__ = {} + del a.__dict__ + return a + + self.racing_creation(make_obj) + + def test_racing_creation_nonmanaged_dict(self): + """Verify that explicit creation of an unmanaged dict is thread safe + outside of the normal attribute setting code path""" + def make_obj(): + def f(): pass + return f + + def set(func, name, val): + # Force creation of the dict via PyObject_GenericGetDict + func.__dict__[name] = val + + self.racing_creation(make_obj, set) + + def racing_creation(self, cls, set=setattr): + objects = [] + processed = [] + + OBJECT_COUNT = 100 + THREAD_COUNT = 10 + CUR = 0 + + for i in range(OBJECT_COUNT): + objects.append(cls()) + + def writer_func(name): + last = -1 + while True: + if CUR == last: + continue + elif CUR == OBJECT_COUNT: + break + + obj = objects[CUR] + set(obj, name, name) + last = CUR + processed.append(name) + + writers = [] + for x in range(THREAD_COUNT): + writer = Thread(target=partial(writer_func, f"a{x:02}")) + writers.append(writer) + writer.start() + + for i in range(OBJECT_COUNT): + CUR = i + while len(processed) != THREAD_COUNT: + time.sleep(0.001) + processed.clear() + + CUR = OBJECT_COUNT + + for writer in writers: + writer.join() + + for obj_idx, obj in enumerate(objects): + assert ( + len(obj.__dict__) == THREAD_COUNT + ), f"{len(obj.__dict__)} {obj.__dict__!r} {obj_idx}" + for i in range(THREAD_COUNT): + assert f"a{i:02}" in obj.__dict__, f"a{i:02} missing at {obj_idx}" + + def test_racing_set_dict(self): + """Races assigning to __dict__ should be thread safe""" + + def f(): pass + l = [] + THREAD_COUNT = 10 + class MyDict(dict): pass + + def writer_func(l): + for i in range(1000): + d = MyDict() + l.append(weakref.ref(d)) + f.__dict__ = d + + lists = [] + writers = [] + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + f.__dict__ = {} + gc.collect() + + for thread_list in lists: + for ref in thread_list: + self.assertIsNone(ref()) + + def test_dict_version(self): + THREAD_COUNT = 10 + DICT_COUNT = 10000 + lists = [] + writers = [] + + def writer_func(thread_list): + for i in range(DICT_COUNT): + thread_list.append(dict_version({})) + + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + total_len = 0 + values = set() + for thread_list in lists: + for v in thread_list: + if v in values: + print('dup', v, (v/4096)%256) + values.add(v) + total_len += len(thread_list) + versions = set(dict_version for thread_list in lists for dict_version in thread_list) + self.assertEqual(len(versions), THREAD_COUNT*DICT_COUNT) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py new file mode 100644 index 00000000000000..6ad806d67a80ed --- /dev/null +++ b/Lib/test/test_free_threading/test_list.py @@ -0,0 +1,80 @@ +import unittest + +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + +class C: + def __init__(self, v): + self.v = v + + +@threading_helper.requires_working_threading() +class TestList(TestCase): + def test_racing_iter_append(self): + + l = [] + OBJECT_COUNT = 10000 + + def writer_func(): + for i in range(OBJECT_COUNT): + l.append(C(i + OBJECT_COUNT)) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_iter_extend(self): + iters = [ + lambda x: [x], + ] + for iter_case in iters: + with self.subTest(iter=iter_case): + l = [] + OBJECT_COUNT = 10000 + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend(iter_case(C(i + OBJECT_COUNT))) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index e170840ca3cb27..3a3f1ba3b605d6 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -7,21 +7,15 @@ import weakref from sys import monitoring -from test.support import is_wasi -from threading import Thread +from test.support import threading_helper +from threading import Thread, _PyRLock from unittest import TestCase class InstrumentationMultiThreadedMixin: - if not hasattr(sys, "gettotalrefcount"): - thread_count = 50 - func_count = 1000 - fib = 15 - else: - # Run a little faster in debug builds... - thread_count = 25 - func_count = 500 - fib = 15 + thread_count = 10 + func_count = 200 + fib = 12 def after_threads(self): """Runs once after all the threads have started""" @@ -93,7 +87,7 @@ def tearDown(self): monitoring.free_tool_id(self.tool_id) -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class SetPreTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): """Sets tracing one time after the threads have started""" @@ -112,7 +106,7 @@ def after_threads(self): sys.settrace(self.trace_func) -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class MonitoringMultiThreaded( MonitoringTestMixin, InstrumentationMultiThreadedMixin, TestCase ): @@ -146,7 +140,7 @@ def during_threads(self): self.set = not self.set -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): """Uses sys.settrace and repeatedly toggles instrumentation on and off""" @@ -172,12 +166,9 @@ def during_threads(self): self.set = not self.set -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): """Uses sys.setprofile and repeatedly toggles instrumentation on and off""" - thread_count = 25 - func_count = 200 - fib = 15 def setUp(self): self.set = False @@ -201,7 +192,7 @@ def during_threads(self): self.set = not self.set -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class MonitoringMisc(MonitoringTestMixin, TestCase): def register_callback(self): def callback(*args): @@ -227,6 +218,28 @@ def test_register_callback(self): for ref in self.refs: self.assertEqual(ref(), None) + def test_set_local_trace_opcodes(self): + def trace(frame, event, arg): + frame.f_trace_opcodes = True + return trace + + sys.settrace(trace) + try: + l = _PyRLock() + + def f(): + for i in range(3000): + with l: + pass + + t = Thread(target=f) + t.start() + for i in range(3000): + with l: + pass + t.join() + finally: + sys.settrace(None) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py new file mode 100644 index 00000000000000..6eead198deed46 --- /dev/null +++ b/Lib/test/test_free_threading/test_type.py @@ -0,0 +1,112 @@ +import unittest + +from concurrent.futures import ThreadPoolExecutor +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper, import_helper + + + +NTHREADS = 6 +BOTTOM = 0 +TOP = 1000 +ITERS = 100 + +class A: + attr = 1 + +@threading_helper.requires_working_threading() +class TestType(TestCase): + def test_attr_cache(self): + def read(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + A.attr + + def write(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + # Make _PyType_Lookup cache hot first + A.attr + A.attr + x = A.attr + x += 1 + A.attr = x + + + with ThreadPoolExecutor(NTHREADS) as pool: + pool.submit(read, (1,)) + pool.submit(write, (1,)) + pool.shutdown(wait=True) + + def test_attr_cache_consistency(self): + class C: + x = 0 + + DONE = False + def writer_func(): + for i in range(3000): + C.x + C.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = C.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test_attr_cache_consistency_subclass(self): + class C: + x = 0 + + class D(C): + pass + + DONE = False + def writer_func(): + for i in range(3000): + D.x + D.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = D.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def run_one(self, writer_func, reader_func): + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index bb4c7cc8701fb4..4a9a7313712f60 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -947,8 +947,13 @@ def mycmp(x, y): @unittest.skipIf(support.MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_cmp_to_signature(self): - self.assertEqual(str(Signature.from_callable(self.cmp_to_key)), - '(mycmp)') + sig = Signature.from_callable(self.cmp_to_key) + self.assertEqual(str(sig), '(mycmp)') + def mycmp(x, y): + return y - x + sig = Signature.from_callable(self.cmp_to_key(mycmp)) + self.assertEqual(str(sig), '(obj)') + @unittest.skipUnless(c_functools, 'requires the C _functools module') @@ -1864,9 +1869,10 @@ def test_staticmethod(x): self.assertIsNone(ref()) def test_common_signatures(self): - def orig(): ... + def orig(a, /, b, c=True): ... lru = self.module.lru_cache(1)(orig) + self.assertEqual(str(Signature.from_callable(lru)), '(a, /, b, c=True)') self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 2c8ceb664cb362..69ae58b0fbcae3 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -203,6 +203,25 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index d48f0d47ba1962..6d36df2c7413e0 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -532,6 +532,26 @@ def f(): with self.assertRaises(RuntimeError): gen.close() + def test_close_releases_frame_locals(self): + # See gh-118272 + + class Foo: + pass + + f = Foo() + f_wr = weakref.ref(f) + + def genfn(): + a = f + yield + + g = genfn() + next(g) + del f + g.close() + support.gc_collect() + self.assertIsNone(f_wr()) + class GeneratorThrowTest(unittest.TestCase): diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 04cb810d9babbf..04122fbdd0ae80 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -49,7 +49,7 @@ ShareableList = None from os import DirEntry from re import Pattern, Match -from types import GenericAlias, MappingProxyType, AsyncGeneratorType +from types import GenericAlias, MappingProxyType, AsyncGeneratorType, CoroutineType, GeneratorType from tempfile import TemporaryDirectory, SpooledTemporaryFile from urllib.parse import SplitResult, ParseResult from unittest.case import _AssertRaisesContext @@ -120,6 +120,7 @@ class BaseTest(unittest.TestCase): KeysView, ItemsView, ValuesView, Sequence, MutableSequence, MappingProxyType, AsyncGeneratorType, + GeneratorType, CoroutineType, DirEntry, chain, LoggerAdapter, StreamHandler, diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 70ee35ed2850bc..b72640bd871ba6 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -6,6 +6,7 @@ import unittest import warnings +from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -366,6 +367,8 @@ def test_glob_named_pipe(self): self.assertEqual(self.rglob('mypipe', 'sub'), []) self.assertEqual(self.rglob('mypipe', '*'), []) + + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 40c3023a827067..64282d0f2d0bcf 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1327,6 +1327,34 @@ def test_import_from_unloaded_package(self): import package2.submodule1 package2.submodule1.submodule2 + def test_rebinding(self): + # The same data is also used for testing pkgutil.resolve_name() + # in test_pkgutil and mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + + def test_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + from package4 import submodule + self.assertEqual(submodule.attr, 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + from package4 import submodule + self.assertEqual(submodule.attr, 'origin') + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + class OverridingImportBuiltinTests(unittest.TestCase): def test_override_builtin(self): @@ -2257,6 +2285,107 @@ def test_disallowed_reimport(self): class TestSinglePhaseSnapshot(ModuleSnapshot): + """A representation of a single-phase init module for testing. + + Fields from ModuleSnapshot: + + * id - id(mod) + * module - mod or a SimpleNamespace with __file__ & __spec__ + * ns - a shallow copy of mod.__dict__ + * ns_id - id(mod.__dict__) + * cached - sys.modules[name] (or None if not there or not snapshotable) + * cached_id - id(sys.modules[name]) (or None if not there) + + Extra fields: + + * summed - the result of calling "mod.sum(1, 2)" + * lookedup - the result of calling "mod.look_up_self()" + * lookedup_id - the object ID of self.lookedup + * state_initialized - the result of calling "mod.state_initialized()" + * init_count - (optional) the result of calling "mod.initialized_count()" + + Overridden methods from ModuleSnapshot: + + * from_module() + * parse() + + Other methods from ModuleSnapshot: + + * build_script() + * from_subinterp() + + ---- + + There are 5 modules in Modules/_testsinglephase.c: + + * _testsinglephase + * has global state + * extra loads skip the init function, copy def.m_base.m_copy + * counts calls to init function + * _testsinglephase_basic_wrapper + * _testsinglephase by another name (and separate init function symbol) + * _testsinglephase_basic_copy + * same as _testsinglephase but with own def (and init func) + * _testsinglephase_with_reinit + * has no global or module state + * mod.state_initialized returns None + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + * _testsinglephase_with_state + * has module state + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + + (See Modules/_testsinglephase.c for more info.) + + For all those modules, the snapshot after the initial load (not in + the global extensions cache) would look like the following: + + * initial load + * id: ID of nww module object + * ns: exactly what the module init put there + * ns_id: ID of new module's __dict__ + * cached_id: same as self.id + * summed: 3 (never changes) + * lookedup_id: same as self.id + * state_initialized: a timestamp between the time of the load + and the time of the snapshot + * init_count: 1 (None for _testsinglephase_with_reinit) + + For the other scenarios it varies. + + For the _testsinglephase, _testsinglephase_basic_wrapper, and + _testsinglephase_basic_copy modules, the snapshot should look + like the following: + + * reloaded + * id: no change + * ns: matches what the module init function put there, + including the IDs of all contained objects, + plus any extra attributes added before the reload + * ns_id: no change + * cached_id: no change + * lookedup_id: no change + * state_initialized: no change + * init_count: no change + * already loaded + * (same as initial load except for ns and state_initialized) + * ns: matches the initial load, incl. IDs of contained objects + * state_initialized: no change from initial load + + For _testsinglephase_with_reinit: + + * reloaded: same as initial load (old module & ns is discarded) + * already loaded: same as initial load (old module & ns is discarded) + + For _testsinglephase_with_state: + + * reloaded + * (same as initial load (old module & ns is discarded), + except init_count) + * init_count: increase by 1 + * already loaded: same as reloaded + """ @classmethod def from_module(cls, mod): @@ -2325,10 +2454,6 @@ def setUpClass(cls): # Start fresh. cls.clean_up() - @classmethod - def tearDownClass(cls): - restore__testsinglephase() - def tearDown(self): # Clean up the module. self.clean_up() @@ -2873,31 +2998,32 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # * module's global state was initialized but cleared # Start with an interpreter that gets destroyed right away. - base = self.import_in_subinterp(postscript=''' - # Attrs set after loading are not in m_copy. - mod.spam = 'spam, spam, mash, spam, eggs, and spam' - ''') + base = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') self.check_common(base) self.check_fresh(base) # At this point: # * alive in 0 interpreters # * module def in _PyRuntime.imports.extensions - # * mod init func ran again - # * m_copy is NULL (claered when the interpreter was destroyed) + # * mod init func ran for the first time (since reset) + # * m_copy is still set (owned by main interpreter) # * module's global state was initialized, not reset # Use a subinterpreter that sticks around. loaded_interp1 = self.import_in_subinterp(interpid1) self.check_common(loaded_interp1) - self.check_semi_fresh(loaded_interp1, loaded_main, base) + self.check_copied(loaded_interp1, base) # At this point: # * alive in 1 interpreter (interp1) # * module def still in _PyRuntime.imports.extensions - # * mod init func ran again - # * m_copy was copied from interp1 (was NULL) - # * module's global state was updated, not reset + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched # Use a subinterpreter while the previous one is still alive. loaded_interp2 = self.import_in_subinterp(interpid2) @@ -2907,9 +3033,9 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # At this point: # * alive in 2 interpreters (interp1, interp2) # * module def still in _PyRuntime.imports.extensions - # * mod init func ran again - # * m_copy was copied from interp2 (was from interp1) - # * module's global state was updated, not reset + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched @requires_subinterpreters def test_basic_multiple_interpreters_reset_each(self): diff --git a/Lib/test/test_import/data/package3/__init__.py b/Lib/test/test_import/data/package3/__init__.py new file mode 100644 index 00000000000000..7033c22a719ec4 --- /dev/null +++ b/Lib/test/test_import/data/package3/__init__.py @@ -0,0 +1,2 @@ +"""Rebinding the package attribute after importing the module.""" +from .submodule import submodule diff --git a/Lib/test/test_import/data/package3/submodule.py b/Lib/test/test_import/data/package3/submodule.py new file mode 100644 index 00000000000000..cd7b30db15e449 --- /dev/null +++ b/Lib/test/test_import/data/package3/submodule.py @@ -0,0 +1,7 @@ +attr = 'submodule' +class A: + attr = 'submodule' +class submodule: + attr = 'rebound' + class B: + attr = 'rebound' diff --git a/Lib/test/test_import/data/package4/__init__.py b/Lib/test/test_import/data/package4/__init__.py new file mode 100644 index 00000000000000..d8af60ab38a319 --- /dev/null +++ b/Lib/test/test_import/data/package4/__init__.py @@ -0,0 +1,5 @@ +"""Binding the package attribute without importing the module.""" +class submodule: + attr = 'origin' + class B: + attr = 'origin' diff --git a/Lib/test/test_import/data/package4/submodule.py b/Lib/test/test_import/data/package4/submodule.py new file mode 100644 index 00000000000000..c861417aece9c3 --- /dev/null +++ b/Lib/test/test_import/data/package4/submodule.py @@ -0,0 +1,3 @@ +attr = 'submodule' +class A: + attr = 'submodule' diff --git a/Lib/test/test_importlib/extension/_test_nonmodule_cases.py b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py new file mode 100644 index 00000000000000..8ffd18d221df4f --- /dev/null +++ b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py @@ -0,0 +1,44 @@ +import types +import unittest +from test.test_importlib import util + +machinery = util.import_importlib('importlib.machinery') + +from test.test_importlib.extension.test_loader import MultiPhaseExtensionModuleTests + + +class NonModuleExtensionTests: + setUp = MultiPhaseExtensionModuleTests.setUp + load_module_by_name = MultiPhaseExtensionModuleTests.load_module_by_name + + def _test_nonmodule(self): + # Test returning a non-module object from create works. + name = self.name + '_nonmodule' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + + # issue 27782 + def test_nonmodule_with_methods(self): + # Test creating a non-module object with methods defined. + name = self.name + '_nonmodule_with_methods' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + self.assertEqual(mod.bar(10, 1), 9) + + def test_null_slots(self): + # Test that NULL slots aren't a problem. + name = self.name + '_null_slots' + module = self.load_module_by_name(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + + +(Frozen_NonModuleExtensionTests, + Source_NonModuleExtensionTests + ) = util.test_both(NonModuleExtensionTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 7607f0e0857595..0dd21e079eba22 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -10,7 +10,8 @@ import warnings import importlib.util import importlib -from test.support import MISSING_C_DOCSTRINGS +from test import support +from test.support import MISSING_C_DOCSTRINGS, script_helper class LoaderTests: @@ -325,29 +326,6 @@ def test_unloadable_nonascii(self): self.load_module_by_name(name) self.assertEqual(cm.exception.name, name) - def test_nonmodule(self): - # Test returning a non-module object from create works. - name = self.name + '_nonmodule' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - - # issue 27782 - def test_nonmodule_with_methods(self): - # Test creating a non-module object with methods defined. - name = self.name + '_nonmodule_with_methods' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - self.assertEqual(mod.bar(10, 1), 9) - - def test_null_slots(self): - # Test that NULL slots aren't a problem. - name = self.name + '_null_slots' - module = self.load_module_by_name(name) - self.assertIsInstance(module, types.ModuleType) - self.assertEqual(module.__name__, name) - def test_bad_modules(self): # Test SystemError is raised for misbehaving extensions. for name_base in [ @@ -401,5 +379,14 @@ def test_nonascii(self): ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery) +class NonModuleExtensionTests(unittest.TestCase): + def test_nonmodule_cases(self): + # The test cases in this file cause the GIL to be enabled permanently + # in free-threaded builds, so they are run in a subprocess to isolate + # this effect. + script = support.findfile("test_importlib/extension/_test_nonmodule_cases.py") + script_helper.run_test_script(script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 9af1e4d505c66e..3477112927a0b6 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -17,7 +17,7 @@ from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) -from test.support import script_helper, threading_helper +from test.support import script_helper, threading_helper, requires_gil_enabled threading_helper.requires_working_threading(module=True) @@ -248,6 +248,9 @@ def test_concurrent_futures_circular_import(self): 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) + # gh-118727 and gh-118729: pool_in_threads.py may crash in free-threaded + # builds, which can hang the Tsan test so temporarily skip it for now. + @requires_gil_enabled("gh-118727: test may crash in free-threaded builds") def test_multiprocessing_pool_circular_import(self): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 6b577090bdff68..82e466e978624f 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -817,6 +817,21 @@ def monkey(filename, module_globals=None): def test_getsource_on_code_object(self): self.assertSourceEqual(mod.eggs.__code__, 12, 18) + def test_getsource_on_generated_class(self): + A = type('A', (), {}) + self.assertEqual(inspect.getsourcefile(A), __file__) + self.assertEqual(inspect.getfile(A), __file__) + self.assertIs(inspect.getmodule(A), sys.modules[__name__]) + self.assertRaises(OSError, inspect.getsource, A) + self.assertRaises(OSError, inspect.getsourcelines, A) + self.assertIsNone(inspect.getcomments(A)) + + def test_getsource_on_class_without_firstlineno(self): + __firstlineno__ = 1 + class C: + nonlocal __firstlineno__ + self.assertRaises(OSError, inspect.getsource, C) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -3069,6 +3084,13 @@ def test_signature_on_builtins_no_signature(self): self.assertEqual(inspect.signature(builtin), inspect.signature(template)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_parsing_with_defaults(self): + _testcapi = import_helper.import_module("_testcapi") + meth = _testcapi.DocStringUnrepresentableSignatureTest.with_default + self.assertEqual(str(inspect.signature(meth)), '(self, /, x=1)') + def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): inspect.signature(42) @@ -4690,6 +4712,16 @@ class D2(D1): self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 47fc50a0e20349..8959ffb6dcc236 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -829,17 +829,28 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_limit) super().tearDown() - def test_pylong_int_to_decimal(self): - n = (1 << 100_000) - 1 - suffix = '9883109375' + def _test_pylong_int_to_decimal(self, n, suffix): s = str(n) - assert s[-10:] == suffix - s = str(-n) - assert s[-10:] == suffix - s = '%d' % n - assert s[-10:] == suffix - s = b'%d' % n - assert s[-10:] == suffix.encode('ascii') + self.assertEqual(s[-10:], suffix) + s2 = str(-n) + self.assertEqual(s2, '-' + s) + s3 = '%d' % n + self.assertEqual(s3, s) + s4 = b'%d' % n + self.assertEqual(s4, s.encode('ascii')) + + def test_pylong_int_to_decimal(self): + self._test_pylong_int_to_decimal((1 << 100_000), '9883109376') + self._test_pylong_int_to_decimal((1 << 100_000) - 1, '9883109375') + self._test_pylong_int_to_decimal(10**30_000, '0000000000') + self._test_pylong_int_to_decimal(10**30_000 - 1, '9999999999') + self._test_pylong_int_to_decimal(3**60_000, '9313200001') + + @support.requires_resource('cpu') + def test_pylong_int_to_decimal_2(self): + self._test_pylong_int_to_decimal(2**1_000_000, '2747109376') + self._test_pylong_int_to_decimal(10**300_000, '0000000000') + self._test_pylong_int_to_decimal(3**600_000, '3132000001') def test_pylong_int_divmod(self): n = (1 << 100_000) @@ -895,6 +906,18 @@ def test_pylong_misbehavior_error_path_from_str( with self.assertRaises(RuntimeError): int(big_value) + def test_pylong_roundtrip(self): + from random import randrange, getrandbits + bits = 5000 + while bits <= 1_000_000: + bits += randrange(-100, 101) # break bitlength patterns + hibit = 1 << (bits - 1) + n = hibit | getrandbits(bits - 1) + assert n.bit_length() == bits + sn = str(n) + self.assertFalse(sn.startswith('0')) + self.assertEqual(n, int(sn)) + bits <<= 1 if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py index 4b16ecc31156a5..52ff553f60d0d7 100644 --- a/Lib/test/test_interpreters/__init__.py +++ b/Lib/test/test_interpreters/__init__.py @@ -1,5 +1,6 @@ import os -from test.support import load_package_tests +from test.support import load_package_tests, Py_GIL_DISABLED -def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) +if not Py_GIL_DISABLED: + def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 95e67911db6a7f..e243da309f03d8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1821,6 +1821,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 2868dd01545b95..ec2aac81682db8 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -622,9 +622,14 @@ def test_exception_in_post_comp_call(self): def test_frame_locals(self): code = """ - val = [sys._getframe().f_locals for a in [0]][0]["a"] + val = "a" in [sys._getframe().f_locals for a in [0]][0] """ import sys + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) def _recursive_replace(self, maybe_code): @@ -666,6 +671,20 @@ def test_code_replace_extended_arg(self): self._check_in_scopes(code, expected) self._check_in_scopes(code, expected, exec_func=self._replacing_exec) + def test_multiple_comprehension_name_reuse(self): + code = """ + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"y": [3]}, ns={"x": 3}) + + code = """ + x = 2 + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"]) + self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"]) __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d8ae607d1d8d01..e651d96f100a83 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4598,25 +4598,55 @@ def test_msecs_has_no_floating_point_precision_loss(self): self.assertEqual(record.msecs, want) self.assertEqual(record.created, ns / 1e9) - # The test overrides a private attribute - @support.cpython_only def test_relativeCreated_has_higher_precision(self): - # See issue gh-102402 - ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC + # See issue gh-102402. + # Run the code in the subprocess, because the time module should + # be patched before the first import of the logging package. + # Temporary unloading and re-importing the logging package has + # side effects (including registering the atexit callback and + # references leak). + start_ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC offsets_ns = (200, 500, 12_354, 99_999, 1_677_903_456_999_123_456) + code = textwrap.dedent(f""" + start_ns = {start_ns!r} + offsets_ns = {offsets_ns!r} + start_monotonic_ns = start_ns - 1 + + import time + # Only time.time_ns needs to be patched for the current + # implementation, but patch also other functions to make + # the test less implementation depending. + old_time_ns = time.time_ns + old_time = time.time + old_monotonic_ns = time.monotonic_ns + old_monotonic = time.monotonic + time_ns_result = start_ns + time.time_ns = lambda: time_ns_result + time.time = lambda: time.time_ns()/1e9 + time.monotonic_ns = lambda: time_ns_result - start_monotonic_ns + time.monotonic = lambda: time.monotonic_ns()/1e9 + try: + import logging - with (patch("time.time_ns") as time_ns_mock, - support.swap_attr(logging, '_startTime', ns)): - for offset_ns in offsets_ns: - # mock for log record creation - new_ns = ns + offset_ns - time_ns_mock.return_value = new_ns - - record = logging.makeLogRecord({'msg': 'test'}) - self.assertAlmostEqual(record.created, new_ns / 1e9, places=6) - + for offset_ns in offsets_ns: + # mock for log record creation + time_ns_result = start_ns + offset_ns + record = logging.makeLogRecord({{'msg': 'test'}}) + print(record.created, record.relativeCreated) + finally: + time.time_ns = old_time_ns + time.time = old_time + time.monotonic_ns = old_monotonic_ns + time.monotonic = old_monotonic + """) + rc, out, err = assert_python_ok("-c", code) + out = out.decode() + for offset_ns, line in zip(offsets_ns, out.splitlines(), strict=True): + with self.subTest(offset_ns=offset_ns): + created, relativeCreated = map(float, line.split()) + self.assertAlmostEqual(created, (start_ns + offset_ns) / 1e9, places=6) # After PR gh-102412, precision (places) increases from 3 to 7 - self.assertAlmostEqual(record.relativeCreated, offset_ns / 1e6, places=7) + self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) class TestBufferingFormatter(logging.BufferingFormatter): diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 0e4dbc0b64a439..b68c442013c09f 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -657,7 +657,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 8192502a40791b..95629ed862d6eb 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -801,7 +801,7 @@ def test_sizeof(self): def _test_cow_mutation(self, mutation): # Common code for all BytesIO copy-on-write mutation tests. - imm = b' ' * 1024 + imm = (' ' * 1024).encode("ascii") old_rc = sys.getrefcount(imm) memio = self.ioclass(imm) self.assertEqual(sys.getrefcount(imm), old_rc + 1) diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 70f9c5d9400bf6..b37b7defe84d1c 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -164,6 +164,7 @@ ... d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['foo'] = 4 d['foo'] = 42 d['bar'] = 123 @@ -183,12 +184,12 @@ ... b = 24 ... meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -206,12 +207,13 @@ prepare: C () [('other', 'booh')] d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['a'] = 1 d['a'] = 2 d['b'] = 3 d['__static_attributes__'] = () meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 30e1c56bf0bc52..2e0ad0606ae9c2 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -36,20 +36,28 @@ def setUp(self): def test_case_sensitivity(self): eq = self.assertEqual - eq(self.db.guess_type("foobar.HTML"), self.db.guess_type("foobar.html")) - eq(self.db.guess_type("foobar.TGZ"), self.db.guess_type("foobar.tgz")) - eq(self.db.guess_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) def test_default_data(self): eq = self.assertEqual - eq(self.db.guess_type("foo.html"), ("text/html", None)) - eq(self.db.guess_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual @@ -63,7 +71,7 @@ def test_file_parsing(self): eq = self.assertEqual sio = io.StringIO("x-application/x-unittest pyunit\n") self.db.readfp(sio) - eq(self.db.guess_type("foo.pyunit"), + eq(self.db.guess_file_type("foo.pyunit"), ("x-application/x-unittest", None)) eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") @@ -95,12 +103,12 @@ def test_read_mime_types(self): def test_non_standard_types(self): eq = self.assertEqual # First try strict - eq(self.db.guess_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict - eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') @@ -124,15 +132,26 @@ def test_filename_with_url_delimiters(self): '//share/server/', '\\\\share\\server\\'): path = prefix + name with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) eq(self.db.guess_type(path), gzip_expected) expected = (None, None) if os.name == 'nt' else gzip_expected for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): path = prefix + name with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') result = self.db.guess_type('http://host.html') msg = 'URL only has a host name, not a file' self.assertSequenceEqual(result, (None, None), msg) @@ -214,6 +233,7 @@ def check_extensions(): self.assertEqual(mimetypes.guess_extension('text/html'), '.html') self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') self.assertEqual(mimetypes.guess_extension('text/rtf'), '.rtf') + self.assertEqual(mimetypes.guess_extension('text/x-rst'), '.rst') self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') @@ -241,22 +261,38 @@ def test_init_stability(self): def test_path_like_ob(self): filename = "LICENSE.txt" - filepath = pathlib.Path(filename) - filepath_with_abs_dir = pathlib.Path('/dir/'+filename) - filepath_relative = pathlib.Path('../dir/'+filename) - path_dir = pathlib.Path('./') + filepath = os_helper.FakePath(filename) + filepath_with_abs_dir = os_helper.FakePath('/dir/'+filename) + filepath_relative = os_helper.FakePath('../dir/'+filename) + path_dir = os_helper.FakePath('./') - expected = self.db.guess_type(filename) + expected = self.db.guess_file_type(filename) + self.assertEqual(self.db.guess_file_type(filepath), expected) self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_file_type( + filepath_with_abs_dir), expected) self.assertEqual(self.db.guess_type( filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_file_type(filepath_relative), expected) self.assertEqual(self.db.guess_type(filepath_relative), expected) + + self.assertEqual(self.db.guess_file_type(path_dir), (None, None)) self.assertEqual(self.db.guess_type(path_dir), (None, None)) + def test_bytes_path(self): + self.assertEqual(self.db.guess_file_type(b'foo.html'), + self.db.guess_file_type('foo.html')) + self.assertEqual(self.db.guess_file_type(b'foo.tar.gz'), + self.db.guess_file_type('foo.tar.gz')) + self.assertEqual(self.db.guess_file_type(b'foo.tgz'), + self.db.guess_file_type('foo.tgz')) + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_file_type( + path="foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_type( - url="foo.html", strict=True), ("text/html", None)) + url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( type='image/jpg', strict=True), []) self.assertEqual(self.db.guess_extension( diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 11c61bc2e0688d..6974bc5517ae5f 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,16 +3,20 @@ import collections import dis import functools +import math import operator import sys import textwrap import types import unittest import asyncio -from test import support + +import test.support from test.support import requires_specialization, script_helper from test.support.import_helper import import_module +_testcapi = test.support.import_helper.import_module("_testcapi") + PAIR = (0,1) def f1(): @@ -652,6 +656,17 @@ def func2(): self.check_lines(func2, [1,2,3,4,5,6]) + def test_generator_with_line(self): + + def f(): + def a(): + yield + def b(): + yield from a() + next(b()) + + self.check_lines(f, [1,3,5,4,2,4]) + class TestDisable(MonitoringTestBase, unittest.TestCase): def gen(self, cond): @@ -1831,15 +1846,17 @@ class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): _testinternalcapi = import_module("_testinternalcapi") - self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(opt) + if hasattr(_testinternalcapi, "get_optimizer"): + self.old_opt = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.new_counter_optimizer() + _testinternalcapi.set_optimizer(opt) super(TestOptimizer, self).setUp() def tearDown(self): super(TestOptimizer, self).tearDown() import _testinternalcapi - _testinternalcapi.set_optimizer(self.old_opt) + if hasattr(_testinternalcapi, "get_optimizer"): + _testinternalcapi.set_optimizer(self.old_opt) def test_for_loop(self): def test_func(x): @@ -1885,5 +1902,180 @@ def test_monitoring_live_at_shutdown(self): # gh-115832: An object destructor running during the final GC of # interpreter shutdown triggered an infinite loop in the # instrumentation code. - script = support.findfile("_test_monitoring_shutdown.py") + script = test.support.findfile("_test_monitoring_shutdown.py") script_helper.run_test_script(script) + + +class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): + + class Scope: + def __init__(self, *args): + self.args = args + + def __enter__(self): + _testcapi.monitoring_enter_scope(*self.args) + + def __exit__(self, *args): + _testcapi.monitoring_exit_scope() + + def setUp(self): + super(TestCApiEventGeneration, self).setUp() + + capi = _testcapi + + self.codelike = capi.CodeLike(2) + + self.cases = [ + # (Event, function, *args) + ( 1, E.PY_START, capi.fire_event_py_start), + ( 1, E.PY_RESUME, capi.fire_event_py_resume), + ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10), + ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), + ( 2, E.CALL, capi.fire_event_call, callable, 40), + ( 1, E.JUMP, capi.fire_event_jump, 60), + ( 1, E.BRANCH, capi.fire_event_branch, 70), + ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), + ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), + ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), + ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, ValueError(7)), + ] + + + def check_event_count(self, event, func, args, expected): + class Counter: + def __init__(self): + self.count = 0 + def __call__(self, *args): + self.count += 1 + + try: + counter = Counter() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + try: + func(*args) + except ValueError as e: + self.assertIsInstance(expected, ValueError) + self.assertEqual(str(e), str(expected)) + return + else: + self.assertEqual(counter.count, expected) + + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_fire_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + self.check_event_count(event, function, args_, expected) + + def test_missing_exception(self): + for _, event, function, *args in self.cases: + if not (args and isinstance(args[-1], BaseException)): + continue + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) + evt = int(math.log2(event)) + expected = ValueError(f"Firing event {evt} with no exception set") + self.check_event_count(event, function, args_, expected) + + + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, + E.EXCEPTION_HANDLED, E.PY_UNWIND } + + def check_disable(self, event, func, args, expected): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.disable = True + if event in self.CANNOT_DISABLE: + # use try-except rather then assertRaises to avoid + # events from framework code + try: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + except ValueError: + pass + else: + self.Error("Expected a ValueError") + else: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected - 1) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_disable_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(2) + with self.subTest(function.__name__): + args_ = (self.codelike, 0) + tuple(args) + self.check_disable(event, function, args_, expected) + + def test_enter_scope_two_events(self): + try: + yield_counter = CounterWithDisable() + unwind_counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND) + + yield_value = int(math.log2(E.PY_YIELD)) + unwind_value = int(math.log2(E.PY_UNWIND)) + cl = _testcapi.CodeLike(2) + common_args = (cl, 0) + with self.Scope(cl, yield_value, unwind_value): + yield_counter.count = 0 + unwind_counter.count = 0 + + _testcapi.fire_event_py_unwind(*common_args, ValueError(42)) + assert(yield_counter.count == 0) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 1) + assert(unwind_counter.count == 1) + + yield_counter.disable = True + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index f4e954fd02148d..92a34113bc0383 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -16,6 +16,8 @@ def disabling_optimizer(func): def wrapper(*args, **kwargs): + if not hasattr(_testinternalcapi, "get_optimizer"): + return func(*args, **kwargs) old_opt = _testinternalcapi.get_optimizer() _testinternalcapi.set_optimizer(None) try: diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py index 899a4507317334..fac4d1a4ab44c4 100644 --- a/Lib/test/test_optimizer.py +++ b/Lib/test/test_optimizer.py @@ -80,6 +80,8 @@ def func(x=0): class TestOptimizerSymbols(unittest.TestCase): + @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"), + "requires _testinternalcapi.uop_symbols_test") def test_optimizer_symbols(self): _testinternalcapi.uop_symbols_test() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 6a34f48f7873ee..9c9c8536dc7542 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1811,6 +1811,25 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = os_helper.TESTFN + path1 = os.path.join(os_helper.TESTFN, 'dir1') + path2 = os.path.join(os_helper.TESTFN, 'dir2') + # mode=0o700 is special-cased to override ACLs on Windows + # There's no way to know exactly how the ACLs will look, so we'll + # check that they are different from a regularly created directory. + os.mkdir(path1, mode=0o700) + os.mkdir(path2, mode=0o777) + + out1 = subprocess.check_output(["icacls.exe", path1], encoding="oem") + out2 = subprocess.check_output(["icacls.exe", path2], encoding="oem") + os.rmdir(path1) + os.rmdir(path2) + out1 = out1.replace(path1, "") + out2 = out2.replace(path2, "") + self.assertNotEqual(out1, out2) + def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') @@ -2365,6 +2384,7 @@ def test_fchown(self): support.is_emscripten or support.is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 298e78ccee3875..1bdab125dc6ef0 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2957,6 +2957,14 @@ def test_invalid_syntax_3(self): pass """) + def test_len1_tuple_sequence_pattern_comma(self): + # correct syntax would be `case(*x,):` + self.assert_syntax_error(""" + match ...: + case (*x): + pass + """) + def test_mapping_pattern_keys_may_only_match_literals_and_attribute_lookups(self): self.assert_syntax_error(""" match ...: diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2d057e2647f13c..f47466410082ef 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -10,11 +10,12 @@ import subprocess import textwrap import linecache +import zipapp from contextlib import ExitStack, redirect_stdout from io import StringIO from test import support -from test.support import os_helper +from test.support import force_not_colorized, os_helper from test.support.import_helper import import_module from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch @@ -45,7 +46,6 @@ def test_pdb_displayhook(): >>> def test_function(foo, bar): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'foo', @@ -54,8 +54,8 @@ def test_pdb_displayhook(): ... 'continue', ... ]): ... test_function(1, None) - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) foo 1 (Pdb) bar @@ -97,6 +97,7 @@ def test_pdb_basic_commands(): ... print(ret) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to line ret = test_function_2('baz') ... 'step', # entering the function call ... 'args', # display function args ... 'list', # list function source @@ -121,6 +122,9 @@ def test_pdb_basic_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) step @@ -144,7 +148,7 @@ def test_pdb_basic_commands(): [EOF] (Pdb) bt ... - (25)() + (26)() -> test_function() (3)test_function() -> ret = test_function_2('baz') @@ -274,8 +278,8 @@ def test_pdb_breakpoint_commands(): ... 'continue', ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) break 4, + @@ -354,6 +358,46 @@ def test_pdb_breakpoint_commands(): 4 """ +def test_pdb_breakpoint_with_filename(): + """Breakpoints with filename:lineno + + >>> def test_function(): + ... # inspect_fodder2 is a great module as the line number is stable + ... from test.test_inspect import inspect_fodder2 as mod2 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... mod2.func88() + ... mod2.func114() + ... # Be a good citizen and clean up the mess + ... reset_Breakpoint() + + First, need to clear bdb state that might be left over from previous tests. + Otherwise, the new breakpoints might get assigned different numbers. + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS + ... 'break test.test_inspect.inspect_fodder2:90', + ... 'continue', # will stop at func88 + ... 'break test/test_inspect/inspect_fodder2.py:115', + ... 'continue', # will stop at func114 + ... 'continue', + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) break test.test_inspect.inspect_fodder2:90 + Breakpoint 1 at ...inspect_fodder2.py:90 + (Pdb) continue + > ...inspect_fodder2.py(90)func88() + -> return 90 + (Pdb) break test/test_inspect/inspect_fodder2.py:115 + Breakpoint 2 at ...inspect_fodder2.py:115 + (Pdb) continue + > ...inspect_fodder2.py(115)func114() + -> return 115 + (Pdb) continue + """ + def test_pdb_breakpoints_preserved_across_interactive_sessions(): """Breakpoints are remembered between interactive sessions @@ -438,8 +482,7 @@ def test_pdb_pp_repr_exc(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p obj *** Exception: repr_exc @@ -479,6 +522,7 @@ def test_list_commands(): ... ret = test_function_2('baz') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to the test function line ... 'list', # list first function ... 'step', # step into second function ... 'list', # list second function @@ -494,6 +538,9 @@ def test_list_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) list @@ -574,8 +621,7 @@ def test_pdb_whatis_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) whatis myvar @@ -604,6 +650,7 @@ def test_pdb_display_command(): ... a = 4 >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'display +', ... 'display', ... 'display a', @@ -619,6 +666,9 @@ def test_pdb_display_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> a = 1 (Pdb) display + @@ -667,6 +717,7 @@ def test_pdb_alias_command(): ... o.method() >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'alias pi', ... 'alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}")', ... 'alias ps pi self', @@ -685,6 +736,9 @@ def test_pdb_alias_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> o.method() (Pdb) alias pi @@ -740,8 +794,7 @@ def test_pdb_where_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) w ... @@ -751,7 +804,7 @@ def test_pdb_where_command(): -> f() (2)f() -> g(); - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) where ... @@ -761,7 +814,7 @@ def test_pdb_where_command(): -> f() (2)f() -> g(); - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) u > (2)f() @@ -774,7 +827,7 @@ def test_pdb_where_command(): -> f() > (2)f() -> g(); - (2)g()->None + (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -810,8 +863,7 @@ def test_pdb_interact_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (4)test_function()->None + > (4)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) interact *pdb interact start* @@ -851,6 +903,7 @@ def test_convenience_variables(): ... util_function() >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # Step to try statement ... '$_frame.f_lineno', # Check frame convenience variable ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable @@ -873,6 +926,9 @@ def test_convenience_variables(): ... 'continue', ... ]): ... test_function() + > (2)util_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)util_function() -> try: (Pdb) $_frame.f_lineno @@ -1346,6 +1402,7 @@ def test_post_mortem(): ... print('Not reached.') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # step to test_function_2() line ... 'next', # step over exception-raising call ... 'bt', # get a backtrace ... 'list', # list code of test_function() @@ -1357,6 +1414,9 @@ def test_post_mortem(): ... test_function() ... except ZeroDivisionError: ... print('Correctly reraised.') + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> test_function_2() (Pdb) next @@ -1366,7 +1426,7 @@ def test_post_mortem(): -> test_function_2() (Pdb) bt ... - (10)() + (11)() -> test_function() > (3)test_function() -> test_function_2() @@ -1403,9 +1463,13 @@ def test_pdb_skip_modules(): >>> with PdbTestInput([ ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() + > (3)skip_module() + -> import pdb; pdb.Pdb(skip=['stri*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (4)skip_module() -> string.capwords('FOO') (Pdb) step @@ -1420,7 +1484,6 @@ def test_pdb_invalid_arg(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'a = 3', @@ -1429,8 +1492,8 @@ def test_pdb_invalid_arg(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) a = 3 *** Invalid argument: = 3 Usage: a(rgs) @@ -1464,10 +1527,14 @@ def test_pdb_skip_modules_with_callback(): ... 'step', ... 'step', ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() ... pass # provides something to "step" to + > (4)skip_module() + -> import pdb; pdb.Pdb(skip=['module_to_skip*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (5)skip_module() -> mod.foo_pony(callback) (Pdb) step @@ -1486,7 +1553,7 @@ def test_pdb_skip_modules_with_callback(): > (5)skip_module()->None -> mod.foo_pony(callback) (Pdb) step - > (10)() + > (11)() -> pass # provides something to "step" to (Pdb) continue """ @@ -1505,6 +1572,7 @@ def test_pdb_continue_in_bottomframe(): ... print(4) >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'step', ... 'next', ... 'break 7', ... 'continue', @@ -1513,6 +1581,9 @@ def test_pdb_continue_in_bottomframe(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> inst.set_trace() + (Pdb) step > (4)test_function() -> inst.botframe = sys._getframe() # hackery to get the right botframe (Pdb) next @@ -1604,8 +1675,8 @@ def test_next_until_return_at_return_event(): ... 'return', ... 'continue']): ... test_function() - > (3)test_function() - -> test_function_2() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_function_2 Breakpoint 1 at :2 (Pdb) continue @@ -1664,12 +1735,16 @@ def test_pdb_next_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1724,12 +1799,16 @@ def test_pdb_next_command_for_coroutine(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'step', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1784,12 +1863,16 @@ def test_pdb_next_command_for_asyncgen(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1842,11 +1925,15 @@ def test_pdb_return_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'return', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1897,9 +1984,13 @@ def test_pdb_return_command_for_coroutine(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1932,11 +2023,15 @@ def test_pdb_until_command_for_generator(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 4', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -1988,9 +2083,13 @@ def test_pdb_until_command_for_coroutine(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 8', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -2029,8 +2128,8 @@ def test_pdb_next_command_in_generator_for_loop(): ... 'next', ... 'continue']): ... test_function() - > (3)test_function() - -> for i in test_gen(): + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_gen Breakpoint 1 at :2 (Pdb) continue @@ -2069,11 +2168,15 @@ def test_pdb_next_command_subiterator(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -2102,7 +2205,6 @@ def test_pdb_multiline_statement(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'def f(x):', @@ -2112,8 +2214,8 @@ def test_pdb_multiline_statement(): ... 'c' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) def f(x): ... return x * 2 ... @@ -2122,14 +2224,76 @@ def test_pdb_multiline_statement(): (Pdb) c """ +def test_pdb_closure(): + """Test for all expressions/statements that involve closure + + >>> k = 0 + >>> g = 1 + >>> def test_function(): + ... x = 2 + ... g = 3 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'k', + ... 'g', + ... 'y = y', + ... 'global g; g', + ... 'global g; (lambda: g)()', + ... '(lambda: x)()', + ... '(lambda: g)()', + ... 'lst = [n for n in range(10) if (n % x) == 0]', + ... 'lst', + ... 'sum(n for n in lst if n > x)', + ... 'x = 1; raise Exception()', + ... 'x', + ... 'def f():', + ... ' return x', + ... '', + ... 'f()', + ... 'c' + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) k + 0 + (Pdb) g + 3 + (Pdb) y = y + *** NameError: name 'y' is not defined + (Pdb) global g; g + 1 + (Pdb) global g; (lambda: g)() + 1 + (Pdb) (lambda: x)() + 2 + (Pdb) (lambda: g)() + 3 + (Pdb) lst = [n for n in range(10) if (n % x) == 0] + (Pdb) lst + [0, 2, 4, 6, 8] + (Pdb) sum(n for n in lst if n > x) + 18 + (Pdb) x = 1; raise Exception() + *** Exception + (Pdb) x + 1 + (Pdb) def f(): + ... return x + ... + (Pdb) f() + 1 + (Pdb) c + """ + def test_pdb_show_attribute_and_item(): - """Test for multiline statement + """Test for expressions with command prefix >>> def test_function(): ... n = lambda x: x ... c = {"a": 1} ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'c["a"]', @@ -2143,8 +2307,8 @@ def test_pdb_show_attribute_and_item(): ... 'c' ... ]): ... test_function() - > (5)test_function() - -> pass + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) c["a"] 1 (Pdb) c.get("a") @@ -2178,12 +2342,12 @@ def test_pdb_issue_20766(): >>> with PdbTestInput(['continue', ... 'continue']): ... test_function() - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 1: - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 2: """ @@ -2204,8 +2368,8 @@ def test_pdb_issue_43318(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) clear :3 @@ -2237,12 +2401,16 @@ def test_pdb_issue_gh_91742(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 5', ... 'continue' ... ]): ... test_function() + > (11)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (12)test_function() -> about() (Pdb) step @@ -2280,6 +2448,7 @@ def test_pdb_issue_gh_94215(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 3', @@ -2292,6 +2461,9 @@ def test_pdb_issue_gh_94215(): ... 'continue' ... ]): ... test_function() + > (8)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (9)test_function() -> func() (Pdb) step @@ -2344,8 +2516,7 @@ def test_pdb_issue_gh_101673(): ... 'continue' ... ]): ... test_function() - --Return-- - > (3)test_function()->None + > (3)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) !a = 2 (Pdb) ll @@ -2369,16 +2540,16 @@ def test_pdb_issue_gh_103225(): ... a = 1 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... b = 2 - > (7)() - -> b = 2 + > (6)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) longlist 1 with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE 2 'longlist', 3 'continue' 4 ]): 5 a = 1 - 6 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - 7 -> b = 2 + 6 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + 7 b = 2 (Pdb) continue """ @@ -2397,9 +2568,8 @@ def test_pdb_issue_gh_101517(): ... 'continue' ... ]): ... test_function() - --Return-- - > (None)test_function()->None - -> Warning: lineno is None + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2415,9 +2585,8 @@ def test_pdb_issue_gh_108976(): ... 'continue' ... ]): ... test_function() - bdb.Bdb.dispatch: unknown debugging event: 'opcode' - > (5)test_function() - -> a = 1 + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2436,9 +2605,8 @@ def test_pdb_issue_gh_80731(): ... raise ValueError('Correct') ... except ValueError: ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass - > (10)() - -> pass + > (9)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) import sys (Pdb) sys.exc_info() (, ValueError('Correct'), ) @@ -2452,6 +2620,7 @@ def test_pdb_ambiguous_statements(): Make sure that ambiguous statements prefixed by '!' are properly disambiguated >>> with PdbTestInput([ + ... 's', # step to the print line ... '! n = 42', # disambiguated statement: reassign the name n ... 'n', # advance the debugger into the print() ... 'continue' @@ -2460,6 +2629,9 @@ def test_pdb_ambiguous_statements(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... print(f"The value of n is {n}") > (8)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s + > (9)() -> print(f"The value of n is {n}") (Pdb) ! n = 42 (Pdb) n @@ -2488,8 +2660,8 @@ def test_pdb_f_trace_lines(): ... 'continue' ... ]): ... test_function() - > (6)test_function() - -> if frame.f_trace_lines != False: + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2512,7 +2684,6 @@ def test_pdb_function_break(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 'break foo', @@ -2522,8 +2693,8 @@ def test_pdb_function_break(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break foo Breakpoint ... at :1 (Pdb) break bar @@ -2553,6 +2724,7 @@ def test_pdb_issue_gh_65052(): ... A() >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 's', + ... 's', ... 'retval', ... 'continue', ... 'args', @@ -2561,6 +2733,9 @@ def test_pdb_issue_gh_65052(): ... 'continue', ... ]): ... test_function() + > (3)__new__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)__new__() -> return object.__new__(cls) (Pdb) s @@ -2570,8 +2745,8 @@ def test_pdb_issue_gh_65052(): (Pdb) retval *** repr(retval) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) continue - > (7)__init__() - -> self.a = 1 + > (6)__init__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) args self = *** repr(self) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) display self @@ -2878,6 +3053,7 @@ def start_pdb(): self.assertNotIn(b'Error', stdout, "Got an error running test script under PDB") + @force_not_colorized def test_issue16180(self): # A syntax error in the debuggee. script = "def f: pass\n" @@ -2891,6 +3067,7 @@ def test_issue16180(self): 'Fail to handle a syntax error in the debuggee.' .format(expected, stderr)) + @force_not_colorized def test_issue84583(self): # A syntax error from ast.literal_eval should not make pdb exit. script = "import ast; ast.literal_eval('')\n" @@ -3492,6 +3669,30 @@ def test_non_utf8_encoding(self): if filename.endswith(".py"): self._run_pdb([os.path.join(script_dir, filename)], 'q') + def test_zipapp(self): + with os_helper.temp_dir() as temp_dir: + os.mkdir(os.path.join(temp_dir, 'source')) + script = textwrap.dedent( + """ + def f(x): + return x + 1 + f(21 + 21) + """ + ) + with open(os.path.join(temp_dir, 'source', '__main__.py'), 'w') as f: + f.write(script) + zipapp.create_archive(os.path.join(temp_dir, 'source'), + os.path.join(temp_dir, 'zipapp.pyz')) + stdout, _ = self._run_pdb([os.path.join(temp_dir, 'zipapp.pyz')], '\n'.join([ + 'b f', + 'c', + 'p x', + 'q' + ])) + self.assertIn('42', stdout) + self.assertIn('return x + 1', stdout) + + class ChecklineTests(unittest.TestCase): def setUp(self): linecache.clearcache() # Pdb.checkline() uses linecache.getline() diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 6989aafd293138..6c27ee4db97af3 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -933,23 +933,6 @@ def f(): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") return f - def test_deleting_local_warns_and_assigns_none(self): - f = self.make_function_with_no_checks() - co_code = f.__code__.co_code - def trace(frame, event, arg): - if event == 'line' and frame.f_lineno == 4: - del frame.f_locals["x"] - sys.settrace(None) - return None - return trace - e = r"assigning None to unbound local 'x'" - with self.assertWarnsRegex(RuntimeWarning, e): - sys.settrace(trace) - f() - self.assertInBytecode(f, "LOAD_FAST") - self.assertNotInBytecode(f, "LOAD_FAST_CHECK") - self.assertEqual(f.__code__.co_code, co_code) - def test_modifying_local_does_not_add_check(self): f = self.make_function_with_no_checks() def trace(frame, event, arg): diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 040be63da11447..7c1bbeed168d2e 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -5,6 +5,7 @@ import sysconfig import os import pathlib +import shutil from test import support from test.support.script_helper import ( make_script, @@ -76,14 +77,27 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - perf_lines = perf_file_contents.splitlines(); - expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + perf_lines = perf_file_contents.splitlines() + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] for expected_symbol in expected_symbols: - perf_line = next((line for line in perf_lines if expected_symbol in line), None) - self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_line = next( + (line for line in perf_lines if expected_symbol in line), None + ) + self.assertIsNotNone( + perf_line, f"Could not find {expected_symbol} in perf file" + ) perf_addr = perf_line.split(" ")[0] - self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") - self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") + self.assertFalse( + perf_addr.startswith("0x"), "Address should not be prefixed with 0x" + ) + self.assertTrue( + set(perf_addr).issubset(string.hexdigits), + "Address should contain only hex characters", + ) def test_trampoline_works_with_forks(self): code = """if 1: @@ -212,11 +226,11 @@ def test_sys_api_get_status(self): assert_python_ok("-c", code) -def is_unwinding_reliable(): +def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False - return "no-omit-frame-pointer" in cflags + return "no-omit-frame-pointer" in cflags and "_Py_JIT" not in cflags def perf_command_works(): @@ -259,14 +273,27 @@ def perf_command_works(): return True -def run_perf(cwd, *args, **env_vars): +def run_perf(cwd, *args, use_jit=False, **env_vars): if env_vars: env = os.environ.copy() env.update(env_vars) else: env = None output_file = cwd + "/perf_output.perf" - base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + if not use_jit: + base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + else: + base_cmd = ( + "perf", + "record", + "-g", + "--call-graph=dwarf,65528", + "-F99", + "-k1", + "-o", + output_file, + "--", + ) proc = subprocess.run( base_cmd + args, stdout=subprocess.PIPE, @@ -274,9 +301,21 @@ def run_perf(cwd, *args, **env_vars): env=env, ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") + if use_jit: + jit_output_file = cwd + "/jit_output.dump" + command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) + proc = subprocess.run( + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + if proc.returncode: + print(proc.stderr) + raise ValueError(f"Perf failed with return code {proc.returncode}") + # Copy the jit_output_file to the output_file + os.rename(jit_output_file, output_file) + base_cmd = ("perf", "script") proc = subprocess.run( ("perf", "script", "-i", output_file), @@ -290,20 +329,9 @@ def run_perf(cwd, *args, **env_vars): ) -@unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(is_unwinding_reliable(), "Unwinding is unreliable") -class TestPerfProfiler(unittest.TestCase): - def setUp(self): - super().setUp() - self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) - - def tearDown(self) -> None: - super().tearDown() - files_to_delete = ( - set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files - ) - for file in files_to_delete: - file.unlink() +class TestPerfProfilerMixin: + def run_perf(self, script_dir, perf_mode, script): + raise NotImplementedError() def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: @@ -322,14 +350,14 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, "-Xperf", script) + stdout, stderr = self.run_perf(script_dir, script) self.assertEqual(stderr, "") self.assertIn(f"py::foo:{script}", stdout) self.assertIn(f"py::bar:{script}", stdout) self.assertIn(f"py::baz:{script}", stdout) - def test_python_calls_do_not_appear_in_the_stack_if_perf_activated(self): + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: def foo(n): @@ -346,13 +374,38 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, script) + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=False + ) self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + is_unwinding_reliable_with_frame_pointers(), + "Unwinding is unreliable with frame pointers", +) +class TestPerfProfiler(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf(script_dir, sys.executable, "-Xperf", script) + return run_perf(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + def test_pre_fork_compile(self): code = """if 1: import sys @@ -370,7 +423,7 @@ def bar_fork(): foo_fork() def foo(): - pass + import time; time.sleep(1) def bar(): foo() @@ -423,12 +476,47 @@ def compile_trampolines_for_all_functions(): # identical in both the parent and child perf-map files. perf_file_lines = perf_file_contents.split("\n") for line in perf_file_lines: - if ( - f"py::foo_fork:{script}" in line - or f"py::bar_fork:{script}" in line - ): + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: self.assertIn(line, child_perf_file_contents) +def _is_perf_vesion_at_least(major, minor): + # The output of perf --version looks like "perf version 6.7-3" but + # it can also be perf version "perf version 5.15.143" + try: + output = subprocess.check_output(["perf", "--version"], text=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return False + version = output.split()[2] + version = version.split("-")[0] + version = version.split(".") + version = tuple(map(int, version)) + return version >= (major, minor) + + +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless(_is_perf_vesion_at_least(6, 6), "perf command may not work due to a perf bug") +class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf( + script_dir, sys.executable, "-Xperf_jit", script, use_jit=True + ) + return run_perf(script_dir, sys.executable, script, use_jit=True) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("jit*.dump")) + self.perf_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete = files_to_delete - self.perf_files + for file in files_to_delete: + file.unlink() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 6fcd726345eeac..e19dce1dbd2583 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -12,6 +12,9 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -318,6 +321,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 1d22869046fd12..7e5f04c22bd6d3 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -564,6 +564,7 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_confstr(self): self.assertRaises(ValueError, posix.confstr, "CS_garbage") self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index c7c5419ffe3e37..46206accbafc36 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -226,7 +226,7 @@ def test_others(self): cm( 'pdb', # pyclbr does not handle elegantly `typing` or properties - ignore=('Union', '_ModuleTarget', '_ScriptTarget'), + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget'), ) cm('pydoc', ignore=('input', 'output',)) # properties diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py new file mode 100644 index 00000000000000..c8990b699b214c --- /dev/null +++ b/Lib/test/test_pyrepl.py @@ -0,0 +1,1006 @@ +import itertools +import os +import rlcompleter +import sys +import tempfile +import unittest +from code import InteractiveConsole +from functools import partial +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from test.support import requires +from test.support.import_helper import import_module + +# Optionally test pyrepl. This currently requires that the +# 'curses' resource be given on the regrtest command line using the -u +# option. Additionally, we need to attempt to import curses and readline. +requires("curses") +curses = import_module("curses") +readline = import_module("readline") + +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent +from _pyrepl.unix_eventqueue import EventQueue +from _pyrepl.simple_interact import InteractiveColoredConsole + + +def more_lines(unicodetext, namespace=None): + if namespace is None: + namespace = {} + src = _strip_final_indent(unicodetext) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def multiline_input(reader, namespace=None): + saved = reader.more_lines + try: + reader.more_lines = partial(more_lines, namespace=namespace) + reader.ps1 = reader.ps2 = ">>>" + reader.ps3 = reader.ps4 = "..." + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + +def code_to_events(code): + for c in code: + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) + + +def prepare_mock_console(events, **kwargs): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def prepare_fake_console(**kwargs): + console = FakeConsole() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def prepare_reader(console, **kwargs): + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def handle_all_events( + events, prepare_console=prepare_mock_console, prepare_reader=prepare_reader +): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + return reader, console + + +handle_events_narrow_console = partial( + handle_all_events, prepare_console=partial(prepare_mock_console, width=10) +) + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8"): + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block: bool = True) -> Event | None: + return next(self.events) + + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: int | bytes) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self) -> None: + pass + + def repaint(self) -> None: + pass + + +class TestCursorPosition(TestCase): + def test_up_arrow_simple(self): + # fmt: off + code = ( + 'def f():\n' + ' ...\n' + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) + + def test_down_arrow_end_of_input(self): + # fmt: off + code = ( + 'def f():\n' + ' ...\n' + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) + + def test_left_arrow_simple(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) + + def test_right_arrow_end_of_line(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 1 for simple character + self.assertEqual(reader.cxy, (1, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 0) + self.assertEqual(reader.cxy, (0, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor here (showing 2nd line only): + # < ' 可口可乐; 可口可樂'> + # ^ + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.cxy, (10, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain( + code_to_events("' 可口可乐; 可口可樂'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.cxy, (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + first_line = "for _ in _:" + second_line = " hello" + + # fmt: off + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline + self.assertEqual(reader.cxy, (len(second_line), 1)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + + # fmt: off + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.cxy, (len(last_line), 3)) + + def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " x = '可口可乐; 可口可樂'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = '可口可乐; 可口可樂' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) + + +class TestPyReplOutput(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_basic(self): + reader = self.prepare_reader(code_to_events("1+1\n")) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_multiline_edit(self): + events = itertools.chain( + code_to_events("def f():\n ...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "def f():\n ...\n ") + output = multiline_input(reader) + self.assertEqual(output, "def g():\n ...\n ") + + def test_history_navigation_with_up_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_navigation_with_down_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_search(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "3+3") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + + +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_simple_completion(self): + events = code_to_events("os.geten\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getenv") + + def test_completion_with_many_options(self): + # Test with something that initially displays many options + # and then complete from one of them. The first time tab is + # pressed, the options are displayed (which corresponds to + # when the repl shows [ not unique ]) and the second completes + # from one of them. + events = code_to_events("os.\t\tO_AP\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_APPEND") + + def test_empty_namespace_completion(self): + events = code_to_events("os.geten\t\n") + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = code_to_events("py\t\n") + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + +@patch("_pyrepl.curses.tigetstr", lambda x: b"") +class TestUnivEventQueue(TestCase): + def setUp(self): + self.file = tempfile.TemporaryFile() + + def tearDown(self) -> None: + self.file.close() + + def test_get(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = EventQueue(self.file.fileno(), "utf-8") + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": "b"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"c": "d"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.push("b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push("d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push("\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push("b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") + + +class TestPasteEvent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_paste(self): + # fmt: off + code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n' + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + # fmt: off + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n' + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n\n' + ) + + expected = ( + 'def f():\n' + ' x = y\n' + ' ' + ) + # fmt: on + + events = code_to_events(code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, expected) + + def test_paste_not_in_paste_mode(self): + # fmt: off + input_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n\n' + ) + + output_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:' + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste(self): + """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" + # fmt: off + input_code = ( + 'def a():\n' + ' for x in range(10):\n' + '\n' + ' if x%2:\n' + ' print(x)\n' + '\n' + ' else:\n' + ' pass\n' + ) + + output_code = ( + 'def a():\n' + ' for x in range(10):\n' + '\n' + ' if x%2:\n' + ' print(x)\n' + '\n' + ' else:\n' + ' pass\n' + ) + # fmt: on + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste_single_line(self): + input_code = "oneline" + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, input_code) + + +class TestReader(TestCase): + def assert_screen_equals(self, reader, expected): + actual = reader.calc_screen() + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + def test_calc_screen_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_calc_screen_wrap_wide_characters(self): + events = code_to_events(8 * "a" + "樂") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") + + def test_calc_screen_wrap_three_lines(self): + events = code_to_events(20 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + + # fmt: off + self.assert_screen_equals(reader, ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + )) + # fmt: on + + def test_calc_screen_backspace(self): + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "aa") + + def test_calc_screen_wrap_removes_after_backspace(self): + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, 9 * "a") + + def test_calc_screen_backspace_in_second_line_after_wrap(self): + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_setpos_from_xy_multiple_lines(self): + # fmt: off + code = ( + "def foo():\n" + " return 1" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + + def test_up_arrow_after_ctrl_r(self): + events = iter([ + Event(evt='key', data='\x12', raw=bytearray(b'\x12')), + Event(evt='key', data='up', raw=bytearray(b'\x1bOA')), + ]) + + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index b1e4ef4197d130..9a44ab1768656a 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -4,6 +4,7 @@ import os import time import pickle +import shlex import warnings import test.support @@ -1397,5 +1398,47 @@ def test_after_fork(self): support.wait_process(pid, exitcode=0) +class CommandLineTest(unittest.TestCase): + def test_parse_args(self): + args, help_text = random._parse_args(shlex.split("--choice a b c")) + self.assertEqual(args.choice, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--integer 5")) + self.assertEqual(args.integer, 5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--float 2.5")) + self.assertEqual(args.float, 2.5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("a b c")) + self.assertEqual(args.input, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("5")) + self.assertEqual(args.input, ["5"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("2.5")) + self.assertEqual(args.input, ["2.5"]) + self.assertTrue(help_text.startswith("usage: ")) + + def test_main(self): + for command, expected in [ + ("--choice a b c", "b"), + ('"a b c"', "b"), + ("a b c", "b"), + ("--choice 'a a' 'b b' 'c c'", "b b"), + ("'a a' 'b b' 'c c'", "b b"), + ("--integer 5", 4), + ("5", 4), + ("--float 2.5", 2.266632777287572), + ("2.5", 2.266632777287572), + ]: + random.seed(0) + self.assertEqual(random.main(shlex.split(command)), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 6d8744ca5f7969..51ce095df41fc1 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -590,6 +590,11 @@ def test_connection_resource_warning(self): del cx gc_collect() + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index d22698168615e2..c06c285c5013a6 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -227,6 +227,9 @@ def test_windows_feature_macros(self): "PyEval_EvalFrameEx", "PyEval_GetBuiltins", "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", "PyEval_GetFuncDesc", "PyEval_GetFuncName", "PyEval_GetGlobals", diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 204787a88a9c5f..40680759d456ac 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2402,7 +2402,7 @@ def integrate(func, low, high, steps=10_000): with self.assertRaises(StatisticsError): kde(sample, h=0.0) # Zero bandwidth with self.assertRaises(StatisticsError): - kde(sample, h=0.0) # Negative bandwidth + kde(sample, h=-1.0) # Negative bandwidth with self.assertRaises(TypeError): kde(sample, h='str') # Wrong bandwidth type with self.assertRaises(StatisticsError): @@ -2426,6 +2426,103 @@ def integrate(func, low, high, steps=10_000): self.assertEqual(f_hat(-1.0), 1/2) self.assertEqual(f_hat(1.0), 1/2) + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + + def test_kde_kernel_invcdfs(self): + kernel_invcdfs = statistics._kernel_invcdfs + kde = statistics.kde + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + for kernel, invcdf in kernel_invcdfs.items(): + with self.subTest(kernel=kernel): + cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) + + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=-1.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] + n = 1_000_000 + h = 1.75 + dx = 0.1 + + def p_observed(x): + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) + return (j - i) / len(big_sample) + + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) + + for x in xarr: + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) + + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + class TestQuantiles(unittest.TestCase): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index e9bec3317811dd..b978838ea7003f 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1213,6 +1213,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -2027,6 +2043,31 @@ def f(x: *b) Invalid expressions in type scopes: + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + >>> type A[T: (x:=3)] = int Traceback (most recent call last): ... @@ -2392,13 +2433,40 @@ def bug(): code += "): yield a" return code - CO_MAXBLOCKS = 20 # static nesting limit of the compiler + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block - for n in range(CO_MAXBLOCKS): + for n in range(MAX_MANAGERS): with self.subTest(f"within range: {n=}"): compile(get_code(n), "", "exec") - for n in range(CO_MAXBLOCKS, CO_MAXBLOCKS + 5): + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): with self.subTest(f"out of range: {n=}"): self._check_error(get_code(n), "too many statically nested blocks") @@ -2536,7 +2604,8 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 14ec51eb757e00..ee3bd0092f9bf3 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1053,6 +1053,12 @@ def test_getallocatedblocks(self): c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) + def test_is_gil_enabled(self): + if support.Py_GIL_DISABLED: + self.assertIs(type(sys._is_gil_enabled()), bool) + else: + self.assertTrue(sys._is_gil_enabled()) + def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) # Don't use the atexit module because _Py_Finalizing is only set @@ -1555,7 +1561,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + check(x, size('3Pi2cP7P2ic??2P')) # function def func(): pass check(func, size('15Pi')) @@ -1600,7 +1606,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module - check(unittest, size('PnPPP')) + if support.Py_GIL_DISABLED: + check(unittest, size('PPPPPP')) + else: + check(unittest, size('PPPPP')) # None check(None, size('')) # NotImplementedType @@ -1779,6 +1788,21 @@ def test_asyncgen_hooks(self): self.assertIsNone(old.finalizer) firstiter = lambda *a: None + finalizer = lambda *a: None + + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer="invalid") + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + # gh-118473 + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter="invalid", finalizer=finalizer) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + sys.set_asyncgen_hooks(firstiter=firstiter) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1786,7 +1810,6 @@ def test_asyncgen_hooks(self): self.assertIs(hooks.finalizer, None) self.assertIs(hooks[1], None) - finalizer = lambda *a: None sys.set_asyncgen_hooks(finalizer=finalizer) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 61c6a5a42502e7..9233304c6a5327 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -232,6 +232,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -347,6 +352,21 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index ebdb58f91d3d8a..553d54329d7939 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -482,29 +482,36 @@ def testfunc(arg): return arg self.interp.createcommand('testfunc', testfunc) self.addCleanup(self.interp.tk.deletecommand, 'testfunc') - def check(value, expected=None, *, eq=self.assertEqual): - if expected is None: - expected = value + def check(value, expected1=None, expected2=None, *, eq=self.assertEqual): + expected = value + if self.wantobjects >= 2: + if expected2 is not None: + expected = expected2 + expected_type = type(expected) + else: + if expected1 is not None: + expected = expected1 + expected_type = str nonlocal result result = None r = self.interp.call('testfunc', value) - self.assertIsInstance(result, str) + self.assertIsInstance(result, expected_type) eq(result, expected) - self.assertIsInstance(r, str) + self.assertIsInstance(r, expected_type) eq(r, expected) def float_eq(actual, expected): self.assertAlmostEqual(float(actual), expected, delta=abs(expected) * 1e-10) - check(True, '1') - check(False, '0') + check(True, '1', 1) + check(False, '0', 0) check('string') check('string\xbd') check('string\u20ac') check('string\U0001f4bb') if sys.platform != 'win32': - check('<\udce2\udc82\udcac>', '<\u20ac>') - check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>') + check('<\udce2\udc82\udcac>', '<\u20ac>', '<\u20ac>') + check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>', '<\U0001f4bb>') check('') check(b'string', 'string') check(b'string\xe2\x82\xac', 'string\xe2\x82\xac') @@ -526,9 +533,13 @@ def float_eq(actual, expected): check(float('inf'), eq=float_eq) check(-float('inf'), eq=float_eq) # XXX NaN representation can be not parsable by float() - check((), '') - check((1, (2,), (3, 4), '5 6', ()), '1 2 {3 4} {5 6} {}') - check([1, [2,], [3, 4], '5 6', []], '1 2 {3 4} {5 6} {}') + check((), '', '') + check((1, (2,), (3, 4), '5 6', ()), + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + check([1, [2,], [3, 4], '5 6', []], + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) def test_splitlist(self): splitlist = self.interp.tk.splitlist diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b64b6a4f2baeb5..19ddeaa169bf93 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -13,6 +13,7 @@ import weakref import gc import shutil +import subprocess from unittest import mock import unittest @@ -803,6 +804,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line.removeprefix(dir).strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 362a3f9c4a01d1..329767aa82e336 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,7 +3,7 @@ """ import test.support -from test.support import threading_helper, requires_subprocess +from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure @@ -49,7 +49,7 @@ def skip_unless_reliable_fork(test): if support.HAVE_ASAN_FORK_BUG: return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) if support.check_sanitizer(thread=True): - return unittest.skip("TSAN doesn't support threads after fork") + return unittest.skip("TSAN doesn't support threads after fork")(test) return test @@ -781,8 +781,7 @@ def func(): "current is main True\n" ) - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @support.requires_fork() + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): code = """if 1: @@ -2025,6 +2024,7 @@ def check_interrupt_main_noerror(self, signum): # Restore original handler signal.signal(signum, handler) + @requires_gil_enabled("gh-118433: Flaky due to a longstanding bug") def test_interrupt_main_subthread(self): # Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index ef1c99f57c6f47..b8e549e314d27d 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -302,7 +302,37 @@ def test_copy(self): image2 = image.copy() self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) - self.assertEqual(image.get(4, 6), image.get(4, 6)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = image.copy(from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(3, 2), image.get(3*2+2, 2*2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) def test_subsample(self): image = self.create() @@ -316,6 +346,13 @@ def test_subsample(self): self.assertEqual(image2.height(), 8) self.assertEqual(image2.get(2, 3), image.get(4, 6)) + image2 = image.subsample(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + def test_zoom(self): image = self.create() image2 = image.zoom(2, 3) @@ -330,6 +367,118 @@ def test_zoom(self): self.assertEqual(image2.get(8, 12), image.get(4, 6)) self.assertEqual(image2.get(9, 13), image.get(4, 6)) + image2 = image.zoom(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + def test_copy_replace(self): + image = self.create() + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(0, 0, 100, 50)) + self.assertEqual(image2.width(), 100) + self.assertEqual(image2.height(), 50) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12, 4+8), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12*2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2, 4+8*3), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(3*3, 2*3), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(3*3+2, 2*3+2), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) + + def checkImgTrans(self, image, expected): + actual = {(x, y) + for x in range(image.width()) + for y in range(image.height()) + if image.transparency_get(x, y)} + self.assertEqual(actual, expected) + + def test_copy_replace_compositingrule(self): + image1 = tkinter.PhotoImage(master=self.root, width=2, height=2) + image1.blank() + image1.put('black', to=(0, 0, 2, 2)) + image1.transparency_set(0, 0, True) + + # default compositingrule + image2 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image2.blank() + image2.put('white', to=(0, 0, 2, 2)) + image2.copy_replace(image1, to=(1, 1)) + self.checkImgTrans(image2, {(0, 2), (2, 0)}) + + image3 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image3.blank() + image3.put('white', to=(0, 0, 2, 2)) + image3.copy_replace(image1, to=(1, 1), compositingrule='overlay') + self.checkImgTrans(image3, {(0, 2), (2, 0)}) + + image4 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image4.blank() + image4.put('white', to=(0, 0, 2, 2)) + image4.copy_replace(image1, to=(1, 1), compositingrule='set') + self.checkImgTrans(image4, {(0, 2), (1, 1), (2, 0)}) + def test_put(self): image = self.create() image.put('{red green} {blue yellow}', to=(4, 6)) @@ -356,6 +505,50 @@ def test_get(self): self.assertRaises(tkinter.TclError, image.get, 16, 15) self.assertRaises(tkinter.TclError, image.get, 15, 16) + def test_read(self): + # Due to the Tk bug https://core.tcl-lang.org/tk/tktview/1576528 + # the -from option does not work correctly for GIF and PNG files. + # Use the PPM file for this test. + testfile = support.findfile('python.ppm', subdir='tkinterdata') + image = tkinter.PhotoImage(master=self.root, file=testfile) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + self.assertRaises(tkinter.TclError, image2.read, self.testfile, 'ppm') + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root, file=testfile) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + image2.read(testfile, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + def test_write(self): filename = os_helper.TESTFN import locale @@ -367,19 +560,58 @@ def test_write(self): image.write(filename) image2 = tkinter.PhotoImage('::img::test2', master=self.root, - format='ppm', - file=filename) + format='ppm', file=filename) self.assertEqual(str(image2), '::img::test2') self.assertEqual(image2.type(), 'photo') self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) self.assertEqual(image2.get(0, 0), image.get(0, 0)) - self.assertEqual(image2.get(15, 8), image.get(15, 8)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) image.write(filename, format='gif', from_coords=(4, 6, 6, 9)) image3 = tkinter.PhotoImage('::img::test3', master=self.root, - format='gif', - file=filename) + format='gif', file=filename) + self.assertEqual(str(image3), '::img::test3') + self.assertEqual(image3.type(), 'photo') + self.assertEqual(image3.width(), 2) + self.assertEqual(image3.height(), 3) + self.assertEqual(image3.get(0, 0), image.get(4, 6)) + self.assertEqual(image3.get(1, 2), image.get(5, 8)) + + image.write(filename, background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', file=filename) + self.assertEqual(image4.get(0, 0), (255, 0, 0)) + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + image.write(filename, grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', file=filename) + c = image5.get(4, 6) + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_data(self): + image = self.create() + + data = image.data() + self.assertIsInstance(data, tuple) + for row in data: + self.assertIsInstance(row, str) + self.assertEqual(data[6].split()[4], '#%02x%02x%02x' % image.get(4, 6)) + + data = image.data('ppm') + image2 = tkinter.PhotoImage('::img::test2', master=self.root, + format='ppm', data=data) + self.assertEqual(str(image2), '::img::test2') + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + data = image.data(format='gif', from_coords=(4, 6, 6, 9)) + image3 = tkinter.PhotoImage('::img::test3', master=self.root, + format='gif', data=data) self.assertEqual(str(image3), '::img::test3') self.assertEqual(image3.type(), 'photo') self.assertEqual(image3.width(), 2) @@ -387,6 +619,19 @@ def test_write(self): self.assertEqual(image3.get(0, 0), image.get(4, 6)) self.assertEqual(image3.get(1, 2), image.get(5, 8)) + data = image.data('ppm', background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', data=data) + self.assertEqual(image4.get(0, 0), (255, 0, 0)) + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + data = image.data('ppm', grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', data=data) + c = image5.get(4, 6) + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_transparency(self): image = self.create() self.assertEqual(image.transparency_get(0, 0), True) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 8927fccc289320..5987ec382e6c85 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -26,9 +26,9 @@ import json import textwrap import traceback -import contextlib from functools import partial from pathlib import Path +import _colorize MODULE_PREFIX = f'{__name__}.' if __name__ == '__main__' else '' @@ -40,25 +40,18 @@ LEVENSHTEIN_DATA_FILE = Path(__file__).parent / 'levenshtein_examples.json' -ORIGINAL_CAN_COLORIZE = traceback._can_colorize - -def setUpModule(): - traceback._can_colorize = lambda: False - -def tearDownModule(): - traceback._can_colorize = ORIGINAL_CAN_COLORIZE class TracebackCases(unittest.TestCase): # For now, a very minimal set of tests. I want to be sure that # formatting of SyntaxErrors works based on changes for 2.1. def setUp(self): super().setUp() - self.colorize = traceback._COLORIZE - traceback._COLORIZE = False + self.colorize = _colorize.COLORIZE + _colorize.COLORIZE = False def tearDown(self): super().tearDown() - traceback._COLORIZE = self.colorize + _colorize.COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -507,7 +500,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -546,7 +539,7 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=, tb=, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), @@ -695,7 +688,6 @@ def f_with_multiline(): ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' return compile(code, "?", "exec")\n' - ' ~~~~~~~^^^^^^^^^^^^^^^^^^^\n' ' File "?", line 7\n' ' foo(a, z\n' ' ^' @@ -785,8 +777,8 @@ def f_with_binary_operator(): def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = 1 - b = "" - return ( a ) +b + b = c = "" + return ( a ) +b + c lineno_f = f_with_binary_operator.__code__.co_firstlineno expected_error = ( @@ -795,7 +787,7 @@ def f_with_binary_operator(): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' - ' return ( a ) +b\n' + ' return ( a ) +b + c\n' ' ~~~~~~~~~~^~\n' ) result_lines = self.get_exception(f_with_binary_operator) @@ -983,7 +975,7 @@ def f1(a): def f2(b): raise RuntimeError("fail") return f2 - return f1("x")("y") + return f1("x")("y")("z") lineno_f = f_with_call.__code__.co_firstlineno expected_error = ( @@ -992,7 +984,7 @@ def f2(b): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' - ' return f1("x")("y")\n' + ' return f1("x")("y")("z")\n' ' ~~~~~~~^^^^^\n' f' File "{__file__}", line {lineno_f+3}, in f2\n' ' raise RuntimeError("fail")\n' @@ -1507,6 +1499,184 @@ def f(): ' raise MemoryError()'] self.assertEqual(actual, expected) + def test_anchors_for_simple_return_statements_are_elided(self): + def g(): + 1/0 + + def f(): + return g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + return g() + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g() + 1", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " 2, 4,", + " 5)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " ~^^^", + " 2, 4,", + " ^^^^^", + " 5) + 1", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def test_anchors_for_simple_assign_statements_are_elided(self): + def g(): + 1/0 + + def f(): + x = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g(1,", + " 2, 3,", + " 4)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + x = y = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g()", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = y = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g(1,", + " ~^^^", + " 2, 3,", + " ^^^^^", + " 4)", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + @requires_debug_ranges() class PurePythonTracebackErrorCaretTests( @@ -1701,7 +1871,7 @@ def f(): # Check a known (limited) number of recursive invocations def g(count=10): if count: - return g(count-1) + return g(count-1) + 1 raise ValueError with captured_output("stderr") as stderr_g: @@ -1715,13 +1885,13 @@ def g(count=10): lineno_g = g.__code__.co_firstlineno result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1760,13 +1930,10 @@ def h(count=10): ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' @@ -1786,13 +1953,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1800,7 +1967,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+80}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -1818,13 +1985,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1833,7 +2000,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+112}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+109}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -3715,6 +3882,27 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) + def test_getattr_suggestions_underscored(self): + class A: + bluch = None + + self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + + class B: + _bluch = None + def method(self, name): + getattr(self, name) + + self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): class A: blech = None @@ -3907,6 +4095,17 @@ def test_import_from_suggestions(self): actual = self.get_import_from_suggestion(code, 'bluch') self.assertIn(suggestion, actual) + def test_import_from_suggestions_underscored(self): + code = "bluch = None" + self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_bluch')) + + code = "_bluch = None" + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_blach')) + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" @@ -4287,11 +4486,14 @@ def foo(*args): x = {'a':{'b': None}} y = x['a']['b']['c'] - def baz(*args): - return foo(1,2,3,4) + def baz2(*args): + return (lambda *args: foo(*args))(1,2,3,4) + + def baz1(*args): + return baz2(1,2,3,4) def bar(): - return baz(1, + return baz1(1, 2,3 ,4) try: @@ -4301,14 +4503,14 @@ def bar(): e, capture_locals=True ) lines = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) - self.assertIn("return " + red + "foo" + reset + boldr + "(1,2,3,4)" + reset, lines) - self.assertIn("return " + red + "baz" + reset + boldr + "(1," + reset, lines) - self.assertIn(boldr + "2,3" + reset, lines) - self.assertIn(boldr + ",4)" + reset, lines) + self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) + self.assertIn("return baz2(1,2,3,4)", lines) + self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) def test_colorized_syntax_error(self): @@ -4319,11 +4521,11 @@ def test_colorized_syntax_error(self): e, capture_locals=True ) actual = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET expected = "".join([ f' File {magenta}""{reset}, line {magenta}1{reset}\n', f' a {boldr}${reset} b\n', @@ -4342,15 +4544,15 @@ def foo(): self.fail("No exception thrown.") except Exception as e: with captured_output("stderr") as tbstderr: - with unittest.mock.patch('traceback._can_colorize', return_value=True): + with unittest.mock.patch('_colorize.can_colorize', return_value=True): exception_print(e) actual = tbstderr.getvalue().splitlines() - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET lno_foo = foo.__code__.co_firstlineno expected = ['Traceback (most recent call last):', f' File {magenta}"{__file__}"{reset}, ' @@ -4364,38 +4566,6 @@ def foo(): f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] self.assertEqual(actual, expected) - @force_not_colorized - def test_colorized_detection_checks_for_environment_variables(self): - if sys.platform == "win32": - virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) - else: - virtual_patching = contextlib.nullcontext() - with virtual_patching: - - flags = unittest.mock.MagicMock(ignore_environment=False) - with (unittest.mock.patch("os.isatty") as isatty_mock, - unittest.mock.patch("sys.flags", flags), - unittest.mock.patch("traceback._can_colorize", ORIGINAL_CAN_COLORIZE)): - isatty_mock.return_value = True - with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): - self.assertEqual(traceback._can_colorize(), False) - isatty_mock.return_value = False - with unittest.mock.patch("os.environ", {}): - self.assertEqual(traceback._can_colorize(), False) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 4b86395ee74f75..82f1007f9ac97b 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -6,7 +6,7 @@ import weakref from test.support import requires_working_socket, check_syntax_error, run_code -from typing import Generic, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args +from typing import Generic, NoDefault, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args class TypeParamsInvalidTest(unittest.TestCase): @@ -412,6 +412,14 @@ def test_comprehension_02(self): func, = T.__bound__ self.assertEqual(func(), 1) + def test_comprehension_03(self): + def F[T: [lambda: T for T in (T, [1])[1]]](): return [lambda: T for T in T.__name__] + func, = F() + self.assertEqual(func(), "T") + T, = F.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + def test_gen_exp_in_nested_class(self): code = """ from test.test_type_params import make_base @@ -591,10 +599,12 @@ class Foo[T: Foo, U: (Foo, Foo)]: self.assertEqual(type_params[0].__name__, "T") self.assertIs(type_params[0].__bound__, Foo) self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[0].__default__, NoDefault) self.assertEqual(type_params[1].__name__, "U") self.assertIs(type_params[1].__bound__, None) self.assertEqual(type_params[1].__constraints__, (Foo, Foo)) + self.assertIs(type_params[1].__default__, NoDefault) def test_evaluation_error(self): class Foo[T: Undefined, U: (Undefined,)]: @@ -605,6 +615,8 @@ class Foo[T: Undefined, U: (Undefined,)]: type_params[0].__bound__ self.assertEqual(type_params[0].__constraints__, ()) self.assertIs(type_params[1].__bound__, None) + self.assertIs(type_params[0].__default__, NoDefault) + self.assertIs(type_params[1].__default__, NoDefault) with self.assertRaises(NameError): type_params[1].__constraints__ @@ -764,6 +776,31 @@ class D[U](T): self.assertIn(int, C.D.__bases__) self.assertIs(C.D.x, str) + +class DynamicClassTest(unittest.TestCase): + def _set_type_params(self, ns, params): + ns['__type_params__'] = params + + def test_types_new_class_with_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}, + lambda ns: self._set_type_params(ns, (T,))) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, (T,)) + self.assertEqual(Klass.__parameters__, (T,)) + + def test_types_new_class_no_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, ()) # must be explicitly set + self.assertEqual(Klass.__parameters__, (T,)) + + class TypeParamsManglingTest(unittest.TestCase): def test_mangling(self): class Foo[__T]: @@ -1158,3 +1195,103 @@ class Inner[U](T): """ with self.assertRaises(RuntimeError): run_code(code) + + +class DefaultsTest(unittest.TestCase): + def test_defaults_on_func(self): + ns = run_code(""" + def func[T=int, **U=float, *V=None](): + pass + """) + + T, U, V = ns["func"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_class(self): + ns = run_code(""" + class C[T=int, **U=float, *V=None]: + pass + """) + + T, U, V = ns["C"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_type_alias(self): + ns = run_code(""" + type Alias[T = int, **U = float, *V = None] = int + """) + + T, U, V = ns["Alias"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_starred_invalid(self): + check_syntax_error(self, "type Alias[T = *int] = int") + check_syntax_error(self, "type Alias[**P = *int] = int") + + def test_starred_typevartuple(self): + ns = run_code(""" + default = tuple[int, str] + type Alias[*Ts = *default] = Ts + """) + + Ts, = ns["Alias"].__type_params__ + self.assertEqual(Ts.__default__, next(iter(ns["default"]))) + + def test_nondefault_after_default(self): + check_syntax_error(self, "def func[T=int, U](): pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "class C[T=int, U]: pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "type A[T=int, U] = int", "non-default type parameter 'U' follows default type parameter") + + def test_lazy_evaluation(self): + ns = run_code(""" + type Alias[T = Undefined, *U = Undefined, **V = Undefined] = int + """) + + T, U, V = ns["Alias"].__type_params__ + + with self.assertRaises(NameError): + T.__default__ + with self.assertRaises(NameError): + U.__default__ + with self.assertRaises(NameError): + V.__default__ + + ns["Undefined"] = "defined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + # Now it is cached + ns["Undefined"] = "redefined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + def test_symtable_key_regression_default(self): + # Test against the bugs that would happen if we used .default_ + # as the key in the symtable. + ns = run_code(""" + type X[T = [T for T in [T]]] = T + """) + + T, = ns["X"].__type_params__ + self.assertEqual(T.__default__, [T]) + + def test_symtable_key_regression_name(self): + # Test against the bugs that would happen if we used .name + # as the key in the symtable. + ns = run_code(""" + type X1[T = A] = T + type X2[T = B] = T + A = "A" + B = "B" + """) + + self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") + self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 703fe84f3aad10..bd116bb1ab7213 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -38,7 +38,7 @@ from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard, TypeIs +from typing import TypeGuard, TypeIs, NoDefault import abc import textwrap import typing @@ -580,6 +580,174 @@ def test_constructor(self): self.assertIs(T.__bound__, None) +class TypeParameterDefaultsTests(BaseTestCase): + def test_typevar(self): + T = TypeVar('T', default=int) + self.assertEqual(T.__default__, int) + self.assertTrue(T.has_default()) + self.assertIsInstance(T, TypeVar) + + class A(Generic[T]): ... + Alias = Optional[T] + + def test_typevar_none(self): + U = TypeVar('U') + U_None = TypeVar('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[T]: ... + T, = X.__type_params__ + self.assertIs(T.__default__, NoDefault) + self.assertFalse(T.has_default()) + + def test_paramspec(self): + P = ParamSpec('P', default=(str, int)) + self.assertEqual(P.__default__, (str, int)) + self.assertTrue(P.has_default()) + self.assertIsInstance(P, ParamSpec) + + class A(Generic[P]): ... + Alias = typing.Callable[P, None] + + P_default = ParamSpec('P_default', default=...) + self.assertIs(P_default.__default__, ...) + + def test_paramspec_none(self): + U = ParamSpec('U') + U_None = ParamSpec('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**P]: ... + P, = X.__type_params__ + self.assertIs(P.__default__, NoDefault) + self.assertFalse(P.has_default()) + + def test_typevartuple(self): + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + self.assertTrue(Ts.has_default()) + self.assertIsInstance(Ts, TypeVarTuple) + + class A(Generic[Unpack[Ts]]): ... + Alias = Optional[Unpack[Ts]] + + def test_typevartuple_specialization(self): + T = TypeVar("T") + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, Unpack[Ts]]): ... + self.assertEqual(A[float].__args__, (float, str, int)) + self.assertEqual(A[float, range].__args__, (float, range)) + self.assertEqual(A[float, *tuple[int, ...]].__args__, (float, *tuple[int, ...])) + + def test_typevar_and_typevartuple_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, U, Unpack[Ts]]): ... + self.assertEqual(A[int].__args__, (int, float, str, int)) + self.assertEqual(A[int, str].__args__, (int, str, str, int)) + self.assertEqual(A[int, str, range].__args__, (int, str, range)) + self.assertEqual(A[int, str, *tuple[int, ...]].__args__, (int, str, *tuple[int, ...])) + + def test_no_default_after_typevar_tuple(self): + T = TypeVar("T", default=int) + Ts = TypeVarTuple("Ts") + Ts_default = TypeVarTuple("Ts_default", default=Unpack[Tuple[str, int]]) + + with self.assertRaises(TypeError): + class X(Generic[*Ts, T]): ... + + with self.assertRaises(TypeError): + class Y(Generic[*Ts_default, T]): ... + + def test_paramspec_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P]): ... + self.assertEqual(A[float].__args__, (float, (str, int))) + self.assertEqual(A[float, [range]].__args__, (float, (range,))) + + def test_typevar_and_paramspec_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, U, P]): ... + self.assertEqual(A[float].__args__, (float, float, (str, int))) + self.assertEqual(A[float, int].__args__, (float, int, (str, int))) + self.assertEqual(A[float, int, [range]].__args__, (float, int, (range,))) + + def test_paramspec_and_typevar_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, (str, int), float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + def test_typevartuple_none(self): + U = TypeVarTuple('U') + U_None = TypeVarTuple('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**Ts]: ... + Ts, = X.__type_params__ + self.assertIs(Ts.__default__, NoDefault) + self.assertFalse(Ts.has_default()) + + def test_no_default_after_non_default(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + + with self.assertRaisesRegex( + TypeError, r"Type parameter ~T without a default follows type parameter with a default" + ): + Test = Generic[DefaultStrT, T] + + def test_need_more_params(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + U = TypeVar('U') + + class A(Generic[T, U, DefaultStrT]): ... + A[int, bool] + A[int, bool, str] + + with self.assertRaisesRegex( + TypeError, r"Too few arguments for .+; actual 1, expected at least 2" + ): + Test = A[int] + + def test_pickle(self): + global U, U_co, U_contra, U_default # pickle wants to reference the class by name + U = TypeVar('U') + U_co = TypeVar('U_co', covariant=True) + U_contra = TypeVar('U_contra', contravariant=True) + U_default = TypeVar('U_default', default=int) + for proto in range(pickle.HIGHEST_PROTOCOL): + for typevar in (U, U_co, U_contra, U_default): + z = pickle.loads(pickle.dumps(typevar, proto)) + self.assertEqual(z.__name__, typevar.__name__) + self.assertEqual(z.__covariant__, typevar.__covariant__) + self.assertEqual(z.__contravariant__, typevar.__contravariant__) + self.assertEqual(z.__bound__, typevar.__bound__) + self.assertEqual(z.__default__, typevar.__default__) + + def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: """Renders templates with possible combinations of replacements. @@ -6140,6 +6308,31 @@ def test_or(self): self.assertEqual(X | "x", Union[X, "x"]) self.assertEqual("x" | X, Union["x", X]) + def test_deprecation_for_no_type_params_passed_to__evaluate(self): + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing._eval_type' is deprecated" + ) + ) as cm: + self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) + + self.assertEqual(cm.filename, __file__) + + f = ForwardRef("int") + + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.ForwardRef._evaluate' is deprecated" + ) + ) as cm: + self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) + + self.assertEqual(cm.filename, __file__) + @lru_cache() def cached_func(x, y): @@ -7121,6 +7314,17 @@ def foo(): g = foo() self.assertIsSubclass(type(g), typing.Generator) + def test_generator_default(self): + g1 = typing.Generator[int] + g2 = typing.Generator[int, None, None] + self.assertEqual(get_args(g1), (int, type(None), type(None))) + self.assertEqual(get_args(g1), get_args(g2)) + + g3 = typing.Generator[int, float] + g4 = typing.Generator[int, float, None] + self.assertEqual(get_args(g3), (int, float, type(None))) + self.assertEqual(get_args(g3), get_args(g4)) + def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator() @@ -7307,6 +7511,15 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + def test_contextmanager_type_params(self): + cm1 = typing.ContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.ContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) + + type gen_cm[T1, T2] = typing.ContextManager[T1, T2] + self.assertEqual(get_args(gen_cm.__value__[int, None]), (int, types.NoneType)) + def test_async_contextmanager(self): class NotACM: pass @@ -7318,11 +7531,17 @@ def manager(): cm = manager() self.assertNotIsInstance(cm, typing.AsyncContextManager) - self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int, bool | None)) with self.assertRaises(TypeError): isinstance(42, typing.AsyncContextManager[int]) with self.assertRaises(TypeError): - typing.AsyncContextManager[int, str] + typing.AsyncContextManager[int, str, float] + + def test_asynccontextmanager_type_params(self): + cm1 = typing.AsyncContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.AsyncContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) class TypeTests(BaseTestCase): @@ -9749,7 +9968,7 @@ def test_special_attrs(self): typing.ValuesView: 'ValuesView', # Subscribed ABC classes typing.AbstractSet[Any]: 'AbstractSet', - typing.AsyncContextManager[Any]: 'AsyncContextManager', + typing.AsyncContextManager[Any, Any]: 'AsyncContextManager', typing.AsyncGenerator[Any, Any]: 'AsyncGenerator', typing.AsyncIterable[Any]: 'AsyncIterable', typing.AsyncIterator[Any]: 'AsyncIterator', @@ -9759,7 +9978,7 @@ def test_special_attrs(self): typing.ChainMap[Any, Any]: 'ChainMap', typing.Collection[Any]: 'Collection', typing.Container[Any]: 'Container', - typing.ContextManager[Any]: 'ContextManager', + typing.ContextManager[Any, Any]: 'ContextManager', typing.Coroutine[Any, Any, Any]: 'Coroutine', typing.Counter[Any]: 'Counter', typing.DefaultDict[Any, Any]: 'DefaultDict', @@ -10001,6 +10220,26 @@ class CustomerModel(ModelBase, init=False): self.assertIsInstance(CustomerModel, Decorated) +class NoDefaultTests(BaseTestCase): + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(NoDefault, proto) + loaded = pickle.loads(s) + self.assertIs(NoDefault, loaded) + + def test_constructor(self): + self.assertIs(NoDefault, type(NoDefault)()) + with self.assertRaises(TypeError): + NoDefault(1) + + def test_repr(self): + self.assertEqual(repr(NoDefault), 'typing.NoDefault') + + def test_no_call(self): + with self.assertRaises(TypeError): + NoDefault() + + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index b81b3049d56bf8..77f6f1eb4b76b9 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -115,6 +115,19 @@ def f(): pass with self.assertRaises(TypeError): mock() + def test_create_autospec_should_be_configurable_by_kwargs(self): + """If kwargs are given to configure mock, the function must configure + the parent mock during initialization.""" + mocked_result = 'mocked value' + class_mock = create_autospec(spec=Something, **{ + 'return_value.meth.side_effect': [ValueError, DEFAULT], + 'return_value.meth.return_value': mocked_result}) + with self.assertRaises(ValueError): + class_mock().meth(a=None, b=None, c=None) + self.assertEqual(class_mock().meth(a=None, b=None, c=None), mocked_result) + # Only the parent mock should be configurable because the user will + # pass kwargs with respect to the parent mock. + self.assertEqual(class_mock().return_value.meth.side_effect, None) def test_repr(self): mock = Mock(name='foo') diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index d0046d702a53f4..be75fda7826af1 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest +import test from test.test_unittest.testmock import support from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -1728,6 +1730,71 @@ def test(mock): 'exception traceback not propagated') + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index bb15f64c59dbd1..35394f29fbe49d 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -673,6 +673,20 @@ def test_quote_in_format_spec(self): self.check_ast_roundtrip("""f'\\'{x:\\"}' """) self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -723,6 +737,20 @@ def test_function_with_type_params_and_bound(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + def test_async_function(self): node = ast.AsyncFunctionDef( name="f", @@ -746,6 +774,20 @@ def test_async_function_with_type_params(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + class DirectoryTestCase(ASTTestCase): """Test roundtrip behaviour on all files in Lib and Lib/test.""" diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 6febb491788b42..eed0599642edfb 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1402,6 +1402,15 @@ def http_open(self, req): request = handler.last_buf self.assertTrue(request.startswith(expected), repr(request)) + def test_redirect_head_request(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + req = Request(from_url, method="HEAD") + fp = MockFile() + new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) + self.assertEqual(new_req.get_method(), "HEAD") + def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index df90647baee31e..16da24d7805b56 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -17,6 +17,7 @@ from test.support import gc_collect from test.support import import_helper from test.support import threading_helper +from test.support import is_wasi, Py_DEBUG # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -960,6 +961,7 @@ def test_hashing(self): self.assertEqual(hash(a), hash(42)) self.assertRaises(TypeError, hash, b) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_trashcan_16602(self): # Issue #16602: when a weakref's target was part of a long # deallocation chain, the trashcan mechanism could delay clearing diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index ae49700294330c..e9c3218d2bb39e 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,8 +1,10 @@ import sys import os import marshal +import glob import importlib import importlib.util +import re import struct import time import unittest @@ -54,6 +56,7 @@ def module_path_to_dotted_name(path): TESTPACK2 = "ziptestpackage2" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = '.pyc' @@ -134,7 +137,9 @@ def getZip64Files(self): def doTest(self, expected_ext, files, *modules, **kw): self.makeZip(files, **kw) + self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) + def doTestWithPreBuiltZip(self, expected_ext, *modules, **kw): sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(".".join(modules)) @@ -810,6 +815,122 @@ def testZip64CruftAndComment(self): files = self.getZip64Files() self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1)) + def testZip64LargeFile(self): + support.requires( + "largefile", + f"test generates files >{0xFFFFFFFF} bytes and takes a long time " + "to run" + ) + + # N.B.: We do alot of gymnastics below in the ZIP_STORED case to save + # and reconstruct a sparse zip on systems that support sparse files. + # Instead of creating a ~8GB zip file mainly consisting of null bytes + # for every run of the test, we create the zip once and save off the + # non-null portions of the resulting file as data blobs with offsets + # that allow re-creating the zip file sparsely. This drops disk space + # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2 + # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the + # bullet. The resulting zip file is ~8MB of non-null data; so the sparse + # trick doesn't work and would result in that full ~8MB zip data file + # being checked in to source control. + parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part" + full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob) + pre_built_zip_parts = glob.glob(full_parts_glob) + + self.addCleanup(os_helper.unlink, TEMP_ZIP) + if not pre_built_zip_parts: + if self.compression != ZIP_STORED: + support.requires( + "cpu", + "test requires a lot of CPU for compression." + ) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + f.write(b"data") + f.write(os.linesep.encode()) + f.seek(0xffff_ffff, os.SEEK_CUR) + f.write(os.linesep.encode()) + os.utime(os_helper.TESTFN, (0.0, 0.0)) + with ZipFile( + TEMP_ZIP, + "w", + compression=self.compression, + strict_timestamps=False + ) as z: + z.write(os_helper.TESTFN, "data1") + z.writestr( + ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src + ) + z.write(os_helper.TESTFN, "data2") + + # This "works" but relies on the zip format having a non-empty + # final page due to the trailing central directory to wind up with + # the correct length file. + def make_sparse_zip_parts(name): + empty_page = b"\0" * 4096 + with open(name, "rb") as f: + part = None + try: + while True: + offset = f.tell() + data = f.read(len(empty_page)) + if not data: + break + if data != empty_page: + if not part: + part_fullname = os.path.join( + TEST_DATA_DIR, + f"sparse-zip64-c{self.compression:d}-" + f"{offset:#011x}.part", + ) + os.makedirs( + os.path.dirname(part_fullname), + exist_ok=True + ) + part = open(part_fullname, "wb") + print("Created", part_fullname) + part.write(data) + else: + if part: + part.close() + part = None + finally: + if part: + part.close() + + if self.compression == ZIP_STORED: + print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:") + make_sparse_zip_parts(TEMP_ZIP) + + else: + def extract_offset(name): + if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name): + return int(m.group(1), base=16) + raise ValueError(f"{name=} does not fit expected pattern.") + offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts] + with open(TEMP_ZIP, "wb") as f: + for offset, part_fn in sorted(offset_parts): + with open(part_fn, "rb") as part: + f.seek(offset, os.SEEK_SET) + f.write(part.read()) + # Confirm that the reconstructed zip file works and looks right. + with ZipFile(TEMP_ZIP, "r") as z: + self.assertEqual( + z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0) + ) + self.assertEqual( + z.read("module.py"), test_src.encode(), + msg=f"Recreate {full_parts_glob}, unexpected contents." + ) + def assertDataEntry(name): + zinfo = z.getinfo(name) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertGreater(zinfo.file_size, 0xffff_ffff) + assertDataEntry("data1") + assertDataEntry("data2") + + self.doTestWithPreBuiltZip(".py", "module") + @support.requires_zlib() class CompressedZipImportTestCase(UncompressedZipImportTestCase): diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part new file mode 100644 index 00000000000000..c6beae8e2552d6 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part new file mode 100644 index 00000000000000..74ab03b4648948 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part new file mode 100644 index 00000000000000..9769a404f675d4 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part differ diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 70a1ed46fd0774..daecf4eb2ea522 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -40,7 +40,8 @@ from tkinter.constants import * import re -wantobjects = 1 +wantobjects = 2 +_debug = False # set to True to print executed Tcl/Tk commands TkVersion = float(_tkinter.TK_VERSION) TclVersion = float(_tkinter.TCL_VERSION) @@ -69,7 +70,10 @@ def _stringify(value): else: value = '{%s}' % _join(value) else: - value = str(value) + if isinstance(value, bytes): + value = str(value, 'latin1') + else: + value = str(value) if not value: value = '{}' elif _magic_re.search(value): @@ -411,7 +415,6 @@ def __del__(self): self._tk.globalunsetvar(self._name) if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self._tk.deletecommand(name) self._tclCommands = None @@ -683,7 +686,6 @@ def destroy(self): this widget in the Tcl interpreter.""" if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) self._tclCommands = None @@ -691,7 +693,6 @@ def deletecommand(self, name): """Internal function. Delete the Tcl command provided in NAME.""" - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) try: self._tclCommands.remove(name) @@ -1761,7 +1762,10 @@ def getint_event(s): try: e.type = EventType(T) except ValueError: - e.type = T + try: + e.type = EventType(str(T)) # can be int + except ValueError: + e.type = T try: e.widget = self._nametowidget(W) except KeyError: @@ -2450,6 +2454,8 @@ def __init__(self, screenName=None, baseName=None, className='Tk', baseName = baseName + ext interactive = False self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if _debug: + self.tk.settrace(_print_command) if useTk: self._loadtk() if not sys.flags.ignore_environment: @@ -2536,6 +2542,14 @@ def __getattr__(self, attr): "Delegate attribute access to the interpreter object" return getattr(self.tk, attr) + +def _print_command(cmd, *, file=sys.stderr): + # Print executed Tcl/Tk commands. + assert isinstance(cmd, tuple) + cmd = _join(cmd) + print(cmd, file=file) + + # Ideally, the classes Pack, Place and Grid disappear, the # pack/place/grid methods are defined on the Widget class, and # everybody uses w.pack_whatever(...) instead of Pack.whatever(w, @@ -4278,33 +4292,112 @@ def cget(self, option): def __getitem__(self, key): return self.tk.call(self.name, 'cget', '-' + key) - # XXX copy -from, -to, ...? - def copy(self): - """Return a new PhotoImage with the same image as this widget.""" + def copy(self, *, from_coords=None, zoom=None, subsample=None): + """Return a new PhotoImage with the same image as this widget. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + """ destImage = PhotoImage(master=self.tk) - self.tk.call(destImage, 'copy', self.name) + destImage.copy_replace(self, from_coords=from_coords, + zoom=zoom, subsample=subsample) return destImage - def zoom(self, x, y=''): + def zoom(self, x, y='', *, from_coords=None): """Return a new PhotoImage with the same image as this widget - but zoom it with a factor of x in the X direction and y in the Y - direction. If y is not given, the default value is the same as x. + but zoom it with a factor of X in the X direction and Y in the Y + direction. If Y is not given, the default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-zoom',x,y) - return destImage + return self.copy(zoom=(x, y), from_coords=from_coords) - def subsample(self, x, y=''): + def subsample(self, x, y='', *, from_coords=None): """Return a new PhotoImage based on the same image as this widget - but use only every Xth or Yth pixel. If y is not given, the - default value is the same as x. + but use only every Xth or Yth pixel. If Y is not given, the + default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-subsample',x,y) - return destImage + return self.copy(subsample=(x, y), from_coords=from_coords) + + def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, + zoom=None, subsample=None, compositingrule=None): + """Copy a region from the source image (which must be a PhotoImage) to + this image, possibly with pixel zooming and/or subsampling. If no + options are specified, this command copies the whole of the source + image into this image, starting at coordinates (0, 0). + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + The TO option specifies a rectangular sub-region of the destination + image to be affected. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is (x1,y1) plus the size of the source region + (after subsampling and zooming, if specified). If x2 and y2 are + specified, the source region will be replicated if necessary to fill + the destination region in a tiled fashion. + + If SHRINK is true, the size of the destination image should be + reduced, if necessary, so that the region being copied into is at + the bottom-right corner of the image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + + The COMPOSITINGRULE option specifies how transparent pixels in the + source image are combined with the destination image. When a + compositing rule of 'overlay' is set, the old contents of the + destination image are visible, as if the source image were printed + on a piece of transparent film and placed over the top of the + destination. When a compositing rule of 'set' is set, the old + contents of the destination image are discarded and the source image + is used as-is. The default compositing rule is 'overlay'. + """ + options = [] + if from_coords is not None: + options.extend(('-from', *from_coords)) + if to is not None: + options.extend(('-to', *to)) + if shrink: + options.append('-shrink') + if zoom is not None: + if not isinstance(zoom, (tuple, list)): + zoom = (zoom,) + options.extend(('-zoom', *zoom)) + if subsample is not None: + if not isinstance(subsample, (tuple, list)): + subsample = (subsample,) + options.extend(('-subsample', *subsample)) + if compositingrule: + options.extend(('-compositingrule', compositingrule)) + self.tk.call(self.name, 'copy', sourceImage, *options) def get(self, x, y): """Return the color (red, green, blue) of the pixel at X,Y.""" @@ -4319,17 +4412,117 @@ def put(self, data, to=None): to = to[1:] args = args + ('-to',) + tuple(to) self.tk.call(args) - # XXX read - - def write(self, filename, format=None, from_coords=None): - """Write image to file FILENAME in FORMAT starting from - position FROM_COORDS.""" - args = (self.name, 'write', filename) - if format: - args = args + ('-format', format) - if from_coords: - args = args + ('-from',) + tuple(from_coords) - self.tk.call(args) + + def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + """Reads image data from the file named FILENAME into the image. + + The FORMAT option specifies the format of the image data in the + file. + + The FROM_COORDS option specifies a rectangular sub-region of the image + file data to be copied to the destination image. It must be a tuple + or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and + (x2, y2) specify diagonally opposite corners of the rectangle. If + x2 and y2 are not specified, the default value is the bottom-right + corner of the source image. The default, if this option is not + specified, is the whole of the image in the image file. + + The TO option specifies the coordinates of the top-left corner of + the region of the image into which data from filename are to be + read. The default is (0, 0). + + If SHRINK is true, the size of the destination image will be + reduced, if necessary, so that the region into which the image file + data are read is at the bottom-right corner of the image. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if shrink: + options += ('-shrink',) + if to is not None: + options += ('-to', *to) + self.tk.call(self.name, 'read', filename, *options) + + def write(self, filename, format=None, from_coords=None, *, + background=None, grayscale=False): + """Writes image data from the image to a file named FILENAME. + + The FORMAT option specifies the name of the image file format + handler to be used to write the data to the file. If this option + is not given, the format is guessed from the file extension. + + The FROM_COORDS option specifies a rectangular region of the image + to be written to the image file. It must be a tuple or a list of 1 + to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, + the region extends from (x1,y1) to the bottom-right corner of the + image. If all four coordinates are given, they specify diagonally + opposite corners of the rectangular region. The default, if this + option is not given, is the whole image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + self.tk.call(self.name, 'write', filename, *options) + + def data(self, format=None, *, from_coords=None, + background=None, grayscale=False): + """Returns image data. + + The FORMAT option specifies the name of the image file format + handler to be used. If this option is not given, this method uses + a format that consists of a tuple (one element per row) of strings + containings space separated (one element per pixel/column) colors + in “#RRGGBB” format (where RR is a pair of hexadecimal digits for + the red channel, GG for green, and BB for blue). + + The FROM_COORDS option specifies a rectangular region of the image + to be returned. It must be a tuple or a list of 1 to 4 integers + (x1, y1, x2, y2). If only x1 and y1 are specified, the region + extends from (x1,y1) to the bottom-right corner of the image. If + all four coordinates are given, they specify diagonally opposite + corners of the rectangular region, including (x1, y1) and excluding + (x2, y2). The default, if this option is not given, is the whole + image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + data = self.tk.call(self.name, 'data', *options) + if isinstance(data, str): # For wantobjects = 0. + if format is None: + data = self.tk.splitlist(data) + else: + data = bytes(data, 'latin1') + return data def transparency_get(self, x, y): """Return True if the pixel at x,y is transparent.""" diff --git a/Lib/traceback.py b/Lib/traceback.py index fccec0c71c3695..9401b461497cc1 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,7 +1,5 @@ """Extract, format and print information about Python stack traces.""" -import os -import io import collections.abc import itertools import linecache @@ -9,6 +7,8 @@ import textwrap import warnings from contextlib import suppress +import _colorize +from _colorize import ANSIColors __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', @@ -21,7 +21,6 @@ # Formatting and printing lists of traceback lines. # -_COLORIZE = True def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or @@ -133,46 +132,15 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _can_colorize(): - if sys.platform == "win32": - try: - import nt - if not nt._supports_virtual_terminal(): - return False - except (ImportError, AttributeError): - return False - if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": - return False - if os.environ.get("PYTHON_COLORS") == "1": - return True - if "NO_COLOR" in os.environ: - return False - if not _COLORIZE: - return False - if not sys.flags.ignore_environment: - if "FORCE_COLOR" in os.environ: - return True - if os.environ.get("TERM") == "dumb": - return False - - if not hasattr(sys.stderr, "fileno"): - return False - - try: - return os.isatty(sys.stderr.fileno()) - except io.UnsupportedOperation: - return sys.stderr.isatty() - def _print_exception_bltin(exc, /): file = sys.stderr if sys.stderr is not None else sys.__stderr__ - colorize = _can_colorize() + colorize = _colorize.can_colorize() return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -181,9 +149,10 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) def format_exception_only(exc, /, value=_sentinel, *, show_group=False): @@ -214,9 +183,9 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= end_char = "\n" if insert_final_newline else "" if colorize: if value is None or not valuestr: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}{end_char}" else: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}: {_ANSIColors.MAGENTA}{valuestr}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}: {ANSIColors.MAGENTA}{valuestr}{ANSIColors.RESET}{end_char}" else: if value is None or not valuestr: line = f"{etype}{end_char}" @@ -224,6 +193,7 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= line = f"{etype}: {valuestr}{end_char}" return line + def _safe_string(value, what, func=str): try: return func(value) @@ -449,17 +419,6 @@ def _get_code_position(code, instruction_index): _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. -class _ANSIColors: - RED = '\x1b[31m' - BOLD_RED = '\x1b[1;31m' - MAGENTA = '\x1b[35m' - BOLD_MAGENTA = '\x1b[1;35m' - GREEN = "\x1b[32m" - BOLD_GREEN = "\x1b[1;32m" - GREY = '\x1b[90m' - RESET = '\x1b[0m' - YELLOW = "\x1b[33m" - class StackSummary(list): """A list of FrameSummary objects, representing a stack of frames.""" @@ -564,15 +523,15 @@ def format_frame_summary(self, frame_summary, **kwargs): filename = "" if colorize: row.append(' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, filename, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.lineno, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.name, - _ANSIColors.RESET, + ANSIColors.RESET, ) ) else: @@ -617,13 +576,10 @@ def format_frame_summary(self, frame_summary, **kwargs): # attempt to parse for anchors anchors = None + show_carets = False with suppress(Exception): anchors = _extract_caret_anchors_from_line_segment(segment) - - # only use carets if there are anchors or the carets do not span all lines - show_carets = False - if anchors or all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): - show_carets = True + show_carets = self.should_show_carets(start_offset, end_offset, all_lines, anchors) result = [] @@ -699,11 +655,11 @@ def output_line(lineno): for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): caret_group = list(group) if color == "^": - colorized_line_parts.append(_ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) elif color == "~": - colorized_line_parts.append(_ANSIColors.RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) else: colorized_line_parts.append("".join(char for char, _ in caret_group)) colorized_carets_parts.append("".join(caret for _, caret in caret_group)) @@ -737,6 +693,37 @@ def output_line(lineno): return ''.join(row) + def should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + def format(self, **kwargs): """Format the stack ready for printing. @@ -1279,12 +1266,12 @@ def _format_syntax_error(self, stype, **kwargs): if self.lineno is not None: if colorize: yield ' File {}"{}"{}, line {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, self.filename or "", - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, self.lineno, - _ANSIColors.RESET, + ANSIColors.RESET, ) else: yield ' File "{}", line {}\n'.format( @@ -1324,11 +1311,11 @@ def _format_syntax_error(self, stype, **kwargs): # colorize from colno to end_colno ltext = ( ltext[:colno] + - _ANSIColors.BOLD_RED + ltext[colno:end_colno] + _ANSIColors.RESET + + ANSIColors.BOLD_RED + ltext[colno:end_colno] + ANSIColors.RESET + ltext[end_colno:] ) - start_color = _ANSIColors.BOLD_RED - end_color = _ANSIColors.RESET + start_color = ANSIColors.BOLD_RED + end_color = ANSIColors.RESET yield ' {}\n'.format(ltext) yield ' {}{}{}{}\n'.format( "".join(caretspace), @@ -1341,12 +1328,12 @@ def _format_syntax_error(self, stype, **kwargs): msg = self.msg or "" if colorize: yield "{}{}{}: {}{}{}{}\n".format( - _ANSIColors.BOLD_MAGENTA, + ANSIColors.BOLD_MAGENTA, stype, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, msg, - _ANSIColors.RESET, + ANSIColors.RESET, filename_suffix) else: yield "{}: {}{}\n".format(stype, msg, filename_suffix) @@ -1482,12 +1469,23 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): obj = exc_value.obj try: d = dir(obj) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] except Exception: return None elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) d = dir(mod) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] except Exception: return None else: diff --git a/Lib/typing.py b/Lib/typing.py index b3f4ba99f9ec21..8e61f50477bcc2 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -38,6 +38,7 @@ ParamSpecKwargs, TypeAliasType, Generic, + NoDefault, ) # Please keep __all__ alphabetized within each category. @@ -138,6 +139,7 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoDefault', 'NoReturn', 'NotRequired', 'overload', @@ -266,6 +268,10 @@ def _collect_parameters(args): >>> _collect_parameters((T, Callable[P, T])) (~T, ~P) """ + # required type parameter cannot appear after parameter with default + default_encountered = False + # or after TypeVarTuple + type_var_tuple_encountered = False parameters = [] for t in args: if isinstance(t, type): @@ -280,27 +286,58 @@ def _collect_parameters(args): parameters.append(collected) elif hasattr(t, '__typing_subst__'): if t not in parameters: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') + parameters.append(t) else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True for x in getattr(t, '__parameters__', ()): if x not in parameters: parameters.append(x) return tuple(parameters) -def _check_generic(cls, parameters, elen): +def _check_generic_specialization(cls, arguments): """Check correct count for parameters of a generic cls (internal helper). This gives a nice error message in case of count mismatch. """ - if not elen: + expected_len = len(cls.__parameters__) + if not expected_len: raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" - f" actual {alen}, expected {elen}") + actual_len = len(arguments) + if actual_len != expected_len: + # deal with defaults + if actual_len < expected_len: + # If the parameter at index `actual_len` in the parameters list + # has a default, then all parameters after it must also have + # one, because we validated as much in _collect_parameters(). + # That means that no error needs to be raised here, despite + # the number of arguments being passed not matching the number + # of parameters: all parameters that aren't explicitly + # specialized in this call are parameters with default values. + if cls.__parameters__[actual_len].has_default(): + return + + expected_len -= sum(p.has_default() for p in cls.__parameters__) + expect_val = f"at least {expected_len}" + else: + expect_val = expected_len + + raise TypeError(f"Too {'many' if actual_len > expected_len else 'few'} arguments" + f" for {cls}; actual {actual_len}, expected {expect_val}") + -def _unpack_args(args): +def _unpack_args(*args): newargs = [] for arg in args: subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) @@ -400,13 +437,38 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset()): +def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: + import warnings + + depr_message = ( + f"Failing to pass a value to the 'type_params' parameter " + f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " + f"when calling {funcname} on a stringified annotation " + f"that references a PEP 695 type parameter. " + f"It will be disallowed in Python 3.15." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) + + +class _Sentinel: + __slots__ = () + def __repr__(self): + return '' + + +_sentinel = _Sentinel() + + +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset()): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing._eval_type") + type_params = () if isinstance(t, ForwardRef): return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): @@ -981,7 +1043,10 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): self.__forward_is_class__ = is_class self.__forward_module__ = module - def _evaluate(self, globalns, localns, type_params, *, recursive_guard): + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing.ForwardRef._evaluate") + type_params = () if self.__forward_arg__ in recursive_guard: return self if not self.__forward_evaluated__ or localns is not globalns: @@ -1089,11 +1154,15 @@ def _typevartuple_prepare_subst(self, alias, args): elif left + right > alen: raise TypeError(f"Too few arguments for {alias};" f" actual {alen}, expected at least {plen-1}") + if left == alen - right and self.has_default(): + replacement = _unpack_args(self.__default__) + else: + replacement = args[left: alen - right] return ( *args[:left], *([fillarg]*(typevartuple_index - left)), - tuple(args[left: alen - right]), + replacement, *([fillarg]*(plen - right - left - typevartuple_index - 1)), *args[alen - right:], ) @@ -1111,6 +1180,8 @@ def _paramspec_subst(self, arg): def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) + if i == len(args) and self.has_default(): + args = [*args, self.__default__] if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1124,33 +1195,33 @@ def _paramspec_prepare_subst(self, alias, args): @_tp_cache -def _generic_class_getitem(cls, params): +def _generic_class_getitem(cls, args): """Parameterizes a generic class. At least, parameterizing a generic class is the *main* thing this method does. For example, for some generic class `Foo`, this is called when we - do `Foo[int]` - there, with `cls=Foo` and `params=int`. + do `Foo[int]` - there, with `cls=Foo` and `args=int`. However, note that this method is also called when defining generic classes in the first place with `class Foo(Generic[T]): ...`. """ - if not isinstance(params, tuple): - params = (params,) + if not isinstance(args, tuple): + args = (args,) - params = tuple(_type_convert(p) for p in params) + args = tuple(_type_convert(p) for p in args) is_generic_or_protocol = cls in (Generic, Protocol) if is_generic_or_protocol: # Generic and Protocol can only be subscripted with unique type variables. - if not params: + if not args: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty" ) - if not all(_is_typevar_like(p) for p in params): + if not all(_is_typevar_like(p) for p in args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be type variables " f"or parameter specification variables.") - if len(set(params)) != len(params): + if len(set(args)) != len(args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be unique") else: @@ -1158,18 +1229,18 @@ def _generic_class_getitem(cls, params): for param in cls.__parameters__: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: - params = prepare(cls, params) - _check_generic(cls, params, len(cls.__parameters__)) + args = prepare(cls, args) + _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, params): + for param, new_arg in zip(cls.__parameters__, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: new_args.append(new_arg) - params = tuple(new_args) + args = tuple(new_args) - return _GenericAlias(cls, params) + return _GenericAlias(cls, args) def _generic_init_subclass(cls, *args, **kwargs): @@ -1285,7 +1356,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', '_defaults'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1390,8 +1461,7 @@ def __getitem__(self, args): # Preprocess `args`. if not isinstance(args, tuple): args = (args,) - args = tuple(_type_convert(p) for p in args) - args = _unpack_args(args) + args = _unpack_args(*(_type_convert(p) for p in args)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) return r @@ -1536,11 +1606,12 @@ def __iter__(self): # parameters are accepted (needs custom __getitem__). class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): if name is None: name = origin.__name__ super().__init__(origin, inst=inst, name=name) self._nparams = nparams + self._defaults = defaults if origin.__module__ == 'builtins': self.__doc__ = f'A generic version of {origin.__qualname__}.' else: @@ -1552,7 +1623,22 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, self._nparams) + if (self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) + actual_len = len(params) + + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" + f" actual {actual_len}, expected {expected}") return self.copy_with(params) def copy_with(self, params): @@ -1813,7 +1899,7 @@ class _TypingEllipsis: '__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', '__static_attributes__', + '__match_args__', '__static_attributes__', '__firstlineno__', }) # These special attributes will be not collected as protocol members. @@ -2766,8 +2852,8 @@ class Other(Leaf): # Error reported by type checker OrderedDict = _alias(collections.OrderedDict, 2) Counter = _alias(collections.Counter, 1) ChainMap = _alias(collections.ChainMap, 2) -Generator = _alias(collections.abc.Generator, 3) -AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2) +Generator = _alias(collections.abc.Generator, 3, defaults=(types.NoneType, types.NoneType)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2, defaults=(types.NoneType,)) Type = _alias(type, 1, inst=False, name='Type') Type.__doc__ = \ """Deprecated alias to builtins.type. @@ -2940,15 +3026,6 @@ def __new__(cls, typename, bases, ns): return nm_tpl -class _Sentinel: - __slots__ = () - def __repr__(self): - return '' - - -_sentinel = _Sentinel() - - def NamedTuple(typename, fields=_sentinel, /, **kwargs): """Typed version of namedtuple. @@ -3706,7 +3783,7 @@ def __getattr__(attr): obj = _alias(getattr(re, attr), 1) elif attr in {"ContextManager", "AsyncContextManager"}: import contextlib - obj = _alias(getattr(contextlib, f"Abstract{attr}"), 1, name=attr) + obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) else: raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") globals()[attr] = obj diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 1799e9bbf58592..a2634b6164062a 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -2788,8 +2788,8 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if _parent is not None and not instance: _parent._mock_children[_name] = mock - wrapped = kwargs.get('wraps') - + # Pop wraps from kwargs because it must not be passed to configure_mock. + wrapped = kwargs.pop('wraps', None) if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, _name='()', _parent=mock, @@ -2814,12 +2814,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, except AttributeError: continue - kwargs = {'spec': original} + child_kwargs = {'spec': original} # Wrap child attributes also. if wrapped and hasattr(wrapped, entry): - kwargs.update(wraps=original) + child_kwargs.update(wraps=original) if spec_set: - kwargs = {'spec_set': original} + child_kwargs = {'spec_set': original} if not isinstance(original, FunctionTypes): new = _SpecState(original, spec_set, mock, entry, instance) @@ -2830,14 +2830,13 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, parent = mock.mock skipfirst = _must_skip(spec, entry, is_type) - kwargs['_eat_self'] = skipfirst + child_kwargs['_eat_self'] = skipfirst if iscoroutinefunction(original): child_klass = AsyncMock else: child_klass = MagicMock new = child_klass(parent=parent, name=entry, _new_name=entry, - _new_parent=parent, - **kwargs) + _new_parent=parent, **child_kwargs) mock._mock_children[entry] = new new.return_value = child_klass() _check_signature(original, new, skipfirst=skipfirst) @@ -2848,6 +2847,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # setting as an instance attribute? if isinstance(new, FunctionTypes): setattr(mock, entry, new) + # kwargs are passed with respect to the parent mock so, they are not used + # for creating return_value of the parent mock. So, this condition + # should be true only for the parent mock if kwargs are given. + if _is_instance_mock(mock) and kwargs: + mock.configure_mock(**kwargs) return mock diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index d22af6618d80f1..ac6719ce854182 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -650,6 +650,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS} return Request(newurl, + method="HEAD" if m == "HEAD" else "GET", headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True) diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 4c167563b6b762..79ebb777354e03 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -262,7 +262,7 @@ class Path: >>> str(path.parent) 'mem' - If the zipfile has no filename, such attribtues are not + If the zipfile has no filename, such attributes are not valid and accessing them will raise an Exception. >>> zf.filename = None diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 21d2dca46f569b..4e41d109865e85 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -517,8 +517,9 @@ def _read_directory(archive): num_extra_values = (len(extra_data) - 4) // 8 if num_extra_values > 3: raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) - values = struct.unpack_from(f"<{min(num_extra_values, 3)}Q", - extra_data, offset=4) + import struct + values = list(struct.unpack_from(f"<{min(num_extra_values, 3)}Q", + extra_data, offset=4)) # N.b. Here be dragons: the ordering of these is different than # the header fields, and it's really easy to get it wrong since diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index 4f74e7dc00520a..e44e74f3a49234 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -17,7 +17,7 @@ Gatekeeper download quarantine, the final package must be signed with a valid Apple Developer ID certificate using productsign. Starting with macOS 10.15 Catalina, Gatekeeper now also requires that installer packages are submitted to and pass Apple's automated -notarization service using the altool command. To pass notarization, +notarization service using the ``notarytool`` command. To pass notarization, the binaries included in the package must be built with at least the macOS 10.9 SDK, must now be signed with the codesign utility, and executables must opt in to the hardened run time option with @@ -27,7 +27,7 @@ available in the on-line Apple Developer Documentation and man pages. A goal of PSF-provided (python.org) Python binaries for macOS is to support a wide-range of operating system releases with one set of binaries. Currently, the oldest release supported by python.org -binaries is macOS 10.9; it is still possible to build Python and +binaries is macOS 10.9; it should still be possible to build Python and Python installers on older versions of macOS but we not regularly test on those systems nor provide binaries for them. @@ -49,20 +49,17 @@ Starting with 3.9.1, Python fully supports macOS "weaklinking", meaning it is now possible to build a Python on a current macOS version with a deployment target of an earlier macOS system. For 3.9.1 and later systems, we provide a "macOS 64-bit universal2 installer" -variant, currently build on macOS 11 Big Sur with fat binaries +variant, currently built on macOS 11 Big Sur with fat binaries natively supporting both Apple Silicon (arm64) and Intel-64 (x86_64) Macs running macOS 10.9 or later. -The legacy "macOS 64-bit Intel installer" variant is expected to -be retired prior to the end of 3.9.x support. - build-installer.py requires Apple Developer tools, either from the Command Line Tools package or from a full Xcode installation. You should use the most recent version of either for the operating system version in use. (One notable exception: on macOS 10.6, Snow Leopard, use Xcode 3, not Xcode 4 which was released later in the 10.6 support cycle.) build-installer.py also must be run -with recent versions of Python 3.x or 2.7. On older systems, +with recent versions of Python 3.x. On older systems, due to changes in TLS practices, it may be easier to manually download and cache third-party source distributions used by build-installer.py rather than have it attempt to automatically @@ -76,12 +73,12 @@ download them. - builds the following third-party libraries - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 + * OpenSSL 3.0.x + * Tcl/Tk 8.6.x * NCurses * SQLite * XZ - * libffi + * mpdecimal - uses system-supplied versions of third-party libraries @@ -97,35 +94,6 @@ download them. * ``MACOSX_DEPLOYMENT_TARGET=10.9`` * Apple ``clang`` -2. legacy Intel 64-bit, x86_64, for OS X 10.9 (and later):: - - /path/to/bootstrap/python3 build-installer.py \ - --universal-archs=intel-64 \ - --dep-target=10.9 - - - builds the following third-party libraries - - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 - * NCurses - * SQLite - * XZ - * libffi - - - uses system-supplied versions of third-party libraries - - * readline module links with Apple BSD editline (libedit) - * zlib - * bz2 - - - recommended build environment: - - * Mac OS X 10.9.5 - * Xcode Command Line Tools 6.2 - * ``MacOSX10.9`` SDK - * ``MACOSX_DEPLOYMENT_TARGET=10.9`` - * Apple ``clang`` - General Prerequisites --------------------- diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 286df4862793fb..8386e407f49aa3 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -378,6 +378,15 @@ def library_recipes(): '--disable-dependency-tracking', ] ), + dict( + name="libmpdec 4.0.0", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", + checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + configure_pre=[ + "--disable-cxx", + "MACHINE=universal", + ] + ), ]) if not PYTHON_3: @@ -1150,6 +1159,7 @@ def buildPython(): print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues") runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-mimalloc=no " + "--with-system-libmpdec " "--with-universal-archs=%s " "%s " "%s " diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index efd76b9b1ae64b..ee5ba4707dfea4 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 @@ -69,7 +69,7 @@ Due to new security checks on macOS 10.15 Catalina, when launching IDLE macOS ma \f1\b \ul Apple Silicon Mac support\ \f0\b0 \ulnone \ -On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the +On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the \f4 python3-intel64 \f0 command instead of just \f4 python3 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 79851e1f4a69cc..49d6e22286be26 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -26,5 +26,5 @@ At the end of this install, click on \ \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Makefile.pre.in b/Makefile.pre.in index 0e52e10602cf85..74a438b015a97d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -488,6 +488,7 @@ PYTHON_OBJS= \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ + Python/perf_jit_trampoline.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ @@ -1022,6 +1023,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/monitoring.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ @@ -1091,6 +1093,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/longobject.h \ $(srcdir)/Include/cpython/memoryobject.h \ $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/monitoring.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ $(srcdir)/Include/cpython/odictobject.h \ @@ -1225,6 +1228,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_stackref.h \ $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ @@ -2036,7 +2040,7 @@ testuniversal: all # a full Xcode install that has an iPhone SE (3rd edition) simulator available. # This must be run *after* a `make install` has completed the build. The # `--with-framework-name` argument *cannot* be used when configuring the build. -XCFOLDER=iOSTestbed.$(MULTIARCH).$(shell date +%s) +XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult .PHONY: testios testios: @@ -2063,7 +2067,7 @@ testios: # Run the test suite for the Xcode project, targeting the iOS simulator. # If the suite fails, touch a file in the test folder as a marker - if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) ; then \ + if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \ touch $(XCFOLDER)/failed; \ fi @@ -2335,6 +2339,7 @@ LIBSUBDIRS= asyncio \ xmlrpc \ zipfile zipfile/_path \ zoneinfo \ + _pyrepl \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ @@ -2380,6 +2385,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/circular_imports/subpkg2/parent \ test/test_import/data/package \ test/test_import/data/package2 \ + test/test_import/data/package3 \ + test/test_import/data/package4 \ test/test_import/data/unwritable \ test/test_importlib \ test/test_importlib/builtin \ @@ -2476,7 +2483,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/typinganndata \ test/wheeldata \ test/xmltestdata \ - test/xmltestdata/c14n-20 + test/xmltestdata/c14n-20 \ + test/zipimport_data COMPILEALL_OPTS=-j0 diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst new file mode 100644 index 00000000000000..8d49ff06efd07b --- /dev/null +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -0,0 +1,1670 @@ +.. date: 2024-03-27-13-50-02 +.. gh-issue: 116741 +.. nonce: ZoGryG +.. release date: 2024-05-08 +.. section: Security + +Update bundled libexpat to 2.6.2 + +.. + +.. date: 2024-03-25-21-25-28 +.. gh-issue: 117233 +.. nonce: E4CyI_ +.. section: Security + +Detect BLAKE2, SHA3, Shake, & truncated SHA512 support in the OpenSSL-ish +libcrypto library at build time. This allows :mod:`hashlib` to be used with +libraries that do not to support every algorithm that upstream OpenSSL does. + +.. + +.. date: 2024-05-07-01-39-24 +.. gh-issue: 118414 +.. nonce: G5GG7l +.. section: Core and Builtins + +Add instrumented opcodes to YIELD_VALUE assertion for tracing cases. + +.. + +.. date: 2024-05-06-10-57-54 +.. gh-issue: 117953 +.. nonce: DqCzIs +.. section: Core and Builtins + +When a builtin or extension module is imported for the first time, while a +subinterpreter is active, the module's init function is now run by the main +interpreter first before import continues in the subinterpreter. +Consequently, single-phase init modules now fail in an isolated +subinterpreter without the init function running under that interpreter, +whereas before it would run under the subinterpreter *before* failing, +potentially leaving behind global state and callbacks and otherwise leaving +the module in an inconsistent state. + +.. + +.. date: 2024-05-05-12-04-02 +.. gh-issue: 117549 +.. nonce: kITawD +.. section: Core and Builtins + +Don't use designated initializer syntax in inline functions in internal +headers. They cause problems for C++ or MSVC users who aren't yet using the +latest C++ standard (C++20). While internal, pycore_backoff.h, is included +(indirectly, via pycore_code.h) by some key 3rd party software that does so +for speed. + +.. + +.. date: 2024-05-03-18-01-26 +.. gh-issue: 95382 +.. nonce: 73FSEv +.. section: Core and Builtins + +Improve performance of :func:`json.dumps` and :func:`json.dump` when using +the argument *indent*. Depending on the data the encoding using +:func:`json.dumps` with *indent* can be up to 2 to 3 times faster. + +.. + +.. date: 2024-05-03-17-49-37 +.. gh-issue: 116322 +.. nonce: Gy6M4j +.. section: Core and Builtins + +In ``--disable-gil`` builds, the GIL will be enabled while loading C +extension modules. If the module indicates that it supports running without +the GIL, the GIL will be disabled once loading is complete. Otherwise, the +GIL will remain enabled for the remainder of the interpreter's lifetime. +This behavior does not apply if the GIL has been explicitly enabled or +disabled with ``PYTHON_GIL`` or ``-Xgil``. + +.. + +.. date: 2024-05-02-21-19-35 +.. gh-issue: 118513 +.. nonce: qHODjb +.. section: Core and Builtins + +Fix incorrect :exc:`UnboundLocalError` when two comprehensions in the same +function both reference the same name, and in one comprehension the name is +bound while in the other it's an implicit global. + +.. + +.. date: 2024-05-02-20-32-42 +.. gh-issue: 118518 +.. nonce: m-JbTi +.. section: Core and Builtins + +Allow the Linux perf support to work without frame pointers using perf's +advanced JIT support. The feature is activated when using the +``PYTHON_PERF_JIT_SUPPORT`` environment variable or when running Python with +``-Xperf_jit``. Patch by Pablo Galindo. + +.. + +.. date: 2024-05-02-16-04-51 +.. gh-issue: 117514 +.. nonce: CJiuC0 +.. section: Core and Builtins + +Add ``sys._is_gil_enabled()`` function that returns whether the GIL is +currently enabled. In the default build it always returns ``True`` because +the GIL is always enabled. In the free-threaded build, it may return +``True`` or ``False``. + +.. + +.. date: 2024-05-02-15-57-07 +.. gh-issue: 118164 +.. nonce: AF6kwI +.. section: Core and Builtins + +Break a loop between the Python implementation of the :mod:`decimal` module +and the Python code for integer to string conversion. Also optimize integer +to string conversion for values in the range from 9_000 to 135_000 decimal +digits. + +.. + +.. date: 2024-05-01-22-43-54 +.. gh-issue: 118473 +.. nonce: QIvq9R +.. section: Core and Builtins + +Fix :func:`sys.set_asyncgen_hooks` not to be partially set when raising +:exc:`TypeError`. + +.. + +.. date: 2024-05-01-17-12-36 +.. gh-issue: 118465 +.. nonce: g3Q8iE +.. section: Core and Builtins + +Compiler populates the new ``__firstlineno__`` field on a class with the +line number of the first line of the class definition. + +.. + +.. date: 2024-05-01-14-20-28 +.. gh-issue: 118492 +.. nonce: VUsSfn +.. section: Core and Builtins + +Fix an issue where the type cache can expose a previously accessed attribute +when a finalizer is run. + +.. + +.. date: 2024-05-01-07-06-48 +.. gh-issue: 117714 +.. nonce: Ip_dm5 +.. section: Core and Builtins + +update ``async_generator.athrow().close()`` and +``async_generator.asend().close()`` to close their section of the underlying +async generator + +.. + +.. date: 2024-04-28-00-41-17 +.. gh-issue: 111201 +.. nonce: cQsh5U +.. section: Core and Builtins + +The :term:`interactive` interpreter is now implemented in Python, which +allows for a number of new features like colors, multiline input, history +viewing, and paste mode. Contributed by Pablo Galindo, Łukasz Langa and +Lysandros Nikolaou based on code from the PyPy project. + +.. + +.. date: 2024-04-27-21-44-40 +.. gh-issue: 74929 +.. nonce: C2nESp +.. section: Core and Builtins + +Implement PEP 667: converted :attr:`FrameType.f_locals ` and +:c:func:`PyFrame_GetLocals` to return a write-through proxy object when the +frame refers to a function or comprehension. + +.. + +.. date: 2024-04-27-16-23-29 +.. gh-issue: 116767 +.. nonce: z9UFpr +.. section: Core and Builtins + +Fix crash in compiler on 'async with' that has many context managers. + +.. + +.. date: 2024-04-26-14-06-18 +.. gh-issue: 118335 +.. nonce: SRFsxO +.. section: Core and Builtins + +Change how to use the tier 2 interpreter. Instead of running Python with +``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this +choice is now made at build time by configuring with +``--enable-experimental-jit=interpreter``. + +**Beware!** This changes the environment variable to enable or disable +micro-ops to ``PYTHON_JIT``. The old ``PYTHON_UOPS`` is no longer used. + +.. + +.. date: 2024-04-26-05-38-18 +.. gh-issue: 118306 +.. nonce: vRUEOU +.. section: Core and Builtins + +Update JIT compilation to use LLVM 18 + +.. + +.. date: 2024-04-25-21-18-19 +.. gh-issue: 118160 +.. nonce: GH5SMc +.. section: Core and Builtins + +:ref:`Annotation scopes ` within classes can now contain +comprehensions. However, such comprehensions are not inlined into their +parent scope at runtime. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-25-12-55-47 +.. gh-issue: 118272 +.. nonce: 5ptjk_ +.. section: Core and Builtins + +Fix bug where ``generator.close`` does not free the generator frame's +locals. + +.. + +.. date: 2024-04-25-11-48-28 +.. gh-issue: 118216 +.. nonce: SVg700 +.. section: Core and Builtins + +Don't consider :mod:`__future__` imports with dots before the module name. + +.. + +.. date: 2024-04-22-08-34-28 +.. gh-issue: 118074 +.. nonce: 5_JnIa +.. section: Core and Builtins + +Make sure that the Executor objects in the COLD_EXITS array aren't assumed +to be GC-able (which would access bytes outside the object). + +.. + +.. date: 2024-04-20-20-30-15 +.. gh-issue: 107674 +.. nonce: GZPOP7 +.. section: Core and Builtins + +Lazy load frame line number to improve performance of tracing + +.. + +.. date: 2024-04-19-11-59-57 +.. gh-issue: 118082 +.. nonce: _FLuOT +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for imports without names, like in ``from +x import`` and ``import`` cases. It now points out to users that +:keyword:`import` expects at least one name after it. + +.. + +.. date: 2024-04-19-11-57-46 +.. gh-issue: 118090 +.. nonce: eGAQ0B +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for empty type param brackets. + +.. + +.. date: 2024-04-19-08-50-48 +.. gh-issue: 102511 +.. nonce: qDEB66 +.. section: Core and Builtins + +Speed up :func:`os.path.splitroot` with a native implementation. + +.. + +.. date: 2024-04-18-03-49-41 +.. gh-issue: 117958 +.. nonce: -EsfUs +.. section: Core and Builtins + +Added a ``get_jit_code()`` method to access JIT compiled machine code from +the UOp Executor when the experimental JIT is enabled. Patch by Anthony +Shaw. + +.. + +.. date: 2024-04-17-22-53-52 +.. gh-issue: 117901 +.. nonce: SsEcVJ +.. section: Core and Builtins + +Add option for compiler's codegen to save nested instruction sequences for +introspection. + +.. + +.. date: 2024-04-17-22-49-15 +.. gh-issue: 116622 +.. nonce: tthNUF +.. section: Core and Builtins + +Redirect stdout and stderr to system log when embedded in an Android app. + +.. + +.. date: 2024-04-17-17-52-32 +.. gh-issue: 109118 +.. nonce: q9iPEI +.. section: Core and Builtins + +:ref:`annotation scope ` within class scopes can now +contain lambdas. + +.. + +.. date: 2024-04-15-13-53-59 +.. gh-issue: 117894 +.. nonce: 8LpZ6m +.. section: Core and Builtins + +Prevent ``agen.aclose()`` objects being re-used after ``.throw()``. + +.. + +.. date: 2024-04-15-07-37-09 +.. gh-issue: 117881 +.. nonce: 07H0wI +.. section: Core and Builtins + +prevent concurrent access to an async generator via athrow().throw() or +asend().throw() + +.. + +.. date: 2024-04-13-18-59-25 +.. gh-issue: 115874 +.. nonce: c3xG-E +.. section: Core and Builtins + +Fixed a possible segfault during garbage collection of +``_asyncio.FutureIter`` objects + +.. + +.. date: 2024-04-13-16-55-53 +.. gh-issue: 117536 +.. nonce: xkVbfv +.. section: Core and Builtins + +Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)``. + +.. + +.. date: 2024-04-12-12-28-49 +.. gh-issue: 117755 +.. nonce: 6ct8kU +.. section: Core and Builtins + +Fix mimalloc allocator for huge memory allocation (around 8,589,934,592 GiB) +on s390x. Patch by Victor Stinner. + +.. + +.. date: 2024-04-12-11-19-18 +.. gh-issue: 117750 +.. nonce: YttK6h +.. section: Core and Builtins + +Fix issue where an object's dict would get out of sync with the object's +internal values when being cleared. ``obj.__dict__.clear()`` now clears the +internal values, but leaves the dict attached to the object. + +.. + +.. date: 2024-04-12-09-09-11 +.. gh-issue: 117431 +.. nonce: lxFEeJ +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!count` +* :meth:`!find` +* :meth:`!index` +* :meth:`!rfind` +* :meth:`!rindex` + +.. + +.. date: 2024-04-10-22-16-18 +.. gh-issue: 117709 +.. nonce: -_1YL0 +.. section: Core and Builtins + +Speed up calls to :func:`str` with positional-only argument, by using the +:pep:`590` ``vectorcall`` calling convention. Patch by Erlend Aasland. + +.. + +.. date: 2024-04-09-16-07-00 +.. gh-issue: 117680 +.. nonce: MRZ78K +.. section: Core and Builtins + +Give ``_PyInstructionSequence`` a Python interface and use it in tests. + +.. + +.. date: 2024-04-09-11-31-25 +.. gh-issue: 115776 +.. nonce: 5Nthd0 +.. section: Core and Builtins + +Statically allocated objects are, by definition, immortal so must be marked +as such regardless of whether they are in extension modules or not. + +.. + +.. date: 2024-04-08-19-30-38 +.. gh-issue: 117641 +.. nonce: oaBGSJ +.. section: Core and Builtins + +Speedup :func:`os.path.commonpath` on Unix. + +.. + +.. date: 2024-04-08-14-33-38 +.. gh-issue: 117636 +.. nonce: exnRKd +.. section: Core and Builtins + +Speedup :func:`os.path.join`. + +.. + +.. date: 2024-04-07-18-42-09 +.. gh-issue: 117607 +.. nonce: C978BD +.. section: Core and Builtins + +Speedup :func:`os.path.relpath`. + +.. + +.. date: 2024-03-30-00-37-53 +.. gh-issue: 117385 +.. nonce: h0OJti +.. section: Core and Builtins + +Remove unhandled ``PY_MONITORING_EVENT_BRANCH`` and +``PY_MONITORING_EVENT_EXCEPTION_HANDLED`` events from :func:`sys.settrace`. + +.. + +.. date: 2024-03-12-13-51-09 +.. gh-issue: 116322 +.. nonce: q8TcDQ +.. section: Core and Builtins + +Extension modules may indicate to the runtime that they can run without the +GIL. Multi-phase init modules do so by calling providing +``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init +modules call ``PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED)`` from +their init function. + +.. + +.. date: 2024-02-29-18-55-45 +.. gh-issue: 116129 +.. nonce: wsFnIq +.. section: Core and Builtins + +Implement :pep:`696`, adding support for defaults on type parameters. Patch +by Jelle Zijlstra. + +.. + +.. date: 2024-02-26-13-14-52 +.. gh-issue: 93502 +.. nonce: JMWRvA +.. section: Core and Builtins + +Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and +:c:func:`PyRefTracer_GetTracer`, that allows to track object creation and +destruction the same way the :mod:`tracemalloc` module does. Patch by Pablo +Galindo + +.. + +.. date: 2024-02-04-07-45-29 +.. gh-issue: 107674 +.. nonce: q8mCmi +.. section: Core and Builtins + +Improved the performance of :func:`sys.settrace` significantly + +.. + +.. date: 2024-01-07-03-38-34 +.. gh-issue: 95754 +.. nonce: aPjEBG +.. section: Core and Builtins + +Improve the error message when a script shadowing a module from the standard +library causes :exc:`AttributeError` to be raised. Similarly, improve the +error message when a script shadowing a third party module attempts to +access an attribute from that third party module while still initialising. + +.. + +.. date: 2023-12-03-18-21-59 +.. gh-issue: 99180 +.. nonce: 5m0V0q +.. section: Core and Builtins + +Elide uninformative traceback indicators in ``return`` and simple +``assignment`` statements. Patch by Pablo Galindo. + +.. + +.. date: 2023-06-18-00-27-57 +.. gh-issue: 105879 +.. nonce: dPw78k +.. section: Core and Builtins + +Allow the *globals* and *locals* arguments to :func:`exec` and :func:`eval` +to be passed as keywords. + +.. + +.. date: 2024-05-07-11-23-11 +.. gh-issue: 118418 +.. nonce: QPMdJm +.. section: Library + +A :exc:`DeprecationWarning` is now emitted if you fail to pass a value to +the new *type_params* parameter of ``typing._eval_type()`` or +``typing.ForwardRef._evaluate()``. (Using either of these private and +undocumented functions is discouraged to begin with, but failing to pass a +value to the ``type_params`` parameter may lead to incorrect behaviour on +Python 3.12 or newer.) + +.. + +.. date: 2024-05-06-18-13-02 +.. gh-issue: 118660 +.. nonce: n01Vb7 +.. section: Library + +Add an optional second type parameter to :class:`typing.ContextManager` and +:class:`typing.AsyncContextManager`, representing the return types of +:meth:`~object.__exit__` and :meth:`~object.__aexit__` respectively. This +parameter defaults to ``bool | None``. + +.. + +.. date: 2024-05-06-16-52-40 +.. gh-issue: 118650 +.. nonce: qKz5lp +.. section: Library + +The ``enum`` module allows method named ``_repr_*`` to be defined on +``Enum`` types. + +.. + +.. date: 2024-05-06-08-23-01 +.. gh-issue: 118648 +.. nonce: OVA3jJ +.. section: Library + +Add type parameter defaults to :class:`typing.Generator` and +:class:`typing.AsyncGenerator`. + +.. + +.. date: 2024-05-05-16-08-03 +.. gh-issue: 101137 +.. nonce: 71ECXu +.. section: Library + +Mime type ``text/x-rst`` is now supported by :mod:`mimetypes`. + +.. + +.. date: 2024-05-04-20-22-59 +.. gh-issue: 118164 +.. nonce: 9D02MQ +.. section: Library + +The Python implementation of the ``decimal`` module could appear to hang in +relatively small power cases (like ``2**117``) if context precision was set +to a very high value. A different method to check for exactly representable +results is used now that doesn't rely on computing ``10**precision`` (which +could be effectively too large to compute). + +.. + +.. date: 2024-05-04-18-40-43 +.. gh-issue: 111744 +.. nonce: nuCtwN +.. section: Library + +``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately +after the call rather than before the next line is executed. + +.. + +.. date: 2024-05-02-04-27-12 +.. gh-issue: 118500 +.. nonce: pBGGtQ +.. section: Library + +Add :mod:`pdb` support for zipapps + +.. + +.. date: 2024-04-30-15-18-19 +.. gh-issue: 118406 +.. nonce: y-GnMo +.. section: Library + +Add signature for :class:`sqlite3.Connection` objects. + +.. + +.. date: 2024-04-30-12-59-04 +.. gh-issue: 101732 +.. nonce: 29zUDu +.. section: Library + +Use a Y2038 compatible openssl time function when available. + +.. + +.. date: 2024-04-29-22-11-54 +.. gh-issue: 118404 +.. nonce: GYfMaD +.. section: Library + +Fix :func:`inspect.signature` for non-comparable callables. + +.. + +.. date: 2024-04-29-21-51-28 +.. gh-issue: 118402 +.. nonce: Z_06Th +.. section: Library + +Fix :func:`inspect.signature` for the result of the +:func:`functools.cmp_to_key` call. + +.. + +.. date: 2024-04-27-20-34-56 +.. gh-issue: 116622 +.. nonce: YlQgXv +.. section: Library + +On Android, :any:`sysconfig.get_platform` now returns the format specified +by :pep:`738`. + +.. + +.. date: 2024-04-26-14-53-28 +.. gh-issue: 118285 +.. nonce: A0_pte +.. section: Library + +Allow to specify the signature of custom callable instances of extension +type by the :attr:`__text_signature__` attribute. Specify signatures of +:class:`operator.attrgetter`, :class:`operator.itemgetter`, and +:class:`operator.methodcaller` instances. + +.. + +.. date: 2024-04-26-12-42-29 +.. gh-issue: 118314 +.. nonce: Z7reGc +.. section: Library + +Fix an edge case in :func:`binascii.a2b_base64` strict mode, where excessive +padding is not detected when no padding is necessary. + +.. + +.. date: 2024-04-25-11-49-11 +.. gh-issue: 118271 +.. nonce: 5N2Xcy +.. section: Library + +Add the :class:`!PhotoImage` methods :meth:`~tkinter.PhotoImage.read` to +read an image from a file and :meth:`~tkinter.PhotoImage.data` to get the +image data. Add *background* and *grayscale* parameters to +:class:`!PhotoImage` method :meth:`~tkinter.PhotoImage.write`. + +.. + +.. date: 2024-04-24-16-07-26 +.. gh-issue: 118225 +.. nonce: KdrcgL +.. section: Library + +Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region +from one image to other image, possibly with pixel zooming and/or +subsampling. Add *from_coords* parameter to :class:`!PhotoImage` methods +:meth:`!copy()`, :meth:`!zoom()` and :meth:`!subsample()`. Add *zoom* and +*subsample* parameters to :class:`!PhotoImage` method :meth:`!copy()`. + +.. + +.. date: 2024-04-24-12-29-33 +.. gh-issue: 118221 +.. nonce: 2k_bac +.. section: Library + +Fix a bug where :meth:`sqlite3.Connection.iterdump` could fail if a custom +:attr:`row factory ` was used. Patch by +Erlend Aasland. + +.. + +.. date: 2024-04-24-12-20-48 +.. gh-issue: 118013 +.. nonce: TKn_kZ +.. section: Library + +Fix regression introduced in gh-103193 that meant that calling +:func:`inspect.getattr_static` on an instance would cause a strong reference +to that instance's class to persist in an internal cache in the +:mod:`inspect` module. This caused unexpected memory consumption if the +class was dynamically created, the class held strong references to other +objects which took up a significant amount of memory, and the cache +contained the sole strong reference to the class. The fix for the regression +leads to a slowdown in :func:`!getattr_static`, but the function should +still be significantly faster than it was in Python 3.11. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-24-07-45-08 +.. gh-issue: 118218 +.. nonce: m1OHbN +.. section: Library + +Speed up :func:`itertools.pairwise` in the common case by up to 1.8x. + +.. + +.. date: 2024-04-23-21-17-00 +.. gh-issue: 117486 +.. nonce: ea3KYD +.. section: Library + +Improve the behavior of user-defined subclasses of :class:`ast.AST`. Such +classes will now require no changes in the usual case to conform with the +behavior changes of the :mod:`ast` module in Python 3.13. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-22-21-54-12 +.. gh-issue: 90848 +.. nonce: 5jHEEc +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to configure parent mock with +keyword arguments. + +.. + +.. date: 2024-04-22-20-42-29 +.. gh-issue: 118168 +.. nonce: Igni7h +.. section: Library + +Fix incorrect argument substitution when :data:`typing.Unpack` is used with +the builtin :class:`tuple`. :data:`!typing.Unpack` now raises +:exc:`TypeError` when used with certain invalid types. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-21-18-55-42 +.. gh-issue: 118131 +.. nonce: eAT0is +.. section: Library + +Add command-line interface for the :mod:`random` module. Patch by Hugo van +Kemenade. + +.. + +.. date: 2024-04-19-09-28-43 +.. gh-issue: 118107 +.. nonce: Mdsr1J +.. section: Library + +Fix :mod:`zipimport` reading of ZIP64 files with file entries that are too +big or offset too far. + +.. + +.. date: 2024-04-18-00-35-11 +.. gh-issue: 117535 +.. nonce: 0m6SIM +.. section: Library + +Change the unknown filename of :mod:`warnings` from ``sys`` to ```` to +clarify that it's not a real filename. + +.. + +.. date: 2024-04-17-22-00-15 +.. gh-issue: 114053 +.. nonce: _JBV4D +.. section: Library + +Fix erroneous :exc:`NameError` when calling :func:`typing.get_type_hints` on +a class that made use of :pep:`695` type parameters in a module that had +``from __future__ import annotations`` at the top of the file. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-17-21-28-24 +.. gh-issue: 116931 +.. nonce: _AS09h +.. section: Library + +Add parameter *fileobj* check for :func:`tarfile.TarFile.addfile` + +.. + +.. date: 2024-04-17-19-41-59 +.. gh-issue: 117995 +.. nonce: Vt76Rv +.. section: Library + +Don't raise :exc:`DeprecationWarning` when a :term:`sequence` of parameters +is used to bind indexed, nameless placeholders. See also :gh:`100668`. + +.. + +.. date: 2024-04-17-18-00-30 +.. gh-issue: 80361 +.. nonce: RstWg- +.. section: Library + +Fix TypeError in :func:`email.Message.get_payload` when the charset is +:rfc:`2231` encoded. + +.. + +.. date: 2024-04-16-18-34-11 +.. gh-issue: 86650 +.. nonce: Zeydyg +.. section: Library + +Fix IndexError when parse some emails with invalid Message-ID (including +one-off addresses generated by Microsoft Outlook). + +.. + +.. date: 2024-04-14-15-59-28 +.. gh-issue: 117691 +.. nonce: 1mtREE +.. section: Library + +Improve the error messages emitted by :mod:`tarfile` deprecation warnings +relating to PEP 706. If a ``filter`` argument is not provided to +``extract()`` or ``extractall``, the deprecation warning now points to the +line in the user's code where the relevant function was called. Patch by +Alex Waygood. + +.. + +.. date: 2024-04-13-01-45-15 +.. gh-issue: 115060 +.. nonce: IxoM03 +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by omitting an initial +:meth:`~pathlib.Path.is_dir` call. As a result of this change, +:meth:`~pathlib.Path.glob` can no longer raise :exc:`OSError`. + +.. + +.. date: 2024-04-12-17-37-11 +.. gh-issue: 77102 +.. nonce: Mk6X_E +.. section: Library + +:mod:`site` module now parses ``.pth`` file with UTF-8 first, and +:term:`locale encoding` if ``UnicodeDecodeError`` happened. It supported +only locale encoding before. + +.. + +.. date: 2024-04-11-18-11-37 +.. gh-issue: 76785 +.. nonce: BWNkhC +.. section: Library + +We've exposed the low-level :mod:`!_interpreters` module for the sake of the +PyPI implementation of :pep:`734`. It was sometimes available as the +:mod:`!_xxsubinterpreters` module and was formerly used only for testing. +For the most part, it should be considered an internal module, like +:mod:`!_thread` and :mod:`!_imp`. See +https://discuss.python.org/t/pep-734-multiple-interpreters-in-the-stdlib/41147/26. + +.. + +.. date: 2024-04-10-22-35-24 +.. gh-issue: 115060 +.. nonce: XEVuOb +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by not scanning directories for +non-wildcard pattern segments. + +.. + +.. date: 2024-04-10-21-30-37 +.. gh-issue: 117727 +.. nonce: uAYNVS +.. section: Library + +Speed up :meth:`pathlib.Path.iterdir` by using :func:`os.scandir` +internally. + +.. + +.. date: 2024-04-10-21-08-32 +.. gh-issue: 117586 +.. nonce: UCL__1 +.. section: Library + +Speed up :meth:`pathlib.Path.walk` by working with strings internally. + +.. + +.. date: 2024-04-10-20-59-10 +.. gh-issue: 117722 +.. nonce: oxIUEI +.. section: Library + +Change the new multi-separator support in :meth:`asyncio.Stream.readuntil` +to only accept tuples of separators rather than arbitrary iterables. + +.. + +.. date: 2024-04-09-23-22-21 +.. gh-issue: 117692 +.. nonce: EciInD +.. section: Library + +Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped +``builtin_function_or_method``. + +.. + +.. date: 2024-04-09-20-14-44 +.. gh-issue: 117348 +.. nonce: A2NAAz +.. section: Library + +Largely restored import time performance of configparser by avoiding +dataclasses. + +.. + +.. date: 2024-04-08-19-12-26 +.. gh-issue: 117663 +.. nonce: CPfc_p +.. section: Library + +Fix ``_simple_enum`` to detect aliases when multiple arguments are present +but only one is the member value. + +.. + +.. date: 2024-04-08-03-23-22 +.. gh-issue: 117618 +.. nonce: -4DCUw +.. section: Library + +Support ``package.module`` as ``filename`` for ``break`` command of +:mod:`pdb` + +.. + +.. date: 2024-04-07-19-39-20 +.. gh-issue: 102247 +.. nonce: h8rqiX +.. section: Library + +the status codes enum with constants in http.HTTPStatus are updated to +include the names from RFC9110. This RFC includes some HTTP statuses +previously only used for WEBDAV and assigns more generic names to them. + +The old constants are preserved for backwards compatibility. + +.. + +.. date: 2024-04-06-20-31-09 +.. gh-issue: 117586 +.. nonce: UgWdRK +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by working with strings internally. + +.. + +.. date: 2024-04-06-18-41-36 +.. gh-issue: 117225 +.. nonce: tJh1Hw +.. section: Library + +Add colour to doctest output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-04-05-15-51-01 +.. gh-issue: 117566 +.. nonce: 54nABf +.. section: Library + +:meth:`ipaddress.IPv6Address.is_loopback` will now return ``True`` for +IPv4-mapped loopback addresses, i.e. addresses in the +``::ffff:127.0.0.0/104`` address space. + +.. + +.. date: 2024-04-05-13-38-53 +.. gh-issue: 117546 +.. nonce: lWjhHE +.. section: Library + +Fix issue where :func:`os.path.realpath` stopped resolving symlinks after +encountering a symlink loop on POSIX. + +.. + +.. date: 2024-04-04-15-28-12 +.. gh-issue: 116720 +.. nonce: aGhXns +.. section: Library + +Improved behavior of :class:`asyncio.TaskGroup` when an external +cancellation collides with an internal cancellation. For example, when two +task groups are nested and both experience an exception in a child task +simultaneously, it was possible that the outer task group would misbehave, +because its internal cancellation was swallowed by the inner task group. + +In the case where a task group is cancelled externally and also must raise +an :exc:`ExceptionGroup`, it will now call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next :keyword:`await`, +so the cancellation is not lost. + +An added benefit of these changes is that task groups now preserve the +cancellation count (:meth:`asyncio.Task.cancelling`). + +In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now +reset the undocumented ``_must_cancel`` flag when the cancellation count +reaches zero. + +.. + +.. date: 2024-04-03-16-01-31 +.. gh-issue: 117516 +.. nonce: 7DlHje +.. section: Library + +Add :data:`typing.TypeIs`, implementing :pep:`742`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-03-15-04-23 +.. gh-issue: 117503 +.. nonce: NMfwup +.. section: Library + +Fix support of non-ASCII user names in bytes paths in +:func:`os.path.expanduser` on Posix. + +.. + +.. date: 2024-04-02-11-17-44 +.. gh-issue: 117394 +.. nonce: 2aoSlb +.. section: Library + +:func:`os.path.ismount` is now 2-3 times faster if the user has permissions. + +.. + +.. date: 2024-03-29-15-14-51 +.. gh-issue: 117313 +.. nonce: ks_ONu +.. section: Library + +Only treat ``'\n'``, ``'\r'`` and ``'\r\n'`` as line separators in +re-folding the :mod:`email` messages. Preserve control characters ``'\v'``, +``'\f'``, ``'\x1c'``, ``'\x1d'`` and ``'\x1e'`` and Unicode line separators +``'\x85'``, ``'\u2028'`` and ``'\u2029'`` as is. + +.. + +.. date: 2024-03-29-12-21-40 +.. gh-issue: 117142 +.. nonce: U0agfh +.. section: Library + +Convert :mod:`!_ctypes` to multi-phase initialisation (:pep:`489`). + +.. + +.. date: 2024-03-26-15-29-39 +.. gh-issue: 66543 +.. nonce: OZBhU5 +.. section: Library + +Add the :func:`mimetypes.guess_file_type` function which works with file +path. Passing file path instead of URL in :func:`~mimetypes.guess_type` is +:term:`soft deprecated`. + +.. + +.. date: 2024-03-20-00-11-39 +.. gh-issue: 68583 +.. nonce: mIlxxb +.. section: Library + +webbrowser CLI: replace getopt with argparse, add long options. Patch by +Hugo van Kemenade. + +.. + +.. date: 2024-03-17-18-24-23 +.. gh-issue: 116871 +.. nonce: 9uSl8M +.. section: Library + +Name suggestions for :exc:`AttributeError` and :exc:`ImportError` now only +include underscored names if the original name was underscored. + +.. + +.. date: 2024-02-28-11-51-51 +.. gh-issue: 116023 +.. nonce: CGYhFh +.. section: Library + +Don't show empty fields (value ``None`` or ``[]``) in :func:`ast.dump` by +default. Add ``show_empty=False`` parameter to optionally show them. + +.. + +.. date: 2024-02-28-10-41-24 +.. gh-issue: 115961 +.. nonce: P-_DU0 +.. section: Library + +Added :attr:`!name` and :attr:`!mode` attributes for compressed and archived +file-like objects in modules :mod:`bz2`, :mod:`lzma`, :mod:`tarfile` and +:mod:`zipfile`. The value of the :attr:`!mode` attribute of +:class:`gzip.GzipFile` was changed from integer (``1`` or ``2``) to string +(``'rb'`` or ``'wb'``). The value of the :attr:`!mode` attribute of the +readable file-like object returned by :meth:`zipfile.ZipFile.open` was +changed from ``'r'`` to ``'rb'``. + +.. + +.. date: 2024-02-11-07-31-43 +.. gh-issue: 82062 +.. nonce: eeS6w7 +.. section: Library + +Fix :func:`inspect.signature()` to correctly handle parameter defaults on +methods in extension modules that use names defined in the module namespace. + +.. + +.. date: 2024-01-19-05-40-46 +.. gh-issue: 83856 +.. nonce: jN5M80 +.. section: Library + +Honor :mod:`atexit` for all :mod:`multiprocessing` start methods + +.. + +.. date: 2023-12-14-02-51-38 +.. gh-issue: 113081 +.. nonce: S-9Qyn +.. section: Library + +Print colorized exception just like built-in traceback in :mod:`pdb` + +.. + +.. date: 2023-12-07-20-05-54 +.. gh-issue: 112855 +.. nonce: ph4ehh +.. section: Library + +Speed up pickling of :class:`pathlib.PurePath` objects. Patch by Barney +Gale. + +.. + +.. date: 2023-11-07-22-41-42 +.. gh-issue: 111744 +.. nonce: TbLxF0 +.. section: Library + +Support opcode events in :mod:`bdb` + +.. + +.. date: 2023-10-24-12-39-04 +.. gh-issue: 109617 +.. nonce: YoI8TV +.. section: Library + +:mod:`ncurses`: fixed a crash that could occur on macOS 13 or earlier when +Python was built with Apple Xcode 15's SDK. + +.. + +.. date: 2023-10-20-03-50-17 +.. gh-issue: 83151 +.. nonce: bcsD40 +.. section: Library + +Enabled arbitrary statements and evaluations in :mod:`pdb` shell to access +the local variables of the current frame, which made it possible for +multi-scope code like generators or nested function to work. + +.. + +.. date: 2023-10-02-10-35-58 +.. gh-issue: 110209 +.. nonce: b5zfIz +.. section: Library + +Add :meth:`~object.__class_getitem__` to :class:`types.GeneratorType` and +:class:`types.CoroutineType` for type hinting purposes. Patch by James +Hilton-Balfe. + +.. + +.. date: 2023-08-21-10-34-43 +.. gh-issue: 108191 +.. nonce: GZM3mv +.. section: Library + +The :class:`types.SimpleNamespace` now accepts an optional positional +argument which specifies initial values of attributes as a dict or an +iterable of key-value pairs. + +.. + +.. date: 2023-05-28-11-25-18 +.. gh-issue: 62090 +.. nonce: opAhDn +.. section: Library + +Fix assertion errors caused by whitespace in metavars or ``SUPPRESS``-ed +groups in :mod:`argparse` by simplifying usage formatting. Patch by Ali +Hamdan. + +.. + +.. date: 2023-03-03-21-13-08 +.. gh-issue: 102402 +.. nonce: fpkRO1 +.. section: Library + +Adjust ``logging.LogRecord`` to use ``time.time_ns()`` and fix minor bug +related to floating point math. + +.. + +.. date: 2022-12-14-15-53-38 +.. gh-issue: 100242 +.. nonce: Ny7VUO +.. section: Library + +Bring pure Python implementation ``functools.partial.__new__`` more in line +with the C-implementation by not just always checking for the presence of +the attribute ``'func'`` on the first argument of ``partial``. Instead, both +the Python version and the C version perform an ``isinstance(func, +partial)`` check on the first argument of ``partial``. + +.. + +.. date: 2022-11-23-17-16-31 +.. gh-issue: 99730 +.. nonce: bDQdaX +.. section: Library + +HEAD requests are no longer upgraded to GET request during redirects in +urllib. + +.. + +.. date: 2022-10-24-12-05-19 +.. gh-issue: 66410 +.. nonce: du4UKW +.. section: Library + +Callbacks registered in the :mod:`tkinter` module now take arguments as +various Python objects (``int``, ``float``, ``bytes``, ``tuple``), not just +``str``. To restore the previous behavior set :mod:`!tkinter` module global +:data:`~tkinter.wantobject` to ``1`` before creating the +:class:`~tkinter.Tk` object or call the :meth:`~tkinter.Tk.wantobject` +method of the :class:`!Tk` object with argument ``1``. Calling it with +argument ``2`` restores the current default behavior. + +.. + +.. bpo: 40943 +.. date: 2020-06-10-19-24-17 +.. nonce: vjiiN_ +.. section: Library + +Fix several IndexError when parse emails with truncated Message-ID, address, +routes, etc, e.g. ``example@``. + +.. + +.. bpo: 39324 +.. date: 2020-01-14-09-46-51 +.. nonce: qUcDrM +.. section: Library + +Add mime type mapping for .md <-> text/markdown + +.. + +.. bpo: 18108 +.. date: 2019-09-09-18-18-34 +.. nonce: ajPLAO +.. section: Library + +:func:`shutil.chown` now supports *dir_fd* and *follow_symlinks* keyword +arguments. + +.. + +.. bpo: 30988 +.. date: 2019-08-29-20-26-08 +.. nonce: b-_h5O +.. section: Library + +Fix parsing of emails with invalid address headers having a leading or +trailing dot. Patch by tsufeki. + +.. + +.. bpo: 32839 +.. date: 2018-02-13-10-02-54 +.. nonce: McbVz3 +.. section: Library + +Add the :meth:`!after_info` method for Tkinter widgets. + +.. + +.. date: 2024-04-25-22-12-20 +.. gh-issue: 117928 +.. nonce: LKdTno +.. section: Documentation + +The minimum Sphinx version required for the documentation is now 6.2.1. + +.. + +.. date: 2024-05-07-21-15-47 +.. gh-issue: 118734 +.. nonce: --GHiS +.. section: Build + +Fixes Windows build when invoked directly (not through the :file:`build.bat` +script) without specifying a value for ``UseTIER2``. + +.. + +.. date: 2024-05-06-00-39-06 +.. gh-issue: 115119 +.. nonce: LT27pF +.. section: Build + +The :file:`configure` option :option:`--with-system-libmpdec` now defaults +to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python +3.15. + +.. + +.. date: 2024-04-15-08-35-06 +.. gh-issue: 117845 +.. nonce: IowzyW +.. section: Build + +Fix building against recent libedit versions by detecting readline hook +signatures in :program:`configure`. + +.. + +.. date: 2024-04-14-19-35-35 +.. gh-issue: 116622 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on Android. + +.. + +.. date: 2024-04-09-12-59-06 +.. gh-issue: 117645 +.. nonce: 0oEVAa +.. section: Build + +Increase WASI stack size from 512 KiB to 8 MiB and the initial memory from +10 MiB to 20 MiB. Patch by Victor Stinner. + +.. + +.. date: 2024-02-13-15-31-28 +.. gh-issue: 115119 +.. nonce: FnQzAW +.. section: Build + +:program:`configure` now uses :program:`pkg-config` to detect :mod:`decimal` +dependencies if the :option:`--with-system-libmpdec` option is given. + +.. + +.. date: 2024-05-02-09-28-04 +.. gh-issue: 115119 +.. nonce: cUKMXo +.. section: Windows + +Update Windows installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-05-01-20-57-09 +.. gh-issue: 118486 +.. nonce: K44KJG +.. section: Windows + +:func:`os.mkdir` now accepts *mode* of ``0o700`` to restrict the new +directory to the current user. + +.. + +.. date: 2024-04-29-13-53-25 +.. gh-issue: 118347 +.. nonce: U5ZRm_ +.. section: Windows + +Fixes launcher updates not being installed. + +.. + +.. date: 2024-04-26-14-23-07 +.. gh-issue: 118293 +.. nonce: ohhPtW +.. section: Windows + +The ``multiprocessing`` module now passes the ``STARTF_FORCEOFFFEEDBACK`` +flag when spawning processes to tell Windows not to change the mouse cursor. + +.. + +.. date: 2024-04-15-21-23-34 +.. gh-issue: 115009 +.. nonce: uhisHP +.. section: Windows + +Update Windows installer to use SQLite 3.45.3. + +.. + +.. date: 2024-04-12-14-02-58 +.. gh-issue: 90329 +.. nonce: YpEeaO +.. section: Windows + +Suppress the warning displayed on virtual environment creation when the +requested and created paths differ only by a short (8.3 style) name. +Warnings will continue to be shown if a junction or symlink in the path +caused the venv to be created in a different location than originally +requested. + +.. + +.. date: 2024-04-12-13-18-42 +.. gh-issue: 117786 +.. nonce: LpI01s +.. section: Windows + +Fixes virtual environments not correctly launching when created from a Store +install. + +.. + +.. date: 2024-05-03-12-13-27 +.. gh-issue: 115119 +.. nonce: ltDtoR +.. section: macOS + +Update macOS installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-04-19-08-40-00 +.. gh-issue: 114099 +.. nonce: _iDfrQ +.. section: macOS + +iOS preprocessor symbol usage was made compatible with older macOS SDKs. + +.. + +.. date: 2024-04-15-21-19-39 +.. gh-issue: 115009 +.. nonce: IdxH9N +.. section: macOS + +Update macOS installer to use SQLite 3.45.3. + +.. + +.. date: 2022-04-17-01-07-42 +.. gh-issue: 91629 +.. nonce: YBGAAt +.. section: macOS + +Use :file:`~/.config/fish/conf.d` configs and :program:`fish_add_path` to +set :envvar:`PATH` when installing for the Fish shell. + +.. + +.. bpo: 34774 +.. date: 2018-09-23-01-36-39 +.. nonce: VeM-X- +.. section: IDLE + +Use user-selected color theme for Help => IDLE Doc. + +.. + +.. date: 2024-04-29-17-44-15 +.. gh-issue: 118124 +.. nonce: czQQ9G +.. section: C API + +Fix :c:macro:`Py_BUILD_ASSERT` and :c:macro:`Py_BUILD_ASSERT_EXPR` for +non-constant expressions: use ``static_assert()`` on C11 and newer. Patch by +Victor Stinner. + +.. + +.. date: 2024-04-29-17-19-07 +.. gh-issue: 110850 +.. nonce: vcpLn1 +.. section: C API + +Add "Raw" variant of PyTime functions + +* :c:func:`PyTime_MonotonicRaw` +* :c:func:`PyTime_PerfCounterRaw` +* :c:func:`PyTime_TimeRaw` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-17-16-48-17 +.. gh-issue: 117987 +.. nonce: zsvNL1 +.. section: C API + +Restore functions removed in Python 3.13 alpha 1: + +* :c:func:`Py_SetPythonHome` +* :c:func:`Py_SetProgramName` +* :c:func:`PySys_SetArgvEx` +* :c:func:`PySys_SetArgv` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-16-13-34-01 +.. gh-issue: 117929 +.. nonce: HSr419 +.. section: C API + +Restore removed :c:func:`PyEval_InitThreads` function. Patch by Victor +Stinner. + +.. + +.. date: 2024-04-08-09-44-29 +.. gh-issue: 117534 +.. nonce: 54ZE_n +.. section: C API + +Improve validation logic in the C implementation of +:meth:`datetime.datetime.fromisoformat` to better handle invalid years. +Patch by Vlad Efanov. + +.. + +.. date: 2024-03-18-17-29-52 +.. gh-issue: 68114 +.. nonce: W7R_lI +.. section: C API + +Fixed skipitem()'s handling of the old 'w' and 'w#' formatters. These are +no longer supported and now raise an exception if used. + +.. + +.. date: 2024-03-13-17-48-24 +.. gh-issue: 111997 +.. nonce: 8ZbHlA +.. section: C API + +Add a C-API for firing monitoring events. diff --git a/Misc/NEWS.d/next/Build/2024-02-13-15-31-28.gh-issue-115119.FnQzAW.rst b/Misc/NEWS.d/next/Build/2024-02-13-15-31-28.gh-issue-115119.FnQzAW.rst deleted file mode 100644 index 5111d8f4db6b83..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-13-15-31-28.gh-issue-115119.FnQzAW.rst +++ /dev/null @@ -1,2 +0,0 @@ -:program:`configure` now uses :program:`pkg-config` to detect :mod:`decimal` -dependencies if the :option:`--with-system-libmpdec` option is given. diff --git a/Misc/NEWS.d/next/Build/2024-04-09-12-59-06.gh-issue-117645.0oEVAa.rst b/Misc/NEWS.d/next/Build/2024-04-09-12-59-06.gh-issue-117645.0oEVAa.rst deleted file mode 100644 index 83df6338c291ab..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-04-09-12-59-06.gh-issue-117645.0oEVAa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Increase WASI stack size from 512 KiB to 8 MiB and the initial memory from 10 -MiB to 20 MiB. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2024-04-15-08-35-06.gh-issue-117845.IowzyW.rst b/Misc/NEWS.d/next/Build/2024-04-15-08-35-06.gh-issue-117845.IowzyW.rst deleted file mode 100644 index 02d62da15b2546..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-04-15-08-35-06.gh-issue-117845.IowzyW.rst +++ /dev/null @@ -1 +0,0 @@ -Fix building against recent libedit versions by detecting readline hook signatures in :program:`configure`. diff --git a/Misc/NEWS.d/next/C API/2024-03-18-17-29-52.gh-issue-68114.W7R_lI.rst b/Misc/NEWS.d/next/C API/2024-03-18-17-29-52.gh-issue-68114.W7R_lI.rst deleted file mode 100644 index fa09d2a0a72df7..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-18-17-29-52.gh-issue-68114.W7R_lI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed skipitem()'s handling of the old 'w' and 'w#' formatters. These are -no longer supported and now raise an exception if used. diff --git a/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst b/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst deleted file mode 100644 index 4b7dda610fc2b2..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve validation logic in the C implementation of :meth:`datetime.fromisoformat` -to better handle invalid years. Patch by Vlad Efanov. diff --git a/Misc/NEWS.d/next/C API/2024-04-16-13-34-01.gh-issue-117929.HSr419.rst b/Misc/NEWS.d/next/C API/2024-04-16-13-34-01.gh-issue-117929.HSr419.rst deleted file mode 100644 index 58d475bbf21981..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-04-16-13-34-01.gh-issue-117929.HSr419.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restore removed :c:func:`PyEval_InitThreads` function. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-04-17-16-48-17.gh-issue-117987.zsvNL1.rst b/Misc/NEWS.d/next/C API/2024-04-17-16-48-17.gh-issue-117987.zsvNL1.rst deleted file mode 100644 index b4cca946906c70..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-04-17-16-48-17.gh-issue-117987.zsvNL1.rst +++ /dev/null @@ -1,8 +0,0 @@ -Restore functions removed in Python 3.13 alpha 1: - -* :c:func:`Py_SetPythonHome` -* :c:func:`Py_SetProgramName` -* :c:func:`PySys_SetArgvEx` -* :c:func:`PySys_SetArgv` - -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-07-03-38-34.gh-issue-95754.aPjEBG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-07-03-38-34.gh-issue-95754.aPjEBG.rst deleted file mode 100644 index 588be2d28cd76e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-07-03-38-34.gh-issue-95754.aPjEBG.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve the error message when a script shadowing a module from the standard -library causes :exc:`AttributeError` to be raised. Similarly, improve the error -message when a script shadowing a third party module attempts to access an -attribute from that third party module while still initialising. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-30-00-37-53.gh-issue-117385.h0OJti.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-30-00-37-53.gh-issue-117385.h0OJti.rst deleted file mode 100644 index 2e385df3938347..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-30-00-37-53.gh-issue-117385.h0OJti.rst +++ /dev/null @@ -1 +0,0 @@ -Remove unhandled ``PY_MONITORING_EVENT_BRANCH`` and ``PY_MONITORING_EVENT_EXCEPTION_HANDLED`` events from :func:`sys.settrace`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-14-33-38.gh-issue-117636.exnRKd.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-14-33-38.gh-issue-117636.exnRKd.rst deleted file mode 100644 index 7d7cb506352193..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-14-33-38.gh-issue-117636.exnRKd.rst +++ /dev/null @@ -1 +0,0 @@ -Speedup :func:`os.path.join`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-19-30-38.gh-issue-117641.oaBGSJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-19-30-38.gh-issue-117641.oaBGSJ.rst deleted file mode 100644 index e313c133b72173..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-19-30-38.gh-issue-117641.oaBGSJ.rst +++ /dev/null @@ -1 +0,0 @@ -Speedup :func:`os.path.commonpath` on Unix. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-09-11-31-25.gh-issue-115776.5Nthd0.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-09-11-31-25.gh-issue-115776.5Nthd0.rst deleted file mode 100644 index 5fc0080bcb9551..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-09-11-31-25.gh-issue-115776.5Nthd0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Statically allocated objects are, by definition, immortal so must be -marked as such regardless of whether they are in extension modules or not. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst deleted file mode 100644 index fd437b7fdd3614..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst +++ /dev/null @@ -1 +0,0 @@ -Give ``_PyInstructionSequence`` a Python interface and use it in tests. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-10-22-16-18.gh-issue-117709.-_1YL0.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-10-22-16-18.gh-issue-117709.-_1YL0.rst deleted file mode 100644 index 2216b53688c378..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-10-22-16-18.gh-issue-117709.-_1YL0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Speed up calls to :func:`str` with positional-only argument, -by using the :pep:`590` ``vectorcall`` calling convention. -Patch by Erlend Aasland. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-12-09-09-11.gh-issue-117431.lxFEeJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-12-09-09-11.gh-issue-117431.lxFEeJ.rst deleted file mode 100644 index 0d94389aba124a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-12-09-09-11.gh-issue-117431.lxFEeJ.rst +++ /dev/null @@ -1,9 +0,0 @@ -Improve the performance of the following :class:`bytes` and -:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` -calling convention: - -* :meth:`!count` -* :meth:`!find` -* :meth:`!index` -* :meth:`!rfind` -* :meth:`!rindex` diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-12-11-19-18.gh-issue-117750.YttK6h.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-12-11-19-18.gh-issue-117750.YttK6h.rst deleted file mode 100644 index d7cf5d6e57d0cb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-12-11-19-18.gh-issue-117750.YttK6h.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix issue where an object's dict would get out of sync with the object's -internal values when being cleared. ``obj.__dict__.clear()`` now clears the -internal values, but leaves the dict attached to the object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-12-12-28-49.gh-issue-117755.6ct8kU.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-12-12-28-49.gh-issue-117755.6ct8kU.rst deleted file mode 100644 index a65ec43e25d1c5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-12-12-28-49.gh-issue-117755.6ct8kU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix mimalloc allocator for huge memory allocation (around 8,589,934,592 GiB) on -s390x. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst deleted file mode 100644 index 2492fd163cb549..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-59-25.gh-issue-115874.c3xG-E.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-59-25.gh-issue-115874.c3xG-E.rst deleted file mode 100644 index 5a6fa4cedd9fe4..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-59-25.gh-issue-115874.c3xG-E.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a possible segfault during garbage collection of ``_asyncio.FutureIter`` objects diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst deleted file mode 100644 index bd32500a54ee21..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent ``agen.aclose()`` objects being re-used after ``.throw()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-17-52-32.gh-issue-109118.q9iPEI.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-17-52-32.gh-issue-109118.q9iPEI.rst deleted file mode 100644 index 124540045547b1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-17-52-32.gh-issue-109118.q9iPEI.rst +++ /dev/null @@ -1,2 +0,0 @@ -:ref:`annotation scope ` within class scopes can now -contain lambdas. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-53-52.gh-issue-117901.SsEcVJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-53-52.gh-issue-117901.SsEcVJ.rst deleted file mode 100644 index 1e412690deecd7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-53-52.gh-issue-117901.SsEcVJ.rst +++ /dev/null @@ -1 +0,0 @@ -Add option for compiler's codegen to save nested instruction sequences for introspection. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-19-08-50-48.gh-issue-102511.qDEB66.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-19-08-50-48.gh-issue-102511.qDEB66.rst deleted file mode 100644 index dfdf250710778e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-19-08-50-48.gh-issue-102511.qDEB66.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :func:`os.path.splitroot` with a native implementation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-19-11-59-57.gh-issue-118082._FLuOT.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-19-11-59-57.gh-issue-118082._FLuOT.rst deleted file mode 100644 index 7b9a726d7c77c2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-19-11-59-57.gh-issue-118082._FLuOT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve :exc:`SyntaxError` message for imports without names, like in -``from x import`` and ``import`` cases. It now points -out to users that :keyword:`import` expects at least one name after it. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-20-20-30-15.gh-issue-107674.GZPOP7.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-20-20-30-15.gh-issue-107674.GZPOP7.rst deleted file mode 100644 index 29d16bd7dd6581..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-20-20-30-15.gh-issue-107674.GZPOP7.rst +++ /dev/null @@ -1 +0,0 @@ -Lazy load frame line number to improve performance of tracing diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-22-08-34-28.gh-issue-118074.5_JnIa.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-22-08-34-28.gh-issue-118074.5_JnIa.rst deleted file mode 100644 index 69d29bce12ee57..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-22-08-34-28.gh-issue-118074.5_JnIa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make sure that the Executor objects in the COLD_EXITS array aren't assumed -to be GC-able (which would access bytes outside the object). diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-25-21-18-19.gh-issue-118160.GH5SMc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-25-21-18-19.gh-issue-118160.GH5SMc.rst deleted file mode 100644 index c4e798df5de702..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-25-21-18-19.gh-issue-118160.GH5SMc.rst +++ /dev/null @@ -1,3 +0,0 @@ -:ref:`Annotation scopes ` within classes can now contain -comprehensions. However, such comprehensions are not inlined into their -parent scope at runtime. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Documentation/2024-04-25-22-12-20.gh-issue-117928.LKdTno.rst b/Misc/NEWS.d/next/Documentation/2024-04-25-22-12-20.gh-issue-117928.LKdTno.rst deleted file mode 100644 index c8a2a6b759bae9..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-04-25-22-12-20.gh-issue-117928.LKdTno.rst +++ /dev/null @@ -1 +0,0 @@ -The minimum Sphinx version required for the documentation is now 6.2.1. diff --git a/Misc/NEWS.d/next/Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst b/Misc/NEWS.d/next/Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst deleted file mode 100644 index 0a2e3e3c540c48..00000000000000 --- a/Misc/NEWS.d/next/Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst +++ /dev/null @@ -1 +0,0 @@ -Add the :meth:`after_info` method for Tkinter widgets. diff --git a/Misc/NEWS.d/next/Library/2019-08-29-20-26-08.bpo-30988.b-_h5O.rst b/Misc/NEWS.d/next/Library/2019-08-29-20-26-08.bpo-30988.b-_h5O.rst deleted file mode 100644 index c776c73ba160e0..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-08-29-20-26-08.bpo-30988.b-_h5O.rst +++ /dev/null @@ -1 +0,0 @@ -Fix parsing of emails with invalid address headers having a leading or trailing dot. Patch by tsufeki. diff --git a/Misc/NEWS.d/next/Library/2019-09-09-18-18-34.bpo-18108.ajPLAO.rst b/Misc/NEWS.d/next/Library/2019-09-09-18-18-34.bpo-18108.ajPLAO.rst deleted file mode 100644 index 70ff76a0c920be..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-09-18-18-34.bpo-18108.ajPLAO.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`shutil.chown` now supports *dir_fd* and *follow_symlinks* keyword arguments. diff --git a/Misc/NEWS.d/next/Library/2020-06-10-19-24-17.bpo-40943.vjiiN_.rst b/Misc/NEWS.d/next/Library/2020-06-10-19-24-17.bpo-40943.vjiiN_.rst deleted file mode 100644 index 2018e857830d1e..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-06-10-19-24-17.bpo-40943.vjiiN_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix several IndexError when parse emails with truncated Message-ID, address, routes, etc, e.g. ``example@``. diff --git a/Misc/NEWS.d/next/Library/2022-12-14-15-53-38.gh-issue-100242.Ny7VUO.rst b/Misc/NEWS.d/next/Library/2022-12-14-15-53-38.gh-issue-100242.Ny7VUO.rst deleted file mode 100644 index e86059942d52db..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-14-15-53-38.gh-issue-100242.Ny7VUO.rst +++ /dev/null @@ -1,5 +0,0 @@ -Bring pure Python implementation ``functools.partial.__new__`` more in line -with the C-implementation by not just always checking for the presence of -the attribute ``'func'`` on the first argument of ``partial``. Instead, both -the Python version and the C version perform an ``isinstance(func, partial)`` -check on the first argument of ``partial``. diff --git a/Misc/NEWS.d/next/Library/2023-03-03-21-13-08.gh-issue-102402.fpkRO1.rst b/Misc/NEWS.d/next/Library/2023-03-03-21-13-08.gh-issue-102402.fpkRO1.rst deleted file mode 100644 index fa8f3750b85a6b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-03-21-13-08.gh-issue-102402.fpkRO1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Adjust ``logging.LogRecord`` to use ``time.time_ns()`` and fix minor bug -related to floating point math. diff --git a/Misc/NEWS.d/next/Library/2023-08-21-10-34-43.gh-issue-108191.GZM3mv.rst b/Misc/NEWS.d/next/Library/2023-08-21-10-34-43.gh-issue-108191.GZM3mv.rst deleted file mode 100644 index da4ce5742549e6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-21-10-34-43.gh-issue-108191.GZM3mv.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :class:`types.SimpleNamespace` now accepts an optional positional -argument which specifies initial values of attributes as a dict or an -iterable of key-value pairs. diff --git a/Misc/NEWS.d/next/Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst b/Misc/NEWS.d/next/Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst deleted file mode 100644 index 6badc7a9dc1c94..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-07-20-05-54.gh-issue-112855.ph4ehh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up pickling of :class:`pathlib.PurePath` objects. Patch by Barney -Gale. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-10-41-24.gh-issue-115961.P-_DU0.rst b/Misc/NEWS.d/next/Library/2024-02-28-10-41-24.gh-issue-115961.P-_DU0.rst deleted file mode 100644 index eef7eb8687b44f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-10-41-24.gh-issue-115961.P-_DU0.rst +++ /dev/null @@ -1,7 +0,0 @@ -Added :attr:`!name` and :attr:`!mode` attributes for compressed and archived -file-like objects in modules :mod:`bz2`, :mod:`lzma`, :mod:`tarfile` and -:mod:`zipfile`. The value of the :attr:`!mode` attribute of -:class:`gzip.GzipFile` was changed from integer (``1`` or ``2``) to string -(``'rb'`` or ``'wb'``). The value of the :attr:`!mode` attribute of the -readable file-like object returned by :meth:`zipfile.ZipFile.open` was -changed from ``'r'`` to ``'rb'``. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-11-51-51.gh-issue-116023.CGYhFh.rst b/Misc/NEWS.d/next/Library/2024-02-28-11-51-51.gh-issue-116023.CGYhFh.rst deleted file mode 100644 index bebb67e585eea6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-11-51-51.gh-issue-116023.CGYhFh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Don't show empty fields (value ``None`` or ``[]``) -in :func:`ast.dump` by default. Add ``show_empty=False`` -parameter to optionally show them. diff --git a/Misc/NEWS.d/next/Library/2024-03-20-00-11-39.gh-issue-68583.mIlxxb.rst b/Misc/NEWS.d/next/Library/2024-03-20-00-11-39.gh-issue-68583.mIlxxb.rst deleted file mode 100644 index 12caed75b79044..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-20-00-11-39.gh-issue-68583.mIlxxb.rst +++ /dev/null @@ -1,2 +0,0 @@ -webbrowser CLI: replace getopt with argparse, add long options. Patch by -Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst b/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst deleted file mode 100644 index 36810bd815c502..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst +++ /dev/null @@ -1 +0,0 @@ -Convert :mod:`!_ctypes` to multi-phase initialisation (:pep:`489`). diff --git a/Misc/NEWS.d/next/Library/2024-03-29-15-14-51.gh-issue-117313.ks_ONu.rst b/Misc/NEWS.d/next/Library/2024-03-29-15-14-51.gh-issue-117313.ks_ONu.rst deleted file mode 100644 index e67576ee574f92..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-29-15-14-51.gh-issue-117313.ks_ONu.rst +++ /dev/null @@ -1,4 +0,0 @@ -Only treat ``'\n'``, ``'\r'`` and ``'\r\n'`` as line separators in -re-folding the :mod:`email` messages. Preserve control characters ``'\v'``, -``'\f'``, ``'\x1c'``, ``'\x1d'`` and ``'\x1e'`` and Unicode line separators -``'\x85'``, ``'\u2028'`` and ``'\u2029'`` as is. diff --git a/Misc/NEWS.d/next/Library/2024-04-02-11-17-44.gh-issue-117394.2aoSlb.rst b/Misc/NEWS.d/next/Library/2024-04-02-11-17-44.gh-issue-117394.2aoSlb.rst deleted file mode 100644 index 7b695be4b35d0c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-02-11-17-44.gh-issue-117394.2aoSlb.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`os.path.ismount` is now 2-3 times faster if the user has permissions. diff --git a/Misc/NEWS.d/next/Library/2024-04-03-15-04-23.gh-issue-117503.NMfwup.rst b/Misc/NEWS.d/next/Library/2024-04-03-15-04-23.gh-issue-117503.NMfwup.rst deleted file mode 100644 index f0ea513ac9d4a9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-03-15-04-23.gh-issue-117503.NMfwup.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix support of non-ASCII user names in bytes paths in -:func:`os.path.expanduser` on Posix. diff --git a/Misc/NEWS.d/next/Library/2024-04-03-16-01-31.gh-issue-117516.7DlHje.rst b/Misc/NEWS.d/next/Library/2024-04-03-16-01-31.gh-issue-117516.7DlHje.rst deleted file mode 100644 index bbf69126d956d2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-03-16-01-31.gh-issue-117516.7DlHje.rst +++ /dev/null @@ -1 +0,0 @@ -Add :data:`typing.TypeIs`, implementing :pep:`742`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-04-04-15-28-12.gh-issue-116720.aGhXns.rst b/Misc/NEWS.d/next/Library/2024-04-04-15-28-12.gh-issue-116720.aGhXns.rst deleted file mode 100644 index 39c7d6b8a1e978..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-04-15-28-12.gh-issue-116720.aGhXns.rst +++ /dev/null @@ -1,18 +0,0 @@ -Improved behavior of :class:`asyncio.TaskGroup` when an external cancellation -collides with an internal cancellation. For example, when two task groups -are nested and both experience an exception in a child task simultaneously, -it was possible that the outer task group would misbehave, because -its internal cancellation was swallowed by the inner task group. - -In the case where a task group is cancelled externally and also must -raise an :exc:`ExceptionGroup`, it will now call the parent task's -:meth:`~asyncio.Task.cancel` method. This ensures that a -:exc:`asyncio.CancelledError` will be raised at the next -:keyword:`await`, so the cancellation is not lost. - -An added benefit of these changes is that task groups now preserve the -cancellation count (:meth:`asyncio.Task.cancelling`). - -In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now -reset the undocumented ``_must_cancel`` flag when the cancellation count -reaches zero. diff --git a/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst b/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst deleted file mode 100644 index 9762991e47a6a4..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :func:`os.path.realpath` stopped resolving symlinks after -encountering a symlink loop on POSIX. diff --git a/Misc/NEWS.d/next/Library/2024-04-05-15-51-01.gh-issue-117566.54nABf.rst b/Misc/NEWS.d/next/Library/2024-04-05-15-51-01.gh-issue-117566.54nABf.rst deleted file mode 100644 index 56c2fb0e25d494..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-05-15-51-01.gh-issue-117566.54nABf.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`ipaddress.IPv6Address.is_loopback` will now return ``True`` for -IPv4-mapped loopback addresses, i.e. addresses in the -``::ffff:127.0.0.0/104`` address space. diff --git a/Misc/NEWS.d/next/Library/2024-04-06-18-41-36.gh-issue-117225.tJh1Hw.rst b/Misc/NEWS.d/next/Library/2024-04-06-18-41-36.gh-issue-117225.tJh1Hw.rst deleted file mode 100644 index 6a0da1c3bc9388..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-06-18-41-36.gh-issue-117225.tJh1Hw.rst +++ /dev/null @@ -1 +0,0 @@ -Add colour to doctest output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-04-06-20-31-09.gh-issue-117586.UgWdRK.rst b/Misc/NEWS.d/next/Library/2024-04-06-20-31-09.gh-issue-117586.UgWdRK.rst deleted file mode 100644 index 65c699977bd807..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-06-20-31-09.gh-issue-117586.UgWdRK.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :meth:`pathlib.Path.glob` by working with strings internally. diff --git a/Misc/NEWS.d/next/Library/2024-04-07-19-39-20.gh-issue-102247.h8rqiX.rst b/Misc/NEWS.d/next/Library/2024-04-07-19-39-20.gh-issue-102247.h8rqiX.rst deleted file mode 100644 index c0f74916ddfb1f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-07-19-39-20.gh-issue-102247.h8rqiX.rst +++ /dev/null @@ -1,3 +0,0 @@ -the status codes enum with constants in http.HTTPStatus are updated to include the names from RFC9110. This RFC includes some HTTP statuses previously only used for WEBDAV and assigns more generic names to them. - -The old constants are preserved for backwards compatibility. diff --git a/Misc/NEWS.d/next/Library/2024-04-08-19-12-26.gh-issue-117663.CPfc_p.rst b/Misc/NEWS.d/next/Library/2024-04-08-19-12-26.gh-issue-117663.CPfc_p.rst deleted file mode 100644 index 2c7a5224b5a6eb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-08-19-12-26.gh-issue-117663.CPfc_p.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``_simple_enum`` to detect aliases when multiple arguments are present -but only one is the member value. diff --git a/Misc/NEWS.d/next/Library/2024-04-09-20-14-44.gh-issue-117348.A2NAAz.rst b/Misc/NEWS.d/next/Library/2024-04-09-20-14-44.gh-issue-117348.A2NAAz.rst deleted file mode 100644 index 2451a4e4f622e4..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-09-20-14-44.gh-issue-117348.A2NAAz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Largely restored import time performance of configparser by avoiding -dataclasses. diff --git a/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst b/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst deleted file mode 100644 index 98a6e125c440ef..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped -``builtin_function_or_method``. diff --git a/Misc/NEWS.d/next/Library/2024-04-10-20-59-10.gh-issue-117722.oxIUEI.rst b/Misc/NEWS.d/next/Library/2024-04-10-20-59-10.gh-issue-117722.oxIUEI.rst deleted file mode 100644 index de999883658898..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-10-20-59-10.gh-issue-117722.oxIUEI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change the new multi-separator support in :meth:`asyncio.Stream.readuntil` -to only accept tuples of separators rather than arbitrary iterables. diff --git a/Misc/NEWS.d/next/Library/2024-04-10-21-08-32.gh-issue-117586.UCL__1.rst b/Misc/NEWS.d/next/Library/2024-04-10-21-08-32.gh-issue-117586.UCL__1.rst deleted file mode 100644 index aefac85f9c61b9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-10-21-08-32.gh-issue-117586.UCL__1.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :meth:`pathlib.Path.walk` by working with strings internally. diff --git a/Misc/NEWS.d/next/Library/2024-04-10-21-30-37.gh-issue-117727.uAYNVS.rst b/Misc/NEWS.d/next/Library/2024-04-10-21-30-37.gh-issue-117727.uAYNVS.rst deleted file mode 100644 index 3a0b6834e91f18..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-10-21-30-37.gh-issue-117727.uAYNVS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :meth:`pathlib.Path.iterdir` by using :func:`os.scandir` -internally. diff --git a/Misc/NEWS.d/next/Library/2024-04-10-22-35-24.gh-issue-115060.XEVuOb.rst b/Misc/NEWS.d/next/Library/2024-04-10-22-35-24.gh-issue-115060.XEVuOb.rst deleted file mode 100644 index b5084a0e86c74f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-10-22-35-24.gh-issue-115060.XEVuOb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :meth:`pathlib.Path.glob` by not scanning directories for -non-wildcard pattern segments. diff --git a/Misc/NEWS.d/next/Library/2024-04-11-18-11-37.gh-issue-76785.BWNkhC.rst b/Misc/NEWS.d/next/Library/2024-04-11-18-11-37.gh-issue-76785.BWNkhC.rst deleted file mode 100644 index f3e4c57b8f19cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-11-18-11-37.gh-issue-76785.BWNkhC.rst +++ /dev/null @@ -1,6 +0,0 @@ -We've exposed the low-level :mod:`!_interpreters` module for the sake of the -PyPI implementation of :pep:`734`. It was sometimes available as the -:mod:`!_xxsubinterpreters` module and was formerly used only for testing. For -the most part, it should be considered an internal module, like :mod:`!_thread` -and :mod:`!_imp`. See -https://discuss.python.org/t/pep-734-multiple-interpreters-in-the-stdlib/41147/26. diff --git a/Misc/NEWS.d/next/Library/2024-04-12-17-37-11.gh-issue-77102.Mk6X_E.rst b/Misc/NEWS.d/next/Library/2024-04-12-17-37-11.gh-issue-77102.Mk6X_E.rst deleted file mode 100644 index 6f91251126dc7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-12-17-37-11.gh-issue-77102.Mk6X_E.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`site` module now parses ``.pth`` file with UTF-8 first, and -:term:`locale encoding` if ``UnicodeDecodeError`` happened. It supported -only locale encoding before. diff --git a/Misc/NEWS.d/next/Library/2024-04-13-01-45-15.gh-issue-115060.IxoM03.rst b/Misc/NEWS.d/next/Library/2024-04-13-01-45-15.gh-issue-115060.IxoM03.rst deleted file mode 100644 index 50b374acb90ad0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-13-01-45-15.gh-issue-115060.IxoM03.rst +++ /dev/null @@ -1,3 +0,0 @@ -Speed up :meth:`pathlib.Path.glob` by omitting an initial -:meth:`~pathlib.Path.is_dir` call. As a result of this change, -:meth:`~pathlib.Path.glob` can no longer raise :exc:`OSError`. diff --git a/Misc/NEWS.d/next/Library/2024-04-14-15-59-28.gh-issue-117691.1mtREE.rst b/Misc/NEWS.d/next/Library/2024-04-14-15-59-28.gh-issue-117691.1mtREE.rst deleted file mode 100644 index d90817a9ebde2f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-14-15-59-28.gh-issue-117691.1mtREE.rst +++ /dev/null @@ -1,5 +0,0 @@ -Improve the error messages emitted by :mod:`tarfile` deprecation warnings -relating to PEP 706. If a ``filter`` argument is not provided to -``extract()`` or ``extractall``, the deprecation warning now points to the -line in the user's code where the relevant function was called. Patch by -Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-04-16-18-34-11.gh-issue-86650.Zeydyg.rst b/Misc/NEWS.d/next/Library/2024-04-16-18-34-11.gh-issue-86650.Zeydyg.rst deleted file mode 100644 index 8a1626fa63c804..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-16-18-34-11.gh-issue-86650.Zeydyg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix IndexError when parse some emails with invalid Message-ID (including -one-off addresses generated by Microsoft Outlook). diff --git a/Misc/NEWS.d/next/Library/2024-04-17-18-00-30.gh-issue-80361.RstWg-.rst b/Misc/NEWS.d/next/Library/2024-04-17-18-00-30.gh-issue-80361.RstWg-.rst deleted file mode 100644 index 3bbae23cc18bf6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-17-18-00-30.gh-issue-80361.RstWg-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix TypeError in :func:`email.Message.get_payload` when the charset is :rfc:`2231` -encoded. diff --git a/Misc/NEWS.d/next/Library/2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst b/Misc/NEWS.d/next/Library/2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst deleted file mode 100644 index a289939d33e830..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-17-19-41-59.gh-issue-117995.Vt76Rv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Don't raise :exc:`DeprecationWarning` when a :term:`sequence` of parameters -is used to bind indexed, nameless placeholders. See also :gh:`100668`. diff --git a/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst b/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst deleted file mode 100644 index 98df8f59f71b76..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst +++ /dev/null @@ -1 +0,0 @@ -Add parameter *fileobj* check for :func:`tarfile.addfile()` diff --git a/Misc/NEWS.d/next/Library/2024-04-17-22-00-15.gh-issue-114053._JBV4D.rst b/Misc/NEWS.d/next/Library/2024-04-17-22-00-15.gh-issue-114053._JBV4D.rst deleted file mode 100644 index 827b2d656b4d38..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-17-22-00-15.gh-issue-114053._JBV4D.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix erroneous :exc:`NameError` when calling :func:`typing.get_type_hints` on -a class that made use of :pep:`695` type parameters in a module that had -``from __future__ import annotations`` at the top of the file. Patch by Alex -Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst b/Misc/NEWS.d/next/Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst deleted file mode 100644 index 72423506ccc07b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-18-00-35-11.gh-issue-117535.0m6SIM.rst +++ /dev/null @@ -1 +0,0 @@ -Change the unknown filename of :mod:`warnings` from ``sys`` to ```` to clarify that it's not a real filename. diff --git a/Misc/NEWS.d/next/Library/2024-04-22-20-42-29.gh-issue-118168.Igni7h.rst b/Misc/NEWS.d/next/Library/2024-04-22-20-42-29.gh-issue-118168.Igni7h.rst deleted file mode 100644 index 78c3e0fe17979a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-22-20-42-29.gh-issue-118168.Igni7h.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix incorrect argument substitution when :data:`typing.Unpack` is used with -the builtin :class:`tuple`. :data:`!typing.Unpack` now raises -:exc:`TypeError` when used with certain invalid types. Patch by Jelle -Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst b/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst deleted file mode 100644 index daa5fe7c0f2917..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst +++ /dev/null @@ -1,9 +0,0 @@ -Fix regression introduced in gh-103193 that meant that calling -:func:`inspect.getattr_static` on an instance would cause a strong reference -to that instance's class to persist in an internal cache in the -:mod:`inspect` module. This caused unexpected memory consumption if the -class was dynamically created, the class held strong references to other -objects which took up a significant amount of memory, and the cache -contained the sole strong reference to the class. The fix for the regression -leads to a slowdown in :func:`getattr_static`, but the function should still -be significantly faster than it was in Python 3.11. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst b/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst deleted file mode 100644 index 9b0ea9978a195e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug where :func:`sqlite3.iterdump` could fail if a custom :attr:`row -factory ` was used. Patch by Erlend Aasland. diff --git a/Misc/NEWS.d/next/Library/2024-04-26-14-53-28.gh-issue-118285.A0_pte.rst b/Misc/NEWS.d/next/Library/2024-04-26-14-53-28.gh-issue-118285.A0_pte.rst deleted file mode 100644 index 6e8f8d368ca5a6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-26-14-53-28.gh-issue-118285.A0_pte.rst +++ /dev/null @@ -1,4 +0,0 @@ -Allow to specify the signature of custom callable instances of extension -type by the :attr:`__text_signature__` attribute. Specify signatures of -:class:`operator.attrgetter`, :class:`operator.itemgetter`, and -:class:`operator.methodcaller` instances. diff --git a/Misc/NEWS.d/next/Security/2024-03-25-21-25-28.gh-issue-117233.E4CyI_.rst b/Misc/NEWS.d/next/Security/2024-03-25-21-25-28.gh-issue-117233.E4CyI_.rst deleted file mode 100644 index a4142ec21b7e5d..00000000000000 --- a/Misc/NEWS.d/next/Security/2024-03-25-21-25-28.gh-issue-117233.E4CyI_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Detect BLAKE2, SHA3, Shake, & truncated SHA512 support in the OpenSSL-ish -libcrypto library at build time. This allows :mod:`hashlib` to be used with -libraries that do not to support every algorithm that upstream OpenSSL does. diff --git a/Misc/NEWS.d/next/Security/2024-03-27-13-50-02.gh-issue-116741.ZoGryG.rst b/Misc/NEWS.d/next/Security/2024-03-27-13-50-02.gh-issue-116741.ZoGryG.rst deleted file mode 100644 index 12a41948066bed..00000000000000 --- a/Misc/NEWS.d/next/Security/2024-03-27-13-50-02.gh-issue-116741.ZoGryG.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled libexpat to 2.6.2 diff --git a/Misc/NEWS.d/next/Windows/2024-04-12-13-18-42.gh-issue-117786.LpI01s.rst b/Misc/NEWS.d/next/Windows/2024-04-12-13-18-42.gh-issue-117786.LpI01s.rst deleted file mode 100644 index a4cd9a9adb3e59..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-04-12-13-18-42.gh-issue-117786.LpI01s.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes virtual environments not correctly launching when created from a Store -install. diff --git a/Misc/NEWS.d/next/Windows/2024-04-12-14-02-58.gh-issue-90329.YpEeaO.rst b/Misc/NEWS.d/next/Windows/2024-04-12-14-02-58.gh-issue-90329.YpEeaO.rst deleted file mode 100644 index 7242428567dd25..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-04-12-14-02-58.gh-issue-90329.YpEeaO.rst +++ /dev/null @@ -1,5 +0,0 @@ -Suppress the warning displayed on virtual environment creation when the -requested and created paths differ only by a short (8.3 style) name. -Warnings will continue to be shown if a junction or symlink in the path -caused the venv to be created in a different location than originally -requested. diff --git a/Misc/NEWS.d/next/Windows/2024-04-15-21-23-34.gh-issue-115009.uhisHP.rst b/Misc/NEWS.d/next/Windows/2024-04-15-21-23-34.gh-issue-115009.uhisHP.rst deleted file mode 100644 index 01269b2f3e5216..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-04-15-21-23-34.gh-issue-115009.uhisHP.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.45.3. diff --git a/Misc/NEWS.d/next/Windows/2024-04-26-14-23-07.gh-issue-118293.ohhPtW.rst b/Misc/NEWS.d/next/Windows/2024-04-26-14-23-07.gh-issue-118293.ohhPtW.rst deleted file mode 100644 index 7383a2b32d7240..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-04-26-14-23-07.gh-issue-118293.ohhPtW.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``multiprocessing`` module now passes the ``STARTF_FORCEOFFFEEDBACK`` -flag when spawning processes to tell Windows not to change the mouse cursor. diff --git a/Misc/NEWS.d/next/macOS/2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst b/Misc/NEWS.d/next/macOS/2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst deleted file mode 100644 index 13f3336c4a6ed8..00000000000000 --- a/Misc/NEWS.d/next/macOS/2022-04-17-01-07-42.gh-issue-91629.YBGAAt.rst +++ /dev/null @@ -1 +0,0 @@ -Use :file:`~/.config/fish/conf.d` configs and :program:`fish_add_path` to set :envvar:`PATH` when installing for the Fish shell. diff --git a/Misc/NEWS.d/next/macOS/2024-04-15-21-19-39.gh-issue-115009.IdxH9N.rst b/Misc/NEWS.d/next/macOS/2024-04-15-21-19-39.gh-issue-115009.IdxH9N.rst deleted file mode 100644 index 0c16f716e6a023..00000000000000 --- a/Misc/NEWS.d/next/macOS/2024-04-15-21-19-39.gh-issue-115009.IdxH9N.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use SQLite 3.45.3. diff --git a/Misc/NEWS.d/next/macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst b/Misc/NEWS.d/next/macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst deleted file mode 100644 index f9af0629ed9434..00000000000000 --- a/Misc/NEWS.d/next/macOS/2024-04-19-08-40-00.gh-issue-114099._iDfrQ.rst +++ /dev/null @@ -1 +0,0 @@ -iOS preprocessor symbol usage was made compatible with older macOS SDKs. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index bef0ce7d7e6175..58f8e0afd71f1b 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -48,14 +48,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "93118043651ffa33dcaaab445bae4f8929fca25d2d749079b78e97f220c3d8b1" + "checksumValue": "338fac3fb8cdd60f406b6326431338756f58a8af94229ffd9bf1e7c2b1ad71ca" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-2.5.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-4.0.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:4.0.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -63,7 +63,7 @@ "name": "mpdecimal", "originator": "Organization: bytereef.org", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.5.1" + "versionInfo": "4.0.0" }, { "SPDXID": "SPDXRef-PACKAGE-openssl", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 5c29e98705aeaf..77473662aaa76c 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2501,3 +2501,9 @@ added = '3.13' [function.PyType_GetModuleByDef] added = '3.13' +[function.PyEval_GetFrameBuiltins] + added = '3.13' +[function.PyEval_GetFrameGlobals] + added = '3.13' +[function.PyEval_GetFrameLocals] + added = '3.13' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 61037f592f82f1..78b979698fcd75 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_abc.c b/Modules/_abc.c index f2a523e6f2fc27..4f4b24b035db4a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -970,6 +970,7 @@ _abcmodule_free(void *module) static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0873d32a9ec1a5..a26714f9755df5 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3795,6 +3795,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9e0fd336419b44..56322c48b7cd35 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -462,6 +462,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 5df9fd3df493ee..78242214764f2b 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -137,6 +137,7 @@ blake2_exec(PyObject *m) static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 3d0d4ee5e79c2b..661847ad26702e 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -802,6 +802,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index c31c1b6d6f2bbc..32373f0799bfeb 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -1050,6 +1050,7 @@ static PyMethodDef _codecs_functions[] = { static PyModuleDef_Slot _codecs_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 309d63c9bf7cbe..644a90a8c71099 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2511,9 +2511,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, /* Only take the fast path when get() and __setitem__() * have not been overridden. */ - mapping_get = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(get)); + mapping_get = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(get)); dict_get = _PyType_Lookup(&PyDict_Type, &_Py_ID(get)); - mapping_setitem = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(__setitem__)); + mapping_setitem = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(__setitem__)); dict_setitem = _PyType_Lookup(&PyDict_Type, &_Py_ID(__setitem__)); if (mapping_get != NULL && mapping_get == dict_get && @@ -2587,6 +2587,8 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, } done: + Py_XDECREF(mapping_get); + Py_XDECREF(mapping_setitem); Py_DECREF(it); Py_XDECREF(key); Py_XDECREF(newval); @@ -2817,6 +2819,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index f621c1de6d42d6..3f96f07909b69a 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -45,6 +45,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index ac948f417cebf5..9d6b66d4938687 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1796,6 +1796,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 3cb0b24668eb2a..574cb8014493c4 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1096,7 +1096,7 @@ static int UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) { /* XXX Should we disallow deleting _fields_? */ - if (-1 == PyObject_GenericSetAttr(self, key, value)) + if (-1 == PyType_Type.tp_setattro(self, key, value)) return -1; if (PyUnicode_Check(key) && @@ -5948,6 +5948,7 @@ module_free(void *module) static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _ctypes_mod_exec}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 1dd3ef19052470..f46f6362ddd03b 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,7 +1,7 @@ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt @@ -1167,6 +1167,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) static struct PyModuleDef_Slot _ctypes_test_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 2ec8f34c5c220b..125c72dbbe7712 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -697,6 +697,7 @@ static PyModuleDef_Slot _curses_slots[] = { // XXX gh-103092: fix isolation. {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d04d1e973af030..ee3d4c6eae7546 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1156,8 +1156,10 @@ int py_mvwdelch(WINDOW *w, int y, int x) #endif #if defined(HAVE_CURSES_IS_PAD) +// is_pad() is defined, either as a macro or as a function #define py_is_pad(win) is_pad(win) #elif defined(WINDOW_HAS_FLAGS) +// is_pad() is not defined, but we can inspect WINDOW structure members #define py_is_pad(win) ((win) ? ((win)->_flags & _ISPAD) != 0 : FALSE) #endif @@ -4586,7 +4588,14 @@ make_ncurses_version(PyTypeObject *type) if (ncurses_version == NULL) { return NULL; } - + const char *str = curses_version(); + unsigned long major = 0, minor = 0, patch = 0; + if (!str || sscanf(str, "%*[^0-9]%lu.%lu.%lu", &major, &minor, &patch) < 3) { + // Fallback to header version, which cannot be that wrong + major = NCURSES_VERSION_MAJOR; + minor = NCURSES_VERSION_MINOR; + patch = NCURSES_VERSION_PATCH; + } #define SetIntItem(flag) \ PyStructSequence_SET_ITEM(ncurses_version, pos++, PyLong_FromLong(flag)); \ if (PyErr_Occurred()) { \ @@ -4594,9 +4603,9 @@ make_ncurses_version(PyTypeObject *type) return NULL; \ } - SetIntItem(NCURSES_VERSION_MAJOR) - SetIntItem(NCURSES_VERSION_MINOR) - SetIntItem(NCURSES_VERSION_PATCH) + SetIntItem(major) + SetIntItem(minor) + SetIntItem(patch) #undef SetIntItem return ncurses_version; @@ -4743,6 +4752,9 @@ PyInit__curses(void) m = PyModule_Create(&_cursesmodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 06004e258b2eff..8164715a66ff09 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -6984,6 +6984,9 @@ PyInit__datetime(void) PyObject *mod = PyModule_Create(&datetimemodule); if (mod == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif if (_datetime_exec(mod) < 0) { Py_DECREF(mod); diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index ee33fe625be3d7..1be4234aad3291 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -616,6 +616,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index c105367a270458..2daa24c823a542 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -4287,7 +4287,7 @@ nm_mpd_qdivmod(PyObject *v, PyObject *w) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -5312,7 +5312,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -6157,6 +6157,7 @@ decimal_free(void *module) static struct PyModuleDef_Slot _decimal_slots[] = { {Py_mod_exec, _decimal_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index aaa0cad76ae5c4..b11983d2caa2d1 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4463,6 +4463,7 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 406fcf0da2f7e4..9dee7bf3062710 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -571,6 +571,17 @@ static PyMemberDef keyobject_members[] = { {NULL} }; +static PyObject * +keyobject_text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj)"); +} + +static PyGetSetDef keyobject_getset[] = { + {"__text_signature__", keyobject_text_signature, (setter)NULL}, + {NULL} +}; + static PyObject * keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds); @@ -585,6 +596,7 @@ static PyType_Slot keyobject_type_slots[] = { {Py_tp_clear, keyobject_clear}, {Py_tp_richcompare, keyobject_richcompare}, {Py_tp_members, keyobject_members}, + {Py_tp_getset, keyobject_getset}, {0, 0} }; @@ -1547,6 +1559,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index db868c18160fda..df7fba67810ed0 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -825,6 +825,7 @@ _gdbm_module_free(void *module) static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index d0b46810dc1489..14d9c186151232 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2289,6 +2289,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 9d4ec256ee9e3e..695ce22f8049df 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -681,6 +681,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 43c96584790fa0..ff8dacf5bd1ad0 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -3326,6 +3326,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 46801bd416495a..556953db6b8039 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1830,6 +1830,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 8fea56977ef3fe..86a4113dcc16f1 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -1519,6 +1519,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 173f5b55e5f732..269070fe2b0a42 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -720,6 +720,7 @@ iomodule_exec(PyObject *m) static struct PyModuleDef_Slot iomodule_slots[] = { {Py_mod_exec, iomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_json.c b/Modules/_json.c index c55299899e77fe..e33ef1f5eea92f 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -85,11 +85,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyEncoderObject *self); static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, Py_ssize_t indent_level); +encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent); static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, Py_ssize_t indent_level); +encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent); static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, Py_ssize_t indent_level); +encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent); static PyObject * _encoded_const(PyObject *obj); static void @@ -1251,6 +1251,17 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)s; } +static PyObject * +_create_newline_indent(PyObject *indent, Py_ssize_t indent_level) +{ + PyObject *newline_indent = PyUnicode_FromOrdinal('\n'); + if (newline_indent != NULL && indent_level) { + PyUnicode_AppendAndDel(&newline_indent, + PySequence_Repeat(indent, indent_level)); + } + return newline_indent; +} + static PyObject * encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) { @@ -1267,10 +1278,20 @@ encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) _PyUnicodeWriter_Init(&writer); writer.overallocate = 1; - if (encoder_listencode_obj(self, &writer, obj, indent_level)) { + PyObject *newline_indent = NULL; + if (self->indent != Py_None) { + newline_indent = _create_newline_indent(self->indent, indent_level); + if (newline_indent == NULL) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + } + if (encoder_listencode_obj(self, &writer, obj, newline_indent)) { _PyUnicodeWriter_Dealloc(&writer); + Py_XDECREF(newline_indent); return NULL; } + Py_XDECREF(newline_indent); result = PyTuple_New(1); if (result == NULL || @@ -1358,7 +1379,7 @@ _steal_accumulate(_PyUnicodeWriter *writer, PyObject *stolen) static int encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *obj, Py_ssize_t indent_level) + PyObject *obj, PyObject *newline_indent) { /* Encode Python object obj to a JSON term */ PyObject *newobj; @@ -1394,14 +1415,14 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, writer, obj, indent_level); + rv = encoder_listencode_list(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, writer, obj, indent_level); + rv = encoder_listencode_dict(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } @@ -1435,7 +1456,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_XDECREF(ident); return -1; } - rv = encoder_listencode_obj(s, writer, newobj, indent_level); + rv = encoder_listencode_obj(s, writer, newobj, newline_indent); _Py_LeaveRecursiveCall(); Py_DECREF(newobj); @@ -1456,7 +1477,9 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, static int encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first, - PyObject *key, PyObject *value, Py_ssize_t indent_level) + PyObject *key, PyObject *value, + PyObject *newline_indent, + PyObject *item_separator) { PyObject *keystr = NULL; PyObject *encoded; @@ -1493,7 +1516,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir *first = false; } else { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator) < 0) { + if (_PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { Py_DECREF(keystr); return -1; } @@ -1511,7 +1534,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir if (_PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { return -1; } - if (encoder_listencode_obj(s, writer, value, indent_level) < 0) { + if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) { return -1; } return 0; @@ -1519,13 +1542,15 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir static int encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *dct, Py_ssize_t indent_level) + PyObject *dct, PyObject *newline_indent) { /* Encode Python dict dct a JSON term */ PyObject *ident = NULL; PyObject *items = NULL; PyObject *key, *value; bool first = true; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; if (PyDict_GET_SIZE(dct) == 0) /* Fast path */ return _PyUnicodeWriter_WriteASCIIString(writer, "{}", 2); @@ -1549,14 +1574,21 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, '{')) goto bail; + PyObject *current_item_separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + separator_indent = PyUnicode_Concat(current_item_separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + // update item separator with a borrowed reference + current_item_separator = separator_indent; + if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } } if (s->sort_keys || !PyDict_CheckExact(dct)) { @@ -1574,7 +1606,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, key = PyTuple_GET_ITEM(item, 0); value = PyTuple_GET_ITEM(item, 1); - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } Py_CLEAR(items); @@ -1582,7 +1616,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } } @@ -1592,12 +1628,15 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - /* TODO DOES NOT RUN; dead code if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + + if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ if (_PyUnicodeWriter_WriteChar(writer, '}')) goto bail; return 0; @@ -1605,16 +1644,20 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, bail: Py_XDECREF(items); Py_XDECREF(ident); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } static int encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *seq, Py_ssize_t indent_level) + PyObject *seq, PyObject *newline_indent) { PyObject *ident = NULL; PyObject *s_fast = NULL; Py_ssize_t i; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; ident = NULL; s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); @@ -1643,22 +1686,31 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, '[')) goto bail; + + PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + + if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } + + separator_indent = PyUnicode_Concat(separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + separator = separator_indent; // assign separator with borrowed reference } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); if (i) { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator)) + if (_PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } - if (encoder_listencode_obj(s, writer, obj, indent_level)) + if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) goto bail; } if (ident != NULL) { @@ -1667,12 +1719,14 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_CLEAR(ident); } - /* TODO: DOES NOT RUN if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ if (_PyUnicodeWriter_WriteChar(writer, ']')) goto bail; Py_DECREF(s_fast); @@ -1681,6 +1735,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, bail: Py_XDECREF(ident); Py_DECREF(s_fast); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } @@ -1721,7 +1777,7 @@ encoder_clear(PyEncoderObject *self) return 0; } -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); +PyDoc_STRVAR(encoder_doc, "Encoder(markers, default, encoder, indent, key_separator, item_separator, sort_keys, skipkeys, allow_nan)"); static PyType_Slot PyEncoderType_slots[] = { {Py_tp_doc, (void *)encoder_doc}, @@ -1780,6 +1836,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index fe8e4c5e30035b..d4923442478b3e 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -860,6 +860,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index a76c3dea555783..5cf9eba243bd20 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -121,7 +121,9 @@ call_timer(ProfilerObject *pObj) return CallExternalTimer(pObj); } else { - return _PyTime_PerfCounterUnchecked(); + PyTime_t t; + (void)PyTime_PerfCounterRaw(&t); + return t; } } @@ -175,8 +177,7 @@ normalizeUserObj(PyObject *obj) PyObject *modname = fn->m_module; if (name != NULL) { - PyObject *mo = _PyType_Lookup(Py_TYPE(self), name); - Py_XINCREF(mo); + PyObject *mo = _PyType_LookupRef(Py_TYPE(self), name); Py_DECREF(name); if (mo != NULL) { PyObject *res = PyObject_Repr(mo); @@ -1006,6 +1007,7 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f6bfbfa62687b8..97f3a8f03da9a8 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1604,6 +1604,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 1f6ab718a36984..cee8cf7b9a83c0 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -277,6 +277,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index d332a4e9d9ea0b..aeb2d79de6f9ed 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,10 +2,10 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -128,6 +128,7 @@ static PyMethodDef module_methods[ ] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 5350adb456b859..cc72cb170ceaed 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -367,7 +367,13 @@ _opcode_get_executor_impl(PyObject *module, PyObject *code, int offset) Py_TYPE(code)->tp_name); return NULL; } +#ifdef _Py_TIER2 return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, offset); +#else + PyErr_Format(PyExc_RuntimeError, + "Executors are not available in this build"); + return NULL; +#endif } static PyMethodDef @@ -400,6 +406,7 @@ _opcode_exec(PyObject *m) { static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_operator.c b/Modules/_operator.c index 306d4508f52a68..5d3f88327d19ad 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1928,6 +1928,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d7ffb04c28c2ac..754a326822e0f0 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7863,6 +7863,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index b160cd78177a17..daec4ad708dea4 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1317,6 +1317,7 @@ static PyMethodDef module_methods[] = { static PyModuleDef_Slot _posixsubprocess_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 5db9b645849fcd..aee8db802d8c3f 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -594,6 +594,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 56b891dfe0f85f..140640ae8fbf3a 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -642,6 +642,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 042738b4ab83a2..e9170f2ce1ae87 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,10 +3,10 @@ * using the SystemConfiguration framework. */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -239,6 +239,7 @@ static PyMethodDef mod_methods[] = { static PyModuleDef_Slot _scproxy_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 74984ca5365743..fc03e4a085c179 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -2561,6 +2561,12 @@ set_autocommit(pysqlite_Connection *self, PyObject *val, void *Py_UNUSED(ctx)) return 0; } +static PyObject * +get_sig(PyObject *self, void *Py_UNUSED(ctx)) +{ + return PyUnicode_FromString("(sql, /)"); +} + static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); @@ -2570,6 +2576,7 @@ static PyGetSetDef connection_getset[] = { {"total_changes", (getter)pysqlite_connection_get_total_changes, (setter)0}, {"in_transaction", (getter)pysqlite_connection_get_in_transaction, (setter)0}, {"autocommit", (getter)get_autocommit, (setter)set_autocommit}, + {"__text_signature__", get_sig, (setter)0}, {NULL} }; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 46fed9f13281f3..2c25ee32e58189 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -758,6 +758,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 00fbd9674b8cdd..c1eff63d921de9 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -3272,6 +3272,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index f7fdbf4b6f90cb..9d50b576ba337f 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -5329,7 +5329,11 @@ PySSLSession_clear(PySSLSession *self) static PyObject * PySSLSession_get_time(PySSLSession *self, void *closure) { +#if OPENSSL_VERSION_NUMBER >= 0x30300000L + return _PyLong_FromTime_t(SSL_SESSION_get_time_ex(self->session)); +#else return PyLong_FromLong(SSL_SESSION_get_time(self->session)); +#endif } PyDoc_STRVAR(PySSLSession_get_time_doc, @@ -6511,6 +6515,7 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_strings}, {Py_mod_exec, sslmodule_init_lock}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_stat.c b/Modules/_stat.c index 8059ec2f1f066d..a4f15e8e65e894 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -679,6 +679,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index 78a6552c4c9ec0..b84f731ad6a1da 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -1,9 +1,9 @@ /* statistics accelerator C extension: _statistics module. */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -136,6 +136,7 @@ PyDoc_STRVAR(statistics_doc, static struct PyModuleDef_Slot _statisticsmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index fa2cd37e003e0a..905dcbdeeddc5a 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2593,6 +2593,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c index 30b524d70c1211..80c7179c4c251c 100644 --- a/Modules/_suggestions.c +++ b/Modules/_suggestions.c @@ -49,15 +49,21 @@ static PyMethodDef module_methods[] = { {NULL, NULL, 0, NULL} // Sentinel }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef suggestions_module = { PyModuleDef_HEAD_INIT, "_suggestions", NULL, - -1, - module_methods + 0, + module_methods, + module_slots, }; PyMODINIT_FUNC PyInit__suggestions(void) { - return PyModule_Create(&suggestions_module); + return PyModuleDef_Init(&suggestions_module); } - diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index c76b9e6b3ebafa..c50c5cfabc2f1f 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -80,6 +80,7 @@ static struct PyMethodDef sysconfig_methods[] = { static PyModuleDef_Slot sysconfig_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index cad21bdb4d85ed..54ee468803261a 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1217,7 +1217,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* convert scalar to list */ if (ndim == 0) { - items = Py_BuildValue("(O)", items); + items = PyTuple_Pack(1, items); if (items == NULL) return NULL; } @@ -2901,6 +2901,9 @@ PyInit__testbuffer(void) if (mod == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif if (_testbuffer_exec(mod) < 0) { Py_DECREF(mod); return NULL; diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 4319906dc4fee0..e80d898118daa5 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -1,7 +1,6 @@ #include "parts.h" #include "util.h" - static PyObject * dict_containsstring(PyObject *self, PyObject *args) { @@ -182,6 +181,18 @@ dict_popstring_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_PopString(dict, key, NULL)); } +static PyObject * +dict_version(PyObject *self, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "expected dict"); + return NULL; + } +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag); +_Py_COMP_DIAG_POP +} static PyMethodDef test_methods[] = { {"dict_containsstring", dict_containsstring, METH_VARARGS}, @@ -193,6 +204,7 @@ static PyMethodDef test_methods[] = { {"dict_pop_null", dict_pop_null, METH_VARARGS}, {"dict_popstring", dict_popstring, METH_VARARGS}, {"dict_popstring_null", dict_popstring_null, METH_VARARGS}, + {"dict_version", dict_version, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/docstring.c b/Modules/_testcapi/docstring.c index d99fbdd904b594..3f7acbae1b181b 100644 --- a/Modules/_testcapi/docstring.c +++ b/Modules/_testcapi/docstring.c @@ -169,6 +169,13 @@ static PyMethodDef DocStringUnrepresentableSignatureTest_methods[] = { "--\n\n" "This docstring has a signature with unrepresentable default." )}, + {"with_default", + (PyCFunction)test_with_docstring, METH_VARARGS, + PyDoc_STR( + "with_default($self, /, x=ONE)\n" + "--\n\n" + "This instance method has a default parameter value from the module scope." + )}, {NULL}, }; @@ -193,5 +200,8 @@ _PyTestCapi_Init_Docstring(PyObject *mod) if (PyModule_AddType(mod, &DocStringUnrepresentableSignatureTest) < 0) { return -1; } + if (PyModule_AddObject(mod, "ONE", PyLong_FromLong(1)) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c index f4feaaafbdc6cc..b472a4185a98af 100644 --- a/Modules/_testcapi/gc.c +++ b/Modules/_testcapi/gc.c @@ -99,10 +99,11 @@ slot_tp_del(PyObject *self) return; } /* Execute __del__ method, if any. */ - del = _PyType_Lookup(Py_TYPE(self), tp_del); + del = _PyType_LookupRef(Py_TYPE(self), tp_del); Py_DECREF(tp_del); if (del != NULL) { res = PyObject_CallOneArg(del, self); + Py_DECREF(del); if (res == NULL) PyErr_WriteUnraisable(del); else diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c new file mode 100644 index 00000000000000..aa90cfc06c1536 --- /dev/null +++ b/Modules/_testcapi/monitoring.c @@ -0,0 +1,507 @@ +#include "parts.h" +#include "util.h" + +#include "monitoring.h" + +#define Py_BUILD_CORE +#include "internal/pycore_instruments.h" + +typedef struct { + PyObject_HEAD + PyMonitoringState *monitoring_states; + uint64_t version; + int num_events; + /* Other fields */ +} PyCodeLikeObject; + + +static PyObject * +CodeLike_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + int num_events; + if (!PyArg_ParseTuple(args, "i", &num_events)) { + return NULL; + } + PyMonitoringState *states = (PyMonitoringState *)PyMem_Calloc( + num_events, sizeof(PyMonitoringState)); + if (states == NULL) { + return NULL; + } + PyCodeLikeObject *self = (PyCodeLikeObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->version = 0; + self->monitoring_states = states; + self->num_events = num_events; + } + else { + PyMem_Free(states); + } + return (PyObject *) self; +} + +static void +CodeLike_dealloc(PyCodeLikeObject *self) +{ + if (self->monitoring_states) { + PyMem_Free(self->monitoring_states); + } + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +CodeLike_str(PyCodeLikeObject *self) +{ + PyObject *res = NULL; + PyObject *sep = NULL; + PyObject *parts = NULL; + if (self->monitoring_states) { + parts = PyList_New(0); + if (parts == NULL) { + goto end; + } + + PyObject *heading = PyUnicode_FromString("PyCodeLikeObject"); + if (heading == NULL) { + goto end; + } + int err = PyList_Append(parts, heading); + Py_DECREF(heading); + if (err < 0) { + goto end; + } + + for (int i = 0; i < self->num_events; i++) { + PyObject *part = PyUnicode_FromFormat(" %d", self->monitoring_states[i].active); + if (part == NULL) { + goto end; + } + int err = PyList_Append(parts, part); + Py_XDECREF(part); + if (err < 0) { + goto end; + } + } + sep = PyUnicode_FromString(": "); + if (sep == NULL) { + goto end; + } + res = PyUnicode_Join(sep, parts); + } +end: + Py_XDECREF(sep); + Py_XDECREF(parts); + return res; +} + +static PyTypeObject PyCodeLike_Type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "monitoring.CodeLike", + .tp_doc = PyDoc_STR("CodeLike objects"), + .tp_basicsize = sizeof(PyCodeLikeObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = CodeLike_new, + .tp_dealloc = (destructor) CodeLike_dealloc, + .tp_str = (reprfunc) CodeLike_str, +}; + +#define RAISE_UNLESS_CODELIKE(v) if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \ + PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \ + return NULL; \ + } + +/*******************************************************************/ + +static PyMonitoringState * +setup_fire(PyObject *codelike, int offset, PyObject *exc) +{ + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = ((PyCodeLikeObject *)codelike); + assert(offset >= 0 && offset < cl->num_events); + PyMonitoringState *state = &cl->monitoring_states[offset]; + + if (exc != NULL) { + PyErr_SetRaisedException(Py_NewRef(exc)); + } + return state; +} + +static int +teardown_fire(int res, PyMonitoringState *state, PyObject *exception) +{ + if (res == -1) { + return -1; + } + if (exception) { + assert(PyErr_Occurred()); + assert(((PyObject*)Py_TYPE(exception)) == PyErr_Occurred()); + } + + else { + assert(!PyErr_Occurred()); + } + PyErr_Clear(); + return state->active; +} + +static PyObject * +fire_event_py_start(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyStartEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_resume(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyResumeEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_yield(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_call(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *callable, *arg0; + if (!PyArg_ParseTuple(args, "OiOO", &codelike, &offset, &callable, &arg0)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_line(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset, lineno; + if (!PyArg_ParseTuple(args, "Oii", &codelike, &offset, &lineno)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireLineEvent(state, codelike, offset, lineno); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_jump(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_throw(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyThrowEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_reraise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireReraiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_exception_handled(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireExceptionHandledEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_unwind(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyUnwindEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_stop_iteration(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +/*******************************************************************/ + +static PyObject * +enter_scope(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int event1, event2=0; + Py_ssize_t num_events = PyTuple_Size(args) - 1; + if (num_events == 1) { + if (!PyArg_ParseTuple(args, "Oi", &codelike, &event1)) { + return NULL; + } + } + else { + assert(num_events == 2); + if (!PyArg_ParseTuple(args, "Oii", &codelike, &event1, &event2)) { + return NULL; + } + } + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = (PyCodeLikeObject *) codelike; + + uint8_t events[] = { event1, event2 }; + + PyMonitoring_EnterScope(cl->monitoring_states, + &cl->version, + events, + num_events); + + Py_RETURN_NONE; +} + +static PyObject * +exit_scope(PyObject *self, PyObject *args) +{ + PyMonitoring_ExitScope(); + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"fire_event_py_start", fire_event_py_start, METH_VARARGS}, + {"fire_event_py_resume", fire_event_py_resume, METH_VARARGS}, + {"fire_event_py_return", fire_event_py_return, METH_VARARGS}, + {"fire_event_c_return", fire_event_c_return, METH_VARARGS}, + {"fire_event_py_yield", fire_event_py_yield, METH_VARARGS}, + {"fire_event_call", fire_event_call, METH_VARARGS}, + {"fire_event_line", fire_event_line, METH_VARARGS}, + {"fire_event_jump", fire_event_jump, METH_VARARGS}, + {"fire_event_branch", fire_event_branch, METH_VARARGS}, + {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS}, + {"fire_event_raise", fire_event_raise, METH_VARARGS}, + {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS}, + {"fire_event_reraise", fire_event_reraise, METH_VARARGS}, + {"fire_event_exception_handled", fire_event_exception_handled, METH_VARARGS}, + {"fire_event_py_unwind", fire_event_py_unwind, METH_VARARGS}, + {"fire_event_stop_iteration", fire_event_stop_iteration, METH_VARARGS}, + {"monitoring_enter_scope", enter_scope, METH_VARARGS}, + {"monitoring_exit_scope", exit_scope, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Monitoring(PyObject *m) +{ + if (PyType_Ready(&PyCodeLike_Type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "CodeLike", (PyObject *) &PyCodeLike_Type) < 0) { + Py_DECREF(m); + return -1; + } + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 0e24e44083ea05..41d190961c69ee 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -58,6 +58,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); +int _PyTestCapi_Init_Monitoring(PyObject *module); int _PyTestCapi_Init_Object(PyObject *module); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c index 4fd98b82d762ff..21244d02967ebf 100644 --- a/Modules/_testcapi/run.c +++ b/Modules/_testcapi/run.c @@ -1,5 +1,7 @@ +#define PYTESTCAPI_NEED_INTERNAL_API #include "parts.h" #include "util.h" +#include "pycore_fileutils.h" // _Py_IsValidFD() #include #include @@ -71,21 +73,18 @@ run_fileexflags(PyObject *mod, PyObject *pos_args) PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); return NULL; } + int fd = fileno(fp); result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); -#if defined(__linux__) || defined(MS_WINDOWS) || defined(__APPLE__) - /* The behavior of fileno() after fclose() is undefined, but it is - * the only practical way to check whether the file was closed. - * Only test this on the known platforms. */ - if (closeit && result && fileno(fp) >= 0) { + if (closeit && result && _Py_IsValidFD(fd)) { PyErr_SetString(PyExc_AssertionError, "File was not closed after excution"); Py_DECREF(result); fclose(fp); return NULL; } -#endif - if (!closeit && fileno(fp) < 0) { + + if (!closeit && !_Py_IsValidFD(fd)) { PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after excution"); Py_XDECREF(result); return NULL; diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c index 68f082bf3f3d88..464cf5c3125012 100644 --- a/Modules/_testcapi/time.c +++ b/Modules/_testcapi/time.c @@ -51,6 +51,25 @@ test_pytime_monotonic(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_Monotonic(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_monotonic_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_MonotonicRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_MonotonicRaw() failed"); return NULL; } assert(res == 0); @@ -64,6 +83,25 @@ test_pytime_perf_counter(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_PerfCounter(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_PerfCounterRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_PerfCounterRaw() failed"); return NULL; } assert(res == 0); @@ -77,6 +115,25 @@ test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_Time(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_TimeRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_TimeRaw() failed"); return NULL; } assert(res == 0); @@ -87,8 +144,11 @@ test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) static PyMethodDef test_methods[] = { {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, {"PyTime_Monotonic", test_pytime_monotonic, METH_NOARGS}, + {"PyTime_MonotonicRaw", test_pytime_monotonic_raw, METH_NOARGS}, {"PyTime_PerfCounter", test_pytime_perf_counter, METH_NOARGS}, + {"PyTime_PerfCounterRaw", test_pytime_perf_counter_raw, METH_NOARGS}, {"PyTime_Time", test_pytime_time, METH_NOARGS}, + {"PyTime_TimeRaw", test_pytime_time_raw, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0bdd252efdabc7..ff31724c0e9ff9 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2475,7 +2475,7 @@ get_basic_static_type(PyObject *self, PyObject *args) PyTypeObject *cls = &BasicStaticTypes[num_basic_static_types_used++]; if (base != NULL) { - cls->tp_bases = Py_BuildValue("(O)", base); + cls->tp_bases = PyTuple_Pack(1, base); if (cls->tp_bases == NULL) { return NULL; } @@ -3219,6 +3219,89 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) _Py_COMP_DIAG_POP } +struct simpletracer_data { + int create_count; + int destroy_count; + void* addresses[10]; +}; + +static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { + struct simpletracer_data* the_data = (struct simpletracer_data*)data; + assert(the_data->create_count + the_data->destroy_count < (int)Py_ARRAY_LENGTH(the_data->addresses)); + the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; + if (event == PyRefTracer_CREATE) { + the_data->create_count++; + } else { + the_data->destroy_count++; + } + return 0; +} + +static PyObject * +test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + // Save the current tracer and data to restore it later + void* current_data; + PyRefTracer current_tracer = PyRefTracer_GetTracer(¤t_data); + + struct simpletracer_data tracer_data = {0}; + void* the_data = &tracer_data; + // Install a simple tracer function + if (PyRefTracer_SetTracer(_simpletracer, the_data) != 0) { + goto failed; + } + + // Check that the tracer was correctly installed + void* data; + if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { + PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); + (void)PyRefTracer_SetTracer(NULL, NULL); + goto failed; + } + + // Create a bunch of objects + PyObject* obj = PyList_New(0); + if (obj == NULL) { + goto failed; + } + PyObject* obj2 = PyDict_New(); + if (obj2 == NULL) { + Py_DECREF(obj); + goto failed; + } + + // Kill all objects + Py_DECREF(obj); + Py_DECREF(obj2); + + // Remove the tracer + (void)PyRefTracer_SetTracer(NULL, NULL); + + // Check that the tracer was removed + if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { + PyErr_SetString(PyExc_ValueError, "The reftracer was not correctly removed"); + goto failed; + } + + if (tracer_data.create_count != 2 || + tracer_data.addresses[0] != obj || + tracer_data.addresses[1] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object creation was not correctly traced"); + goto failed; + } + + if (tracer_data.destroy_count != 2 || + tracer_data.addresses[2] != obj || + tracer_data.addresses[3] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); + goto failed; + } + PyRefTracer_SetTracer(current_tracer, current_data); + Py_RETURN_NONE; +failed: + PyRefTracer_SetTracer(current_tracer, current_data); + return NULL; +} static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3257,6 +3340,7 @@ static PyMethodDef TestMethods[] = { {"get_type_fullyqualname", get_type_fullyqualname, METH_O}, {"get_type_module_name", get_type_module_name, METH_O}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, + {"test_reftracer", test_reftracer, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, @@ -3474,7 +3558,7 @@ typedef struct { static PyObject * ipowType_ipow(PyObject *self, PyObject *other, PyObject *mod) { - return Py_BuildValue("OO", other, mod); + return PyTuple_Pack(2, other, mod); } static PyNumberMethods ipowType_as_number = { @@ -3851,6 +3935,9 @@ PyInit__testcapi(void) m = PyModule_Create(&_testcapimodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { @@ -4048,6 +4135,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Time(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Monitoring(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Object(m) < 0) { return NULL; } diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 454173b434fb6b..4187e13231dc69 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1243,7 +1243,7 @@ _testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, PyObject *arg) /*[clinic end generated code: output=fe7e49d96cbb7718 input=d1b83d3b853af6d9]*/ { - return Py_BuildValue("(OO)", cls, arg); + return PyTuple_Pack(2, cls, arg); } static struct PyMethodDef test_class_methods[] = { @@ -1955,6 +1955,9 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddType(m, &TestClass) < 0) { goto error; } diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 29f1b7c13e4c50..370433b3e2a0d9 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -146,5 +146,8 @@ PyInit__testclinic_limited(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif return m; } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index e2f96cdad5c58e..2a665affb5e7f8 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -553,12 +553,12 @@ get_stack_trace(PyObject* self, PyObject* args) if (bytes_read == -1) { return NULL; } - off_t thread_state_list_head = local_debug_offsets.runtime_state.interpreters_head; + off_t interpreter_state_list_head = local_debug_offsets.runtime_state.interpreters_head; void* address_of_interpreter_state; bytes_read = read_memory( pid, - (void*)(runtime_start_address + thread_state_list_head), + (void*)(runtime_start_address + interpreter_state_list_head), sizeof(void*), &address_of_interpreter_state); if (bytes_read == -1) { @@ -627,6 +627,12 @@ PyMODINIT_FUNC PyInit__testexternalinspection(void) { PyObject* mod = PyModule_Create(&module); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { Py_DECREF(mod); diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index a65ca513a12516..c147596f88a3a8 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -6,18 +6,24 @@ #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x03020000 +# define Py_LIMITED_API 0x030d0000 #endif #include +static PyModuleDef_Slot shared_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _testimportmultiple = { PyModuleDef_HEAD_INIT, "_testimportmultiple", "_testimportmultiple doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -25,16 +31,16 @@ static struct PyModuleDef _testimportmultiple = { PyMODINIT_FUNC PyInit__testimportmultiple(void) { - return PyModule_Create(&_testimportmultiple); + return PyModuleDef_Init(&_testimportmultiple); } static struct PyModuleDef _foomodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_foo", "_testimportmultiple_foo doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -42,21 +48,21 @@ static struct PyModuleDef _foomodule = { PyMODINIT_FUNC PyInit__testimportmultiple_foo(void) { - return PyModule_Create(&_foomodule); + return PyModuleDef_Init(&_foomodule); } static struct PyModuleDef _barmodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_bar", "_testimportmultiple_bar doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL }; PyMODINIT_FUNC PyInit__testimportmultiple_bar(void){ - return PyModule_Create(&_barmodule); + return PyModuleDef_Init(&_barmodule); } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 99e80baa223e8f..e209c7e05264f2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -985,6 +985,8 @@ get_co_framesize(PyObject *self, PyObject *arg) return PyLong_FromLong(code->co_framesize); } +#ifdef _Py_TIER2 + static PyObject * new_counter_optimizer(PyObject *self, PyObject *arg) { @@ -1012,7 +1014,10 @@ set_optimizer(PyObject *self, PyObject *opt) static PyObject * get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *opt = (PyObject *)PyUnstable_GetOptimizer(); + PyObject *opt = NULL; +#ifdef _Py_TIER2 + opt = (PyObject *)PyUnstable_GetOptimizer(); +#endif if (opt == NULL) { Py_RETURN_NONE; } @@ -1045,6 +1050,8 @@ invalidate_executors(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +#endif + static int _pending_callback(void *arg) { /* we assume the argument is callable object to which we own a reference */ @@ -1978,6 +1985,17 @@ set_immortalize_deferred(PyObject *self, PyObject *value) #endif } +static PyObject * +get_immortalize_deferred(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = PyInterpreterState_Get(); + return PyBool_FromLong(interp->gc.immortalize.enable_on_thread_created); +#else + Py_RETURN_FALSE; +#endif +} + static PyObject * has_inline_values(PyObject *self, PyObject *obj) { @@ -2020,12 +2038,14 @@ static PyMethodDef module_functions[] = { {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, {"get_co_framesize", get_co_framesize, METH_O, NULL}, +#ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, +#endif {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, @@ -2072,7 +2092,10 @@ static PyMethodDef module_functions[] = { {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif {"set_immortalize_deferred", set_immortalize_deferred, METH_VARARGS}, + {"get_immortalize_deferred", get_immortalize_deferred, METH_NOARGS}, +#ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; @@ -2131,6 +2154,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index 94da0468fcf149..cdf8a70fc79ff3 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -204,6 +204,90 @@ test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) Py_DECREF(test_data.obj1); Py_RETURN_NONE; } + +static void +pysleep(int ms) +{ +#ifdef MS_WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +struct test_data_gc { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t id; + Py_ssize_t countdown; + PyEvent done_event; + PyEvent ready; +}; + +static void +thread_gc(void *arg) +{ + struct test_data_gc *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + Py_ssize_t id = _Py_atomic_add_ssize(&test_data->id, 1); + if (id == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + else { + // wait for all test threads to more reliably reproduce the issue. + PyEvent_Wait(&test_data->ready); + } + + if (id == 0) { + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + // pause long enough that the lock would be handed off directly to + // a waiting thread. + pysleep(5); + PyGC_Collect(); + Py_END_CRITICAL_SECTION(); + } + else if (id == 1) { + pysleep(1); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + else if (id == 2) { + // sleep long enough so that thread 0 is waiting to stop the world + pysleep(6); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + + PyGILState_Release(gil); + if (_Py_atomic_add_ssize(&test_data->countdown, -1) == 1) { + // last thread to finish sets done_event + _PyEvent_Notify(&test_data->done_event); + } +} + +static PyObject * +test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-118332: Contended critical sections should not deadlock with GC + const Py_ssize_t NUM_THREADS = 3; + struct test_data_gc test_data = { + .obj = PyDict_New(), + .countdown = NUM_THREADS, + .num_threads = NUM_THREADS, + }; + assert(test_data.obj != NULL); + + for (int i = 0; i < NUM_THREADS; i++) { + PyThread_start_new_thread(&thread_gc, &test_data); + } + PyEvent_Wait(&test_data.done_event); + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + #endif static PyMethodDef test_methods[] = { @@ -212,6 +296,7 @@ static PyMethodDef test_methods[] = { {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, + {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, #endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 1c5048170e9f2e..4900459c689279 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -2,7 +2,6 @@ #include "parts.h" #include "pycore_lock.h" -#include "pycore_time.h" // _PyTime_MonotonicUnchecked() #include "clinic/test_lock.c.h" @@ -290,7 +289,10 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, goto exit; } - PyTime_t start = _PyTime_MonotonicUnchecked(); + PyTime_t start, end; + if (PyTime_PerfCounter(&start) < 0) { + goto exit; + } for (Py_ssize_t i = 0; i < num_threads; i++) { thread_data[i].bench_data = &bench_data; @@ -307,7 +309,9 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, } Py_ssize_t total_iters = bench_data.total_iters; - PyTime_t end = _PyTime_MonotonicUnchecked(); + if (PyTime_PerfCounter(&end) < 0) { + goto exit; + } // Return the total number of acquisitions and the number of acquisitions // for each thread. @@ -319,7 +323,8 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, PyList_SET_ITEM(thread_iters, i, iter); } - double rate = total_iters * 1000000000.0 / (end - start); + assert(end != start); + double rate = total_iters * 1e9 / (end - start); res = Py_BuildValue("(dO)", rate, thread_iters); exit: diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 598071fe0ddbad..fb5cdb6ca9e1d3 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -25,6 +25,9 @@ PyInit__testlimitedcapi(void) if (mod == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif if (_PyTestLimitedCAPI_Init_Abstract(mod) < 0) { return NULL; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 21c5f696a4f2ec..886b260aceb20d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -431,6 +431,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -519,13 +520,18 @@ PyInit__testmultiphase_nonmodule_with_methods(void) /**** Non-ASCII-named modules ****/ +static PyModuleDef_Slot nonascii_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef def_nonascii_latin = { \ PyModuleDef_HEAD_INIT, /* m_base */ "_testmultiphase_nonascii_latin", /* m_name */ PyDoc_STR("Module named in Czech"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -543,7 +549,7 @@ static PyModuleDef def_nonascii_kana = { \ PyDoc_STR("Module named in Japanese"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -757,6 +763,7 @@ static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -778,6 +785,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -800,6 +808,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -822,6 +831,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -857,6 +867,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -889,6 +900,9 @@ PyInit__test_module_state_shared(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddObjectRef(module, "Error", PyExc_Exception) < 0) { Py_DECREF(module); @@ -903,6 +917,7 @@ PyInit__test_module_state_shared(void) static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -920,6 +935,7 @@ PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) static PyModuleDef_Slot non_isolated_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -940,6 +956,7 @@ static PyModuleDef_Slot shared_gil_only_slots[] = { We put it here explicitly to draw attention to the contrast with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 092673a9ea43e1..448be502466e79 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,6 +1,172 @@ /* Testing module for single-phase initialization of extension modules - */ + +This file contains 5 distinct modules, meaning each as its own name +and its own init function (PyInit_...). The default import system will +only find the one matching the filename: _testsinglephase. To load the +others you must do so manually. For example: + +```python +name = '_testsinglephase_base_wrapper' +filename = _testsinglephase.__file__ +loader = importlib.machinery.ExtensionFileLoader(name, filename) +spec = importlib.util.spec_from_file_location(name, filename, loader=loader) +mod = importlib._bootstrap._load(spec) +``` + +Here are the 5 modules: + +* _testsinglephase + * def: _testsinglephase_basic, + * m_name: "_testsinglephase" + * m_size: -1 + * state + * process-global + * initialized_count (default to -1; will never be 0) + * module (see module state below) + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. clear .module + 3. initialize .module: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions + * (3 common, see below) + * initialized_count() - return .module.initialized_count + * import system + * caches + * global extensions cache: yes + * def.m_base.m_copy: yes + * def.m_base.m_init: no + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + 1. get init function from shared object file + 2. run init function + 3. copy __dict__ into def.m_base.m_copy + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. get module from sys.modules + 3. update module with contents of def.m_base.m_copy + * already loaded in other interpreter (already in global cache) + * same as reload, but create new module and update *it* + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * same as main interpreter + * unload: never (all single-phase init modules) +* _testsinglephase_basic_wrapper + * identical to _testsinglephase except module name +* _testsinglephase_basic_copy + * def: static local variable in init function + * m_name: "_testsinglephase_basic_copy" + * m_size: -1 + * state: same as _testsinglephase + * init function: same as _testsinglephase + * functions: same as _testsinglephase + * import system: same as _testsinglephase +* _testsinglephase_with_reinit + * def: _testsinglephase_with_reinit, + * m_name: "_testsinglephase_with_reinit" + * m_size: 0 + * state + * process-global state: no + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. initialize temporary module state (local var): see module state below + 3. initialize module: set initial __dict__ + * functions: see common functions below + * import system + * caches + * global extensions cache: only if loaded in main interpreter + * def.m_base.m_copy: no + * def.m_base.m_init: only if loaded in the main interpreter + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + * (same as _testsinglephase except step 3) + 1. get init function from shared object file + 2. run init function + 3. set def.m_base.m_init to the init function + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. call def->m_base.m_init to get a new module object + 3. replace the existing module in sys.modules + * already loaded in other interpreter (already in global cache) + * same as reload (since will only be in cache for main interp) + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * initial (not already in global cache) + * (same as main interpreter except skip steps 3 & 4 there) + 1. get init function from shared object file + 2. run init function + ... + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + * same as initial (load from scratch) + * already loaded in other interpreter (already in global cache) + * same as initial (load from scratch) + * not in any sys.modules, still in global cache + * same as initial (load from scratch) + * unload: never (all single-phase init modules) +* _testsinglephase_with_state + * def: _testsinglephase_with_state, + * m_name: "_testsinglephase_with_state" + * m_size: sizeof(module_state) + * state + * process-global: no + * module state: see module state below + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 3. initialize module state: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions: see common functions below + * import system: same as _testsinglephase_basic_copy + +Module state: + +* fields + * initialized - when the module was first initialized + * *error + * *int_const + * *str_const +* initialization + 1. set state.initialized to the current time + 2. set state.error to a new exception class + 3. set state->int_const to int(1969) + 4. set state->str_const to "something different" + +Common initial __dict__: + +* error: state.error +* int_const: state.int_const +* str_const: state.str_const +* _module_initialized: state.initialized + +Common functions: + +* look_up_self() - return the module from the per-interpreter "by-index" cache +* sum() - return a + b +* state_initialized() - return state->initialized (or None if m_size == 0) + +See Python/import.c, especially the long comments, for more about +single-phase init modules. +*/ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif @@ -305,6 +471,9 @@ init__testsinglephase_basic(PyModuleDef *def) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = &global_state.module; // It may have been set by a previous run or under a different name. @@ -396,6 +565,9 @@ PyInit__testsinglephase_with_reinit(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif assert(get_module_state(module) == NULL); @@ -458,6 +630,9 @@ PyInit__testsinglephase_with_state(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = get_module_state(module); assert(state != NULL); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 5aa719c3834e61..39d309729d88b8 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -501,7 +501,8 @@ ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) // Wait until the deadline for the thread to exit. PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0; - while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns)) { + int detach = 1; + while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns, detach)) { if (deadline) { // _PyDeadline_Get will return a negative value if the deadline has // been exceeded. @@ -2543,6 +2544,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index e3789867dc085f..c7e271faa4cf34 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -306,6 +306,7 @@ typedef struct { int threaded; /* True if tcl_platform[threaded] */ Tcl_ThreadId thread_id; int dispatching; + PyObject *trace; /* We cannot include tclInt.h, as this is internal. So we cache interesting types here. */ const Tcl_ObjType *OldBooleanType; @@ -570,6 +571,7 @@ Tkapp_New(const char *screenName, const char *className, TCL_GLOBAL_ONLY) != NULL; v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; + v->trace = NULL; #ifndef TCL_THREADS if (v->threaded) { @@ -1306,6 +1308,29 @@ Tkapp_ObjectResult(TkappObject *self) return res; } +static int +Tkapp_Trace(TkappObject *self, PyObject *args) +{ + if (args == NULL) { + return 0; + } + if (self->trace) { + PyObject *res = PyObject_CallObject(self->trace, args); + if (res == NULL) { + Py_DECREF(args); + return 0; + } + Py_DECREF(res); + } + Py_DECREF(args); + return 1; +} + +#define TRACE(_self, ARGS) do { \ + if ((_self)->trace && !Tkapp_Trace((_self), Py_BuildValue ARGS)) { \ + return NULL; \ + } \ + } while (0) /* Tkapp_CallProc is the event procedure that is executed in the context of the Tcl interpreter thread. Initially, it holds the Tcl lock, and doesn't @@ -1320,7 +1345,12 @@ Tkapp_CallProc(Tcl_Event *evPtr, int flags) int objc; int i; ENTER_PYTHON - objv = Tkapp_CallArgs(e->args, objStore, &objc); + if (e->self->trace && !Tkapp_Trace(e->self, PyTuple_Pack(1, e->args))) { + objv = NULL; + } + else { + objv = Tkapp_CallArgs(e->args, objStore, &objc); + } if (!objv) { *(e->exc) = PyErr_GetRaisedException(); *(e->res) = NULL; @@ -1413,6 +1443,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) } else { + TRACE(self, ("(O)", args)); objv = Tkapp_CallArgs(args, objStore, &objc); if (!objv) @@ -1455,6 +1486,8 @@ _tkinter_tkapp_eval_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "eval", script)); + ENTER_TCL err = Tcl_Eval(Tkapp_Interp(self), script); ENTER_OVERLAP @@ -1484,6 +1517,8 @@ _tkinter_tkapp_evalfile_impl(TkappObject *self, const char *fileName) CHECK_STRING_LENGTH(fileName); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "source", fileName)); + ENTER_TCL err = Tcl_EvalFile(Tkapp_Interp(self), fileName); ENTER_OVERLAP @@ -1513,6 +1548,8 @@ _tkinter_tkapp_record_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ssss))", "history", "add", script, "exec")); + ENTER_TCL err = Tcl_RecordAndEval(Tkapp_Interp(self), script, TCL_NO_EVAL); ENTER_OVERLAP @@ -1702,6 +1739,15 @@ SetVar(TkappObject *self, PyObject *args, int flags) newval = AsObj(newValue); if (newval == NULL) return NULL; + + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((ssssO))", "uplevel", "#0", "set", + name1, newValue)); + } + else { + TRACE((TkappObject *)self, ("((ssO))", "set", name1, newValue)); + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, newval, flags); @@ -1719,8 +1765,22 @@ SetVar(TkappObject *self, PyObject *args, int flags) return NULL; CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + /* XXX must hold tcl lock already??? */ newval = AsObj(newValue); + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((sssNO))", "uplevel", "#0", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + else { + TRACE((TkappObject *)self, ("((sNO))", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); ENTER_OVERLAP @@ -1807,6 +1867,28 @@ UnsetVar(TkappObject *self, PyObject *args, int flags) CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + if (name2) { + TRACE((TkappObject *)self, ("((sssN))", "uplevel", "#0", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ssss))", "uplevel", "#0", "unset", name1)); + } + } + else { + if (name2) { + TRACE((TkappObject *)self, ("((sN))", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ss))", "unset", name1)); + } + } + } + ENTER_TCL code = Tcl_UnsetVar2(Tkapp_Interp(self), name1, name2, flags); ENTER_OVERLAP @@ -1973,6 +2055,8 @@ _tkinter_tkapp_exprstring_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprString(Tkapp_Interp(self), s); ENTER_OVERLAP @@ -2003,6 +2087,8 @@ _tkinter_tkapp_exprlong_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprLong(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2032,6 +2118,9 @@ _tkinter_tkapp_exprdouble_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprDouble(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2061,6 +2150,9 @@ _tkinter_tkapp_exprboolean_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprBoolean(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2156,7 +2248,7 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /* Client data struct */ typedef struct { - PyObject *self; + TkappObject *self; PyObject *func; } PythonCmd_ClientData; @@ -2180,6 +2272,7 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, PyObject *args, *res; int i; Tcl_Obj *obj_res; + int objargs = data->self->wantobjects >= 2; ENTER_PYTHON @@ -2188,7 +2281,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, return PythonCmd_Error(interp); for (i = 0; i < (objc - 1); i++) { - PyObject *s = unicodeFromTclObj(objv[i + 1]); + PyObject *s = objargs ? FromObj(data->self, objv[i + 1]) + : unicodeFromTclObj(objv[i + 1]); if (!s) { Py_DECREF(args); return PythonCmd_Error(interp); @@ -2286,10 +2380,13 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, !WaitForMainloop(self)) return NULL; + TRACE(self, ("((ss()O))", "proc", name, func)); + data = PyMem_NEW(PythonCmd_ClientData, 1); if (!data) return PyErr_NoMemory(); - data->self = Py_NewRef(self); + Py_INCREF(self); + data->self = self; data->func = Py_NewRef(func); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { Tcl_Condition cond = NULL; @@ -2344,6 +2441,8 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) CHECK_STRING_LENGTH(name); + TRACE(self, ("((sss))", "rename", name, "")); + if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { Tcl_Condition cond = NULL; CommandEvent *ev; @@ -2469,6 +2568,8 @@ _tkinter_tkapp_createfilehandler_impl(TkappObject *self, PyObject *file, return NULL; } + TRACE(self, ("((ssiiO))", "#", "createfilehandler", tfile, mask, func)); + data = NewFHCD(func, file, tfile); if (data == NULL) return NULL; @@ -2500,6 +2601,8 @@ _tkinter_tkapp_deletefilehandler(TkappObject *self, PyObject *file) if (tfile < 0) return NULL; + TRACE(self, ("((ssi))", "#", "deletefilehandler", tfile)); + DeleteFHCD(tfile); /* Ought to check for null Tcl_File object... */ @@ -2534,6 +2637,7 @@ _tkinter_tktimertoken_deletetimerhandler_impl(TkttObject *self) PyObject *func = v->func; if (v->token != NULL) { + /* TRACE(...) */ Tcl_DeleteTimerHandler(v->token); v->token = NULL; } @@ -2636,6 +2740,8 @@ _tkinter_tkapp_createtimerhandler_impl(TkappObject *self, int milliseconds, CHECK_TCL_APPARTMENT; + TRACE(self, ("((siO))", "after", milliseconds, func)); + v = Tktt_New(func); if (v) { v->token = Tcl_CreateTimerHandler(milliseconds, TimerHandler, @@ -2794,15 +2900,56 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) { int wantobjects = -1; - if (!PyArg_ParseTuple(args, "|p:wantobjects", &wantobjects)) + if (!PyArg_ParseTuple(args, "|i:wantobjects", &wantobjects)) return NULL; if (wantobjects == -1) - return PyBool_FromLong(((TkappObject*)self)->wantobjects); + return PyLong_FromLong(((TkappObject*)self)->wantobjects); ((TkappObject*)self)->wantobjects = wantobjects; Py_RETURN_NONE; } +/*[clinic input] +_tkinter.tkapp.settrace + + func: object + / + +Set the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_settrace(TkappObject *self, PyObject *func) +/*[clinic end generated code: output=847f6ebdf46e84fa input=31b260d46d3d018a]*/ +{ + if (func == Py_None) { + func = NULL; + } + else { + Py_INCREF(func); + } + Py_XSETREF(self->trace, func); + Py_RETURN_NONE; +} + +/*[clinic input] +_tkinter.tkapp.gettrace + +Get the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self) +/*[clinic end generated code: output=d4e2ba7d63e77bb5 input=ac2aea5be74e8c4c]*/ +{ + PyObject *func = self->trace; + if (!func) { + func = Py_None; + } + Py_INCREF(func); + return func; +} + /*[clinic input] _tkinter.tkapp.willdispatch @@ -2828,6 +2975,7 @@ Tkapp_Dealloc(PyObject *self) ENTER_TCL Tcl_DeleteInterp(Tkapp_Interp(self)); LEAVE_TCL + Py_XDECREF(((TkappObject *)self)->trace); PyObject_Free(self); Py_DECREF(tp); DisableEventHook(); @@ -2941,7 +3089,7 @@ _tkinter.create baseName: str = "" className: str = "Tk" interactive: bool = False - wantobjects: bool = False + wantobjects: int = 0 wantTk: bool = True if false, then Tk_Init() doesn't get called sync: bool = False @@ -2957,7 +3105,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName, const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=e3315607648e6bb4 input=09afef9adea70a19]*/ +/*[clinic end generated code: output=e3315607648e6bb4 input=7e382ba431bed537]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ @@ -3038,6 +3186,8 @@ static PyMethodDef Tkapp_methods[] = { _TKINTER_TKAPP_WILLDISPATCH_METHODDEF {"wantobjects", Tkapp_WantObjects, METH_VARARGS}, + _TKINTER_TKAPP_SETTRACE_METHODDEF + _TKINTER_TKAPP_GETTRACE_METHODDEF {"call", Tkapp_Call, METH_VARARGS}, _TKINTER_TKAPP_EVAL_METHODDEF _TKINTER_TKAPP_EVALFILE_METHODDEF @@ -3205,6 +3355,9 @@ PyInit__tkinter(void) m = PyModule_Create(&_tkintermodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Tkinter_TclError = PyErr_NewException("_tkinter.TclError", NULL, NULL); if (PyModule_AddObjectRef(m, "TclError", Tkinter_TclError)) { diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 6dba3cac01c1c8..887a1e820e250e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -219,6 +219,9 @@ PyInit__tracemalloc(void) m = PyModule_Create(&module_def); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (_PyTraceMalloc_Init() < 0) { Py_DECREF(m); diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 9ea72bf89ce0b2..09fbb3c5e8b91d 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -63,12 +63,16 @@ _typing_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeAliasType", (PyObject *)&_PyTypeAlias_Type) < 0) { return -1; } + if (PyModule_AddObjectRef(m, "NoDefault", (PyObject *)&_Py_NoDefaultStruct) < 0) { + return -1; + } return 0; } static struct PyModuleDef_Slot _typingmodule_slots[] = { {Py_mod_exec, _typing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index 052cb9fef3b21c..c5e78b1510b5e3 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,10 +3,10 @@ * DCE compatible Universally Unique Identifier library. */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -111,6 +111,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 1ea3ed5e40b761..a5c15c0f10b930 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -171,6 +171,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 23e3c0d87f0319..cd5dd503abe61f 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -3189,6 +3189,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxtestfuzz/_xxtestfuzz.c b/Modules/_xxtestfuzz/_xxtestfuzz.c index a2dbabce71ed67..2952d7043e01fe 100644 --- a/Modules/_xxtestfuzz/_xxtestfuzz.c +++ b/Modules/_xxtestfuzz/_xxtestfuzz.c @@ -28,13 +28,18 @@ static PyMethodDef module_methods[] = { {NULL}, }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _fuzzmodule = { PyModuleDef_HEAD_INIT, "_fuzz", NULL, 0, module_methods, - NULL, + module_slots, NULL, NULL, NULL @@ -43,5 +48,5 @@ static struct PyModuleDef _fuzzmodule = { PyMODINIT_FUNC PyInit__xxtestfuzz(void) { - return PyModule_Create(&_fuzzmodule); + return PyModuleDef_Init(&_fuzzmodule); } diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index fcd4af64df0be9..38c3f0c45d803f 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2760,6 +2760,7 @@ zoneinfomodule_exec(PyObject *m) static PyModuleDef_Slot zoneinfomodule_slots[] = { {Py_mod_exec, zoneinfomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 317f4974814945..a3b833d47cd9ea 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3220,6 +3220,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 8e908da2534c55..297a8d74ba3bf4 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -322,6 +322,7 @@ Two public functions, register and unregister, are defined.\n\ static PyModuleDef_Slot atexitmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 86493241a1fb7e..6bb01d148b6faa 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -414,6 +414,13 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) if (this_ch == BASE64_PAD) { padding_started = 1; + if (strict_mode && quad_pos == 0) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Excess padding not allowed"); + } + goto error_end; + } if (quad_pos >= 2 && quad_pos + ++pads >= 4) { /* A pad sequence means we should not parse more input. ** We've already interpreted the data from the quad at this point. @@ -1278,6 +1285,7 @@ binascii_exec(PyObject *module) static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 766f82983025e4..2b446ba5226ac0 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -503,6 +503,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index e5433d7dd85306..1c671adb4ff188 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -2094,6 +2094,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 188bcc773cfc41..2b1ac954b4d570 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -622,6 +622,33 @@ _tkinter_tkapp_loadtk(TkappObject *self, PyObject *Py_UNUSED(ignored)) return _tkinter_tkapp_loadtk_impl(self); } +PyDoc_STRVAR(_tkinter_tkapp_settrace__doc__, +"settrace($self, func, /)\n" +"--\n" +"\n" +"Set the tracing function."); + +#define _TKINTER_TKAPP_SETTRACE_METHODDEF \ + {"settrace", (PyCFunction)_tkinter_tkapp_settrace, METH_O, _tkinter_tkapp_settrace__doc__}, + +PyDoc_STRVAR(_tkinter_tkapp_gettrace__doc__, +"gettrace($self, /)\n" +"--\n" +"\n" +"Get the tracing function."); + +#define _TKINTER_TKAPP_GETTRACE_METHODDEF \ + {"gettrace", (PyCFunction)_tkinter_tkapp_gettrace, METH_NOARGS, _tkinter_tkapp_gettrace__doc__}, + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self); + +static PyObject * +_tkinter_tkapp_gettrace(TkappObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _tkinter_tkapp_gettrace_impl(self); +} + PyDoc_STRVAR(_tkinter_tkapp_willdispatch__doc__, "willdispatch($self, /)\n" "--\n" @@ -649,7 +676,7 @@ PyDoc_STRVAR(_tkinter__flatten__doc__, PyDoc_STRVAR(_tkinter_create__doc__, "create($module, screenName=None, baseName=\'\', className=\'Tk\',\n" -" interactive=False, wantobjects=False, wantTk=True, sync=False,\n" +" interactive=False, wantobjects=0, wantTk=True, sync=False,\n" " use=None, /)\n" "--\n" "\n" @@ -750,8 +777,8 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - wantobjects = PyObject_IsTrue(args[4]); - if (wantobjects < 0) { + wantobjects = PyLong_AsInt(args[4]); + if (wantobjects == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 6) { @@ -861,4 +888,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=d447501ec5aa9447 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d90c1a9850c63249 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 57bc55632be485..d901b350bc5343 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1363,6 +1363,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 97e5f0180d76fb..3f96f2f846d612 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,9 +1,9 @@ /* Errno module */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -951,6 +951,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index c70d43a36b5cc7..cfa3cbdc34bc86 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1292,6 +1292,7 @@ static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, // XXX gh-103092: fix isolation. //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index e24e5f98f4bc4d..b6eeec2c66f6e5 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -745,6 +745,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 8a1b483eddae35..57e4aae9ed557e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -535,6 +535,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index a1fa6cf20f71fd..f7d3e12f347ec2 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -342,6 +342,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6ee447ef6a8cd6..8641c2f87e6db2 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -270,6 +270,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; + PyObject *result; } pairwiseobject; /*[clinic input] @@ -301,6 +302,11 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; + po->result = PyTuple_Pack(2, Py_None, Py_None); + if (po->result == NULL) { + Py_DECREF(po); + return NULL; + } return (PyObject *)po; } @@ -311,6 +317,7 @@ pairwise_dealloc(pairwiseobject *po) PyObject_GC_UnTrack(po); Py_XDECREF(po->it); Py_XDECREF(po->old); + Py_XDECREF(po->result); tp->tp_free(po); Py_DECREF(tp); } @@ -321,6 +328,7 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) Py_VISIT(Py_TYPE(po)); Py_VISIT(po->it); Py_VISIT(po->old); + Py_VISIT(po->result); return 0; } @@ -355,8 +363,30 @@ pairwise_next(pairwiseobject *po) Py_DECREF(old); return NULL; } - /* Future optimization: Reuse the result tuple as we do in enumerate() */ - result = PyTuple_Pack(2, old, new); + + result = po->result; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + PyObject *last_old = PyTuple_GET_ITEM(result, 0); + PyObject *last_new = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + Py_DECREF(last_old); + Py_DECREF(last_new); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result != NULL) { + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + } + } + Py_XSETREF(po->old, new); Py_DECREF(old); return result; @@ -4751,6 +4781,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/main.c b/Modules/main.c index df2ce550245088..8eded2639ad90a 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -513,8 +513,13 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + return (run != 0); + } + int run = pymain_run_module(L"_pyrepl", 0); return (run != 0); } @@ -537,9 +542,15 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int res = PyRun_AnyFileFlags(stdin, "", &cf); - *exitcode = (res != 0); + if (!isatty(fileno(stdin))) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + *exitcode = (run != 0); + return; + } + int run = pymain_run_module(L"_pyrepl", 0); + *exitcode = (run != 0); + return; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 8ba0431f4a47b7..a79694730a8e4e 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1253,7 +1253,7 @@ FUNC1(tanh, tanh, 0, "Return the hyperbolic tangent of x.") /* Precision summation function as msum() by Raymond Hettinger in - , + , enhanced with the exact partials sum and roundoff from Mark Dickinson's post at . See those links for more details, proofs and other references. @@ -4177,6 +4177,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 9cbf11feaa9c32..ef9163e8be5b6c 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -375,6 +375,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 0cce7c27f9b16a..dfc16ff4370349 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1801,6 +1801,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index b9881d91ded244..77ee70ae133c85 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -2070,6 +2070,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c9d67ccbb8c908..9f4be98b35186e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -37,6 +37,8 @@ # include # include // UNLEN # include "osdefs.h" // SEP +# include // SetEntriesInAcl +# include // SDDL_REVISION_1 # if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SYMLINK # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ @@ -5500,7 +5502,7 @@ os__path_splitroot_ex_impl(PyObject *module, PyObject *path) if (tail == NULL) { goto exit; } - result = Py_BuildValue("(OOO)", drv, root, tail); + result = PyTuple_Pack(3, drv, root, tail); exit: PyMem_Free(buffer); Py_XDECREF(drv); @@ -5539,6 +5541,146 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } +#ifdef MS_WINDOWS + +/* We centralise SECURITY_ATTRIBUTE initialization based around +templates that will probably mostly match common POSIX mode settings. +The _Py_SECURITY_ATTRIBUTE_DATA structure contains temporary data, as +a constructed SECURITY_ATTRIBUTE structure typically refers to memory +that has to be alive while it's being used. + +Typical use will look like: + SECURITY_ATTRIBUTES *pSecAttr = NULL; + struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; + int error, error2; + + Py_BEGIN_ALLOW_THREADS + switch (mode) { + case 0x1C0: // 0o700 + error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); + break; + ... + default: + error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); + break; + } + + if (!error) { + // do operation, passing pSecAttr + } + + // Unconditionally clear secAttrData. + error2 = clearSecurityAttributes(&pSecAttr, &secAttrData); + if (!error) { + error = error2; + } + Py_END_ALLOW_THREADS + + if (error) { + PyErr_SetFromWindowsErr(error); + return NULL; + } +*/ + +struct _Py_SECURITY_ATTRIBUTE_DATA { + SECURITY_ATTRIBUTES securityAttributes; + PACL acl; + SECURITY_DESCRIPTOR sd; + EXPLICIT_ACCESS_W ea[4]; + char sid[64]; +}; + +static int +initializeDefaultSecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + memset(data, 0, sizeof(*data)); + return 0; +} + +static int +initializeMkdir700SecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + memset(data, 0, sizeof(*data)); + + if (!InitializeSecurityDescriptor(&data->sd, SECURITY_DESCRIPTOR_REVISION) + || !SetSecurityDescriptorGroup(&data->sd, NULL, TRUE)) { + return GetLastError(); + } + + int use_alias = 0; + DWORD cbSid = sizeof(data->sid); + if (!CreateWellKnownSid(WinCreatorOwnerRightsSid, NULL, (PSID)data->sid, &cbSid)) { + use_alias = 1; + } + + data->securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES); + data->ea[0].grfAccessPermissions = GENERIC_ALL; + data->ea[0].grfAccessMode = SET_ACCESS; + data->ea[0].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + if (use_alias) { + data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; + } else { + data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID; + data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; + data->ea[0].Trustee.ptstrName = (LPWCH)(SID*)data->sid; + } + + data->ea[1].grfAccessPermissions = GENERIC_ALL; + data->ea[1].grfAccessMode = SET_ACCESS; + data->ea[1].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + data->ea[1].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[1].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[1].Trustee.ptstrName = L"SYSTEM"; + + data->ea[2].grfAccessPermissions = GENERIC_ALL; + data->ea[2].grfAccessMode = SET_ACCESS; + data->ea[2].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + data->ea[2].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[2].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[2].Trustee.ptstrName = L"ADMINISTRATORS"; + + int r = SetEntriesInAclW(3, data->ea, NULL, &data->acl); + if (r) { + return r; + } + if (!SetSecurityDescriptorDacl(&data->sd, TRUE, data->acl, FALSE)) { + return GetLastError(); + } + data->securityAttributes.lpSecurityDescriptor = &data->sd; + *securityAttributes = &data->securityAttributes; + return 0; +} + +static int +clearSecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + if (data->acl) { + if (LocalFree((void *)data->acl)) { + return GetLastError(); + } + } + return 0; +} + +#endif + /*[clinic input] os.mkdir @@ -5568,6 +5710,12 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ { int result; +#ifdef MS_WINDOWS + int error = 0; + int pathError = 0; + SECURITY_ATTRIBUTES *pSecAttr = NULL; + struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; +#endif #ifdef HAVE_MKDIRAT int mkdirat_unavailable = 0; #endif @@ -5579,11 +5727,30 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - result = CreateDirectoryW(path->wide, NULL); + switch (mode) { + case 0x1C0: // 0o700 + error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); + break; + default: + error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); + break; + } + if (!error) { + result = CreateDirectoryW(path->wide, pSecAttr); + error = clearSecurityAttributes(&pSecAttr, &secAttrData); + } else { + // Ignore error from "clear" - we have a more interesting one already + clearSecurityAttributes(&pSecAttr, &secAttrData); + } Py_END_ALLOW_THREADS - if (!result) + if (error) { + PyErr_SetFromWindowsErr(error); + return NULL; + } + if (!result) { return path_error(path); + } #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT @@ -17904,6 +18071,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index f58735aff99799..2240e2078b2d98 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -344,6 +344,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index f04f96bc2f7601..f67d480f19de00 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -2117,6 +2117,7 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/readline.c b/Modules/readline.c index c5c34535de0154..35655c70a4618f 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1552,6 +1552,9 @@ PyInit_readline(void) if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION) < 0) { diff --git a/Modules/resource.c b/Modules/resource.c index 8ee07bd0c8054c..3fe18e7c98e3d8 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -513,6 +513,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 6ea141ab1f9189..3eaee22c652c28 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2802,6 +2802,7 @@ _select_exec(PyObject *m) static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 345a6c215eb167..34a427a39b5cf8 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -371,6 +371,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 60be4228a00a03..7d6a1e40243f9d 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -866,6 +866,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha3module.c b/Modules/sha3module.c index c30e924a7072f7..084332c1efa0e0 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -602,6 +602,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 08fedeacd96d28..7de5ebe0899b35 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1698,6 +1698,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7720d59e46590e..daec560ddfcac7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8896,6 +8896,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index ddc9ac3324356d..b4dbb54c3b47b0 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -110,6 +110,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index cb3f2b03990cb8..14e7ca591a076b 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -439,6 +439,7 @@ syslog_exec(PyObject *module) static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index a29474d650127f..f2c5a4bafa7012 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1364,6 +1364,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3211c7530da0b2..0511339978897a 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -2128,6 +2128,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 6ae35b9372b830..333ffe68a454e4 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1668,6 +1668,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 3357b8076b67b1..d86741e1dfc18c 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,10 +62,10 @@ pass */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -395,6 +395,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 52690d9d10a81f..1063e54217b746 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -297,6 +297,10 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, +#ifdef Py_GIL_DISABLED + // These definitions are in the limited API, but not until 3.13. + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 1e4e0ea3743ce3..a46bf8f0e64ee2 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -384,6 +384,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 560f43e5b3a643..9c548f44558d41 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -288,6 +288,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index fe9a6d8d4150ab..b115f67f228ba7 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2106,6 +2106,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/classobject.c b/Objects/classobject.c index 9cbb9442c6059c..69a7d5f046e30d 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -188,15 +188,18 @@ method_getattro(PyObject *obj, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); } if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, obj, (PyObject *)Py_TYPE(obj)); + if (f != NULL) { + PyObject *res = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } @@ -410,14 +413,17 @@ instancemethod_getattro(PyObject *self, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, self, (PyObject *)Py_TYPE(self)); + if (f != NULL) { + PyObject *res = f(descr, self, (PyObject *)Py_TYPE(self)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 2bb0a98a2ed64c..0ba4ff48bc8804 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -9,16 +9,16 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywordsWithVararg() PyDoc_STRVAR(typevar_new__doc__, -"typevar(name, *constraints, bound=None, covariant=False,\n" -" contravariant=False, infer_variance=False)\n" +"typevar(name, *constraints, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a TypeVar."); static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance); + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance); static PyObject * typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -26,14 +26,14 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -42,20 +42,21 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevar", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[7]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *constraints = NULL; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -80,7 +81,13 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[3]) { - covariant = PyObject_IsTrue(fastargs[3]); + default_value = fastargs[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + covariant = PyObject_IsTrue(fastargs[4]); if (covariant < 0) { goto exit; } @@ -88,8 +95,8 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[4]) { - contravariant = PyObject_IsTrue(fastargs[4]); + if (fastargs[5]) { + contravariant = PyObject_IsTrue(fastargs[5]); if (contravariant < 0) { goto exit; } @@ -97,12 +104,12 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[5]); + infer_variance = PyObject_IsTrue(fastargs[6]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = typevar_new_impl(type, name, constraints, bound, covariant, contravariant, infer_variance); + return_value = typevar_new_impl(type, name, constraints, bound, default_value, covariant, contravariant, infer_variance); exit: Py_XDECREF(constraints); @@ -117,6 +124,36 @@ PyDoc_STRVAR(typevar_typing_subst__doc__, #define TYPEVAR_TYPING_SUBST_METHODDEF \ {"__typing_subst__", (PyCFunction)typevar_typing_subst, METH_O, typevar_typing_subst__doc__}, +PyDoc_STRVAR(typevar_typing_prepare_subst__doc__, +"__typing_prepare_subst__($self, alias, args, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF \ + {"__typing_prepare_subst__", _PyCFunction_CAST(typevar_typing_prepare_subst), METH_FASTCALL, typevar_typing_prepare_subst__doc__}, + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args); + +static PyObject * +typevar_typing_prepare_subst(typevarobject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *alias; + PyObject *__clinic_args; + + if (!_PyArg_CheckPositional("__typing_prepare_subst__", nargs, 2, 2)) { + goto exit; + } + alias = args[0]; + __clinic_args = args[1]; + return_value = typevar_typing_prepare_subst_impl(self, alias, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(typevar_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -134,6 +171,23 @@ typevar_reduce(typevarobject *self, PyObject *Py_UNUSED(ignored)) return typevar_reduce_impl(self); } +PyDoc_STRVAR(typevar_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevar_has_default, METH_NOARGS, typevar_has_default__doc__}, + +static PyObject * +typevar_has_default_impl(typevarobject *self); + +static PyObject * +typevar_has_default(typevarobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevar_has_default_impl(self); +} + PyDoc_STRVAR(paramspecargs_new__doc__, "paramspecargs(origin)\n" "--\n" @@ -243,15 +297,16 @@ paramspeckwargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(paramspec_new__doc__, -"paramspec(name, *, bound=None, covariant=False, contravariant=False,\n" -" infer_variance=False)\n" +"paramspec(name, *, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a ParamSpec object."); static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance); + PyObject *default_value, int covariant, int contravariant, + int infer_variance); static PyObject * paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -259,14 +314,14 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -275,19 +330,20 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "paramspec", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -311,7 +367,13 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - covariant = PyObject_IsTrue(fastargs[2]); + default_value = fastargs[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + covariant = PyObject_IsTrue(fastargs[3]); if (covariant < 0) { goto exit; } @@ -319,8 +381,8 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[3]) { - contravariant = PyObject_IsTrue(fastargs[3]); + if (fastargs[4]) { + contravariant = PyObject_IsTrue(fastargs[4]); if (contravariant < 0) { goto exit; } @@ -328,12 +390,12 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[4]); + infer_variance = PyObject_IsTrue(fastargs[5]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = paramspec_new_impl(type, name, bound, covariant, contravariant, infer_variance); + return_value = paramspec_new_impl(type, name, bound, default_value, covariant, contravariant, infer_variance); exit: return return_value; @@ -394,14 +456,32 @@ paramspec_reduce(paramspecobject *self, PyObject *Py_UNUSED(ignored)) return paramspec_reduce_impl(self); } +PyDoc_STRVAR(paramspec_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define PARAMSPEC_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)paramspec_has_default, METH_NOARGS, paramspec_has_default__doc__}, + +static PyObject * +paramspec_has_default_impl(paramspecobject *self); + +static PyObject * +paramspec_has_default(paramspecobject *self, PyObject *Py_UNUSED(ignored)) +{ + return paramspec_has_default_impl(self); +} + PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name)\n" +"typevartuple(name, *, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name); +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value); static PyObject * typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -409,14 +489,14 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, + .ob_item = { &_Py_ID(name), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -425,17 +505,19 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", NULL}; + static const char * const _keywords[] = {"name", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *default_value = &_Py_NoDefaultStruct; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -446,7 +528,12 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - return_value = typevartuple_impl(type, name); + if (!noptargs) { + goto skip_optional_kwonly; + } + default_value = fastargs[1]; +skip_optional_kwonly: + return_value = typevartuple_impl(type, name, default_value); exit: return return_value; @@ -507,6 +594,23 @@ typevartuple_reduce(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) return typevartuple_reduce_impl(self); } +PyDoc_STRVAR(typevartuple_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVARTUPLE_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevartuple_has_default, METH_NOARGS, typevartuple_has_default__doc__}, + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self); + +static PyObject * +typevartuple_has_default(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevartuple_has_default_impl(self); +} + PyDoc_STRVAR(typealias_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -591,4 +695,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5a582d9d89ad787b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73b39e550e4e336c input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 014632962bfcf3..3d804f73a54088 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -5,6 +5,8 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs #include "pycore_object.h" // _PyObject_SetDeferredRefcount #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches @@ -100,10 +102,20 @@ PyCode_ClearWatcher(int watcher_id) * generic helpers ******************/ -/* all_name_chars(s): true iff s matches [a-zA-Z0-9_]* */ static int -all_name_chars(PyObject *o) +should_intern_string(PyObject *o) { +#ifdef Py_GIL_DISABLED + // The free-threaded build interns (and immortalizes) all string constants + // unless we've disabled immortalizing objects that use deferred reference + // counting. + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->gc.immortalize.enable_on_thread_created) { + return 1; + } +#endif + + // compute if s matches [a-zA-Z0-9_] const unsigned char *s, *e; if (!PyUnicode_IS_ASCII(o)) @@ -118,6 +130,10 @@ all_name_chars(PyObject *o) return 1; } +#ifdef Py_GIL_DISABLED +static PyObject *intern_one_constant(PyObject *op); +#endif + static int intern_strings(PyObject *tuple) { @@ -135,14 +151,16 @@ intern_strings(PyObject *tuple) return 0; } -/* Intern selected string constants */ +/* Intern constants. In the default build, this interns selected string + constants. In the free-threaded build, this also interns non-string + constants. */ static int -intern_string_constants(PyObject *tuple, int *modified) +intern_constants(PyObject *tuple, int *modified) { for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { PyObject *v = PyTuple_GET_ITEM(tuple, i); if (PyUnicode_CheckExact(v)) { - if (all_name_chars(v)) { + if (should_intern_string(v)) { PyObject *w = v; PyUnicode_InternInPlace(&v); if (w != v) { @@ -154,7 +172,7 @@ intern_string_constants(PyObject *tuple, int *modified) } } else if (PyTuple_CheckExact(v)) { - if (intern_string_constants(v, NULL) < 0) { + if (intern_constants(v, NULL) < 0) { return -1; } } @@ -165,7 +183,7 @@ intern_string_constants(PyObject *tuple, int *modified) return -1; } int tmp_modified = 0; - if (intern_string_constants(tmp, &tmp_modified) < 0) { + if (intern_constants(tmp, &tmp_modified) < 0) { Py_DECREF(tmp); return -1; } @@ -184,6 +202,59 @@ intern_string_constants(PyObject *tuple, int *modified) } Py_DECREF(tmp); } +#ifdef Py_GIL_DISABLED + else if (PySlice_Check(v)) { + PySliceObject *slice = (PySliceObject *)v; + PyObject *tmp = PyTuple_New(3); + if (tmp == NULL) { + return -1; + } + PyTuple_SET_ITEM(tmp, 0, Py_NewRef(slice->start)); + PyTuple_SET_ITEM(tmp, 1, Py_NewRef(slice->stop)); + PyTuple_SET_ITEM(tmp, 2, Py_NewRef(slice->step)); + int tmp_modified = 0; + if (intern_constants(tmp, &tmp_modified) < 0) { + Py_DECREF(tmp); + return -1; + } + if (tmp_modified) { + v = PySlice_New(PyTuple_GET_ITEM(tmp, 0), + PyTuple_GET_ITEM(tmp, 1), + PyTuple_GET_ITEM(tmp, 2)); + if (v == NULL) { + Py_DECREF(tmp); + return -1; + } + PyTuple_SET_ITEM(tuple, i, v); + Py_DECREF(slice); + if (modified) { + *modified = 1; + } + } + Py_DECREF(tmp); + } + + // Intern non-string consants in the free-threaded build, but only if + // we are also immortalizing objects that use deferred reference + // counting. + PyThreadState *tstate = PyThreadState_GET(); + if (!_Py_IsImmortal(v) && !PyCode_Check(v) && + !PyUnicode_CheckExact(v) && + tstate->interp->gc.immortalize.enable_on_thread_created) + { + PyObject *interned = intern_one_constant(v); + if (interned == NULL) { + return -1; + } + else if (interned != v) { + PyTuple_SET_ITEM(tuple, i, interned); + Py_SETREF(v, interned); + if (modified) { + *modified = 1; + } + } + } +#endif } return 0; } @@ -390,6 +461,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); co->co_qualname = Py_NewRef(con->qualname); + PyUnicode_InternInPlace(&co->co_filename); + PyUnicode_InternInPlace(&co->co_name); + PyUnicode_InternInPlace(&co->co_qualname); co->co_flags = con->flags; co->co_firstlineno = con->firstlineno; @@ -416,10 +490,16 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->func_state.mutex); +#endif co->co_version = interp->func_state.next_version; if (interp->func_state.next_version != 0) { interp->func_state.next_version++; } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->func_state.mutex); +#endif co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ @@ -531,18 +611,41 @@ remove_column_info(PyObject *locations) return res; } -/* The caller is responsible for ensuring that the given data is valid. */ - -PyCodeObject * -_PyCode_New(struct _PyCodeConstructor *con) +static int +intern_code_constants(struct _PyCodeConstructor *con) { +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _py_code_state *state = &interp->code_state; + PyMutex_Lock(&state->mutex); +#endif if (intern_strings(con->names) < 0) { - return NULL; + goto error; } - if (intern_string_constants(con->consts, NULL) < 0) { - return NULL; + if (intern_constants(con->consts, NULL) < 0) { + goto error; } if (intern_strings(con->localsplusnames) < 0) { + goto error; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return 0; + +error: +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return -1; +} + +/* The caller is responsible for ensuring that the given data is valid. */ + +PyCodeObject * +_PyCode_New(struct _PyCodeConstructor *con) +{ + if (intern_code_constants(con) < 0) { return NULL; } @@ -1496,13 +1599,16 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } +#ifdef _Py_TIER2 + static void clear_executors(PyCodeObject *co) { assert(co->co_executors); for (int i = 0; i < co->co_executors->size; i++) { if (co->co_executors->executors[i]) { - _Py_ExecutorClear(co->co_executors->executors[i]); + _Py_ExecutorDetach(co->co_executors->executors[i]); + assert(co->co_executors->executors[i] == NULL); } } PyMem_Free(co->co_executors); @@ -1515,6 +1621,8 @@ _PyCode_Clear_Executors(PyCodeObject *code) clear_executors(code); } +#endif + static void deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { @@ -1739,9 +1847,11 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } +#ifdef _Py_TIER2 if (co->co_executors != NULL) { clear_executors(co); } +#endif Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); @@ -2381,3 +2491,183 @@ _PyCode_ConstantKey(PyObject *op) } return key; } + +#ifdef Py_GIL_DISABLED +static PyObject * +intern_one_constant(PyObject *op) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_hashtable_t *consts = interp->code_state.constants; + + assert(!PyUnicode_CheckExact(op)); // strings are interned separately + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op); + if (entry == NULL) { + if (_Py_hashtable_set(consts, op, op) != 0) { + return NULL; + } + +#ifdef Py_REF_DEBUG + Py_ssize_t refcnt = Py_REFCNT(op); + if (refcnt != 1) { + // Adjust the reftotal to account for the fact that we only + // restore a single reference in _PyCode_Fini. + _Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1)); + } +#endif + + _Py_SetImmortal(op); + return op; + } + + assert(_Py_IsImmortal(entry->value)); + return (PyObject *)entry->value; +} + +static int +compare_constants(const void *key1, const void *key2) { + PyObject *op1 = (PyObject *)key1; + PyObject *op2 = (PyObject *)key2; + if (op1 == op2) { + return 1; + } + if (Py_TYPE(op1) != Py_TYPE(op2)) { + return 0; + } + // We compare container contents by identity because we have already + // internalized the items. + if (PyTuple_CheckExact(op1)) { + Py_ssize_t size = PyTuple_GET_SIZE(op1); + if (size != PyTuple_GET_SIZE(op2)) { + return 0; + } + for (Py_ssize_t i = 0; i < size; i++) { + if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) { + return 0; + } + } + return 1; + } + else if (PyFrozenSet_CheckExact(op1)) { + if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) { + return 0; + } + Py_ssize_t pos1 = 0, pos2 = 0; + PyObject *obj1, *obj2; + Py_hash_t hash1, hash2; + while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) && + (_PySet_NextEntry(op2, &pos2, &obj2, &hash2))) + { + if (obj1 != obj2) { + return 0; + } + } + return 1; + } + else if (PySlice_Check(op1)) { + PySliceObject *s1 = (PySliceObject *)op1; + PySliceObject *s2 = (PySliceObject *)op2; + return (s1->start == s2->start && + s1->stop == s2->stop && + s1->step == s2->step); + } + else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) { + return PyObject_RichCompareBool(op1, op2, Py_EQ); + } + else if (PyFloat_CheckExact(op1)) { + // Ensure that, for example, +0.0 and -0.0 are distinct + double f1 = PyFloat_AS_DOUBLE(op1); + double f2 = PyFloat_AS_DOUBLE(op2); + return memcmp(&f1, &f2, sizeof(double)) == 0; + } + else if (PyComplex_CheckExact(op1)) { + Py_complex c1 = ((PyComplexObject *)op1)->cval; + Py_complex c2 = ((PyComplexObject *)op2)->cval; + return memcmp(&c1, &c2, sizeof(Py_complex)) == 0; + } + _Py_FatalErrorFormat("unexpected type in compare_constants: %s", + Py_TYPE(op1)->tp_name); + return 0; +} + +static Py_uhash_t +hash_const(const void *key) +{ + PyObject *op = (PyObject *)key; + if (PySlice_Check(op)) { + PySliceObject *s = (PySliceObject *)op; + PyObject *data[3] = { s->start, s->stop, s->step }; + return _Py_HashBytes(&data, sizeof(data)); + } + else if (PyTuple_CheckExact(op)) { + Py_ssize_t size = PyTuple_GET_SIZE(op); + PyObject **data = _PyTuple_ITEMS(op); + return _Py_HashBytes(data, sizeof(PyObject *) * size); + } + Py_hash_t h = PyObject_Hash(op); + if (h == -1) { + // This should never happen: all the constants we support have + // infallible hash functions. + Py_FatalError("code: hash failed"); + } + return (Py_uhash_t)h; +} + +static int +clear_containers(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + // First clear containers to avoid recursive deallocation later on in + // destroy_key. + PyObject *op = (PyObject *)key; + if (PyTuple_CheckExact(op)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) { + Py_CLEAR(_PyTuple_ITEMS(op)[i]); + } + } + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + Py_SETREF(slice->start, Py_None); + Py_SETREF(slice->stop, Py_None); + Py_SETREF(slice->step, Py_None); + } + else if (PyFrozenSet_CheckExact(op)) { + _PySet_ClearInternal((PySetObject *)op); + } + return 0; +} + +static void +destroy_key(void *key) +{ + _Py_ClearImmortal(key); +} +#endif + +PyStatus +_PyCode_Init(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + state->constants = _Py_hashtable_new_full(&hash_const, &compare_constants, + &destroy_key, NULL, NULL); + if (state->constants == NULL) { + return _PyStatus_NO_MEMORY(); + } +#endif + return _PyStatus_OK(); +} + +void +_PyCode_Fini(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + // Free interned constants + struct _py_code_state *state = &interp->code_state; + if (state->constants) { + _Py_hashtable_foreach(state->constants, &clear_containers, NULL); + _Py_hashtable_destroy(state->constants); + state->constants = NULL; + } +#endif +} diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 1f21f70f149c80..b0fce09d7940e0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -924,16 +924,15 @@ new_dict(PyInterpreterState *interp, return (PyObject *)mp; } -/* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } + dictkeys_incref(keys); for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -2268,15 +2267,13 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) * exception occurred. */ int -_PyDict_GetItemRef_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +_PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result) { - PyDictObject*mp = (PyDictObject *)op; - PyObject *value; #ifdef Py_GIL_DISABLED - Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup_threadsafe(op, key, hash, &value); #else - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); #endif assert(ix >= 0 || value == NULL); if (ix == DKIX_ERROR) { @@ -2314,9 +2311,38 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) } } - return _PyDict_GetItemRef_KnownHash(op, key, hash, result); + return _PyDict_GetItemRef_KnownHash((PyDictObject *)op, key, hash, result); } +int +_PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result) +{ + ASSERT_DICT_LOCKED(op); + assert(PyUnicode_CheckExact(key)); + + Py_hash_t hash; + if ((hash = unicode_get_hash(key)) == -1) { + hash = PyObject_Hash(key); + if (hash == -1) { + *result = NULL; + return -1; + } + } + + PyObject *value; + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } + *result = Py_NewRef(value); + return 1; // key is present +} /* Variant of PyDict_GetItem() that doesn't suppress exceptions. This returns NULL *with* an exception set if an exception occurred. @@ -6666,8 +6692,6 @@ materialize_managed_dict_lock_held(PyObject *obj) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); - OBJECT_STAT_INC(dict_materialized_on_request); - PyDictValues *values = _PyObject_InlineValues(obj); PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); @@ -6704,8 +6728,8 @@ _PyObject_MaterializeManagedDict(PyObject *obj) return dict; } -static int -set_or_del_lock_held(PyDictObject *dict, PyObject *name, PyObject *value) +int +_PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) { if (value == NULL) { Py_hash_t hash; @@ -6767,7 +6791,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, // so that no one else will see it. dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); if (dict == NULL || - set_or_del_lock_held(dict, name, value) < 0) { + _PyDict_SetItem_LockHeld(dict, name, value) < 0) { Py_XDECREF(dict); return -1; } @@ -6779,7 +6803,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); - res = set_or_del_lock_held (dict, name, value); + res = _PyDict_SetItem_LockHeld(dict, name, value); return res; } @@ -6822,7 +6846,7 @@ store_instance_attr_dict(PyObject *obj, PyDictObject *dict, PyObject *name, PyOb res = store_instance_attr_lock_held(obj, values, name, value); } else { - res = set_or_del_lock_held(dict, name, value); + res = _PyDict_SetItem_LockHeld(dict, name, value); } Py_END_CRITICAL_SECTION(); return res; @@ -7159,35 +7183,77 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) return 0; } -PyObject * -PyObject_GenericGetDict(PyObject *obj, void *context) +static inline PyObject * +ensure_managed_dict(PyObject *obj) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - PyDictObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - dict = _PyObject_GetManagedDict(obj); - if (dict == NULL && - (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(obj)->valid)) { dict = _PyObject_MaterializeManagedDict(obj); } - else if (dict == NULL) { - Py_BEGIN_CRITICAL_SECTION(obj); - + else { +#ifdef Py_GIL_DISABLED // Check again that we're not racing with someone else creating the dict + Py_BEGIN_CRITICAL_SECTION(obj); dict = _PyObject_GetManagedDict(obj); - if (dict == NULL) { - OBJECT_STAT_INC(dict_materialized_on_request); - dictkeys_incref(CACHED_KEYS(tp)); - dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, - (PyDictObject *)dict); + if (dict != NULL) { + goto done; } +#endif + dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), + CACHED_KEYS(tp)); + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); +#ifdef Py_GIL_DISABLED +done: Py_END_CRITICAL_SECTION(); +#endif } - return Py_XNewRef((PyObject *)dict); + } + return (PyObject *)dict; +} + +static inline PyObject * +ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) +{ + PyDictKeysObject *cached; + + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + dict = *dictptr; + if (dict != NULL) { + goto done; + } +#endif + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); + dict = new_dict_with_shared_keys(interp, cached); + } + else { + dict = PyDict_New(); + } + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, dict); +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif + } + return dict; +} + +PyObject * +PyObject_GenericGetDict(PyObject *obj, void *context) +{ + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { + return Py_XNewRef(ensure_managed_dict(obj)); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -7196,64 +7262,28 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - PyObject *dict = *dictptr; - if (dict == NULL) { - PyTypeObject *tp = Py_TYPE(obj); - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys( - interp, CACHED_KEYS(tp)); - } - else { - *dictptr = dict = PyDict_New(); - } - } - return Py_XNewRef(dict); + + return Py_XNewRef(ensure_nonmanaged_dict(obj, dictptr)); } } int -_PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, +_PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *key, PyObject *value) { PyObject *dict; int res; - PyDictKeysObject *cached; - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); - if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - assert(dictptr != NULL); - dict = *dictptr; - if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); - dictkeys_incref(cached); - dict = new_dict_with_shared_keys(interp, cached); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } - else { - res = PyDict_SetItem(dict, key, value); - } - } else { - dict = *dictptr; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } else { - res = PyDict_SetItem(dict, key, value); - } + dict = ensure_nonmanaged_dict(obj, dictptr); + if (dict == NULL) { + return -1; } + + Py_BEGIN_CRITICAL_SECTION(dict); + res = _PyDict_SetItem_LockHeld((PyDictObject *)dict, key, value); ASSERT_CONSISTENT(dict); + Py_END_CRITICAL_SECTION(); return res; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 36538b1f6d53fe..d7fcb1925d286c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -16,12 +16,720 @@ #define OFF(x) offsetof(PyFrameObject, x) + +// Returns borrowed reference or NULL +static PyObject * +framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) +{ + PyObject **fast = _PyFrame_GetLocalsArray(frame); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *value = fast[i]; + PyObject *cell = NULL; + + if (value == NULL) { + return NULL; + } + + if (kind == CO_FAST_FREE || kind & CO_FAST_CELL) { + // The cell was set when the frame was created from + // the function's closure. + assert(PyCell_Check(value)); + cell = value; + } + + if (cell != NULL) { + value = PyCell_GET(cell); + } + + if (value == NULL) { + return NULL; + } + + return value; +} + +static int +framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) +{ + /* + * Returns the fast locals index of the key + * - if read == true, returns the index if the value is not NULL + * - if read == false, returns the index if the value is not hidden + */ + + assert(PyUnicode_CheckExact(key)); + + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + int found_key = false; + + // We do 2 loops here because it's highly possible the key is interned + // and we can do a pointer comparison. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (name == key) { + found_key = true; + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + + if (!found_key) { + // This is unlikely, but we need to make sure. This means the key + // is not interned. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (_PyUnicode_EQ(name, key)) { + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + } + + return -1; +} + +static PyObject * +framelocalsproxy_getitem(PyObject *self, PyObject *key) +{ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + assert(value != NULL); + return Py_NewRef(value); + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + if (extra != NULL) { + PyObject *value = PyDict_GetItem(extra, key); + if (value != NULL) { + return Py_NewRef(value); + } + } + + PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); + return NULL; +} + +static int +framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + /* Merge locals into fast locals */ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); + return -1; + } + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i >= 0) { + _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); + + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + PyObject *oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue != NULL && PyCell_Check(oldvalue)); + cell = oldvalue; + } else if (kind & CO_FAST_CELL && oldvalue != NULL) { + if (PyCell_Check(oldvalue)) { + cell = oldvalue; + } + } + if (cell != NULL) { + oldvalue = PyCell_GET(cell); + if (value != oldvalue) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue); + } + } else if (value != oldvalue) { + Py_XSETREF(fast[i], Py_NewRef(value)); + } + return 0; + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + + if (extra == NULL) { + extra = PyDict_New(); + if (extra == NULL) { + return -1; + } + frame->f_extra_locals = extra; + } + + assert(PyDict_Check(extra)); + + return PyDict_SetItem(extra, key, value); +} + +static int +framelocalsproxy_merge(PyObject* self, PyObject* other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + return -1; + } + + PyObject *keys = PyMapping_Keys(other); + if (keys == NULL) { + return -1; + } + + PyObject *iter = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iter == NULL) { + return -1; + } + + PyObject *key = NULL; + PyObject *value = NULL; + + while ((key = PyIter_Next(iter)) != NULL) { + value = PyObject_GetItem(other, key); + if (value == NULL) { + Py_DECREF(key); + Py_DECREF(iter); + return -1; + } + + if (framelocalsproxy_setitem(self, key, value) < 0) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); + return -1; + } + + Py_DECREF(key); + Py_DECREF(value); + } + + Py_DECREF(iter); + + return 0; +} + +static PyObject * +framelocalsproxy_keys(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *names = PyList_New(0); + if (names == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *val = framelocalsproxy_getval(frame->f_frame, co, i); + if (val) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (PyList_Append(names, name) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + assert(PyDict_Check(frame->f_extra_locals)); + + Py_ssize_t i = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { + if (PyList_Append(names, key) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + return names; +} + +static void +framelocalsproxy_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + PyFrameObject *frame = (PyFrameObject*)PyTuple_GET_ITEM(args, 0); + assert(PyFrame_Check(frame)); + + ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); + + return (PyObject *)self; +} + +static int +framelocalsproxy_tp_clear(PyObject *self) +{ + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static int +framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static PyObject * +framelocalsproxy_iter(PyObject *self) +{ + PyObject* keys = framelocalsproxy_keys(self, NULL); + if (keys == NULL) { + return NULL; + } + + PyObject* iter = PyObject_GetIter(keys); + Py_XDECREF(keys); + + return iter; +} + +static PyObject * +framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +{ + if (PyFrameLocalsProxy_Check(other)) { + bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; + if (op == Py_EQ) { + return PyBool_FromLong(result); + } else if (op == Py_NE) { + return PyBool_FromLong(!result); + } + } else if (PyDict_Check(other)) { + PyObject *dct = PyDict_New(); + if (dct == NULL) { + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + return NULL; + } + + PyObject *result = PyObject_RichCompare(dct, other, op); + Py_DECREF(dct); + return result; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +static PyObject * +framelocalsproxy_repr(PyObject *self) +{ + int i = Py_ReprEnter(self); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("{...}") : NULL; + } + + PyObject *dct = PyDict_New(); + if (dct == NULL) { + Py_ReprLeave(self); + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + Py_ReprLeave(self); + return NULL; + } + + PyObject *repr = PyObject_Repr(dct); + Py_DECREF(dct); + + Py_ReprLeave(self); + + return repr; +} + +static PyObject* +framelocalsproxy_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + if (PyDict_Update(result, other) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_inplace_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (framelocalsproxy_merge(self, other) < 0) { + Py_RETURN_NOTIMPLEMENTED; + } + + return Py_NewRef(self); +} + +static PyObject* +framelocalsproxy_values(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *values = PyList_New(0); + if (values == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + if (value) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + return values; +} + +static PyObject * +framelocalsproxy_items(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *items = PyList_New(0); + if (items == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + + if (value) { + PyObject *pair = PyTuple_Pack(2, name, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyObject *pair = PyTuple_Pack(2, key, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + return items; +} + +static Py_ssize_t +framelocalsproxy_length(PyObject *self) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + Py_ssize_t size = 0; + + if (frame->f_extra_locals != NULL) { + assert(PyDict_Check(frame->f_extra_locals)); + size += PyDict_Size(frame->f_extra_locals); + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + size++; + } + } + return size; +} + +static int +framelocalsproxy_contains(PyObject *self, PyObject *key) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + return 1; + } + } + + PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + if (extra != NULL) { + return PyDict_Contains(extra, key); + } + + return 0; +} + +static PyObject* framelocalsproxy___contains__(PyObject *self, PyObject *key) +{ + int result = framelocalsproxy_contains(self, key); + if (result < 0) { + return NULL; + } + return PyBool_FromLong(result); +} + +static PyObject* +framelocalsproxy_update(PyObject *self, PyObject *other) +{ + if (framelocalsproxy_merge(self, other) < 0) { + PyErr_SetString(PyExc_TypeError, "update() argument must be dict or another FrameLocalsProxy"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "get expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "setdefault expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + if (framelocalsproxy_setitem(self, key, default_value) < 0) { + return NULL; + } + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_reversed(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject *result = framelocalsproxy_keys(self, NULL); + + if (result == NULL) { + return NULL; + } + + if (PyList_Reverse(result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + +static PyNumberMethods framelocalsproxy_as_number = { + .nb_or = framelocalsproxy_or, + .nb_inplace_or = framelocalsproxy_inplace_or, +}; + +static PySequenceMethods framelocalsproxy_as_sequence = { + .sq_contains = framelocalsproxy_contains, +}; + +static PyMappingMethods framelocalsproxy_as_mapping = { + framelocalsproxy_length, // mp_length + framelocalsproxy_getitem, // mp_subscript + framelocalsproxy_setitem, // mp_ass_subscript +}; + +static PyMethodDef framelocalsproxy_methods[] = { + {"__contains__", framelocalsproxy___contains__, METH_O | METH_COEXIST, + NULL}, + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, + NULL}, + {"update", framelocalsproxy_update, METH_O, + NULL}, + {"__reversed__", _PyCFunction_CAST(framelocalsproxy_reversed), METH_NOARGS, + NULL}, + {"keys", _PyCFunction_CAST(framelocalsproxy_keys), METH_NOARGS, + NULL}, + {"values", _PyCFunction_CAST(framelocalsproxy_values), METH_NOARGS, + NULL}, + {"items", _PyCFunction_CAST(framelocalsproxy_items), METH_NOARGS, + NULL}, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, + NULL}, + {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, + NULL}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyFrameLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "FrameLocalsProxy", + .tp_basicsize = sizeof(PyFrameLocalsProxyObject), + .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_repr = &framelocalsproxy_repr, + .tp_as_number = &framelocalsproxy_as_number, + .tp_as_sequence = &framelocalsproxy_as_sequence, + .tp_as_mapping = &framelocalsproxy_as_mapping, + .tp_getattro = PyObject_GenericGetAttr, + .tp_setattro = PyObject_GenericSetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = framelocalsproxy_visit, + .tp_clear = framelocalsproxy_tp_clear, + .tp_richcompare = framelocalsproxy_richcompare, + .tp_iter = framelocalsproxy_iter, + .tp_methods = framelocalsproxy_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = framelocalsproxy_new, + .tp_free = PyObject_GC_Del, +}; + +PyObject * +_PyFrameLocalsProxy_New(PyFrameObject *frame) +{ + PyObject* args = PyTuple_Pack(1, frame); + if (args == NULL) { + return NULL; + } + + PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); + Py_DECREF(args); + return proxy; +} + static PyMemberDef frame_memberlist[] = { {"f_trace_lines", Py_T_BOOL, OFF(f_trace_lines), 0}, {NULL} /* Sentinel */ }; - static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { @@ -30,11 +738,23 @@ frame_getlocals(PyFrameObject *f, void *closure) return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); - PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); - if (locals) { - f->f_fast_as_locals = 1; + + PyCodeObject *co = _PyFrame_GetCode(f->f_frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f->f_frame)) { + if (f->f_frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + f->f_frame->f_locals = PyDict_New(); + if (f->f_frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(f->f_frame->f_locals); } - return locals; + + return _PyFrameLocalsProxy_New(f); } int @@ -595,20 +1315,6 @@ first_line_not_before(int *lines, int len, int line) return result; } -static bool -frame_is_cleared(PyFrameObject *frame) -{ - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - if (frame->f_frame->stacktop == 0) { - return true; - } - if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state == FRAME_CLEARED; - } - return false; -} - static bool frame_is_suspended(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); @@ -900,6 +1606,7 @@ frame_dealloc(PyFrameObject *f) } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); PyObject_GC_Del(f); Py_XDECREF(co); Py_TRASHCAN_END; @@ -910,6 +1617,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_extra_locals); if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } @@ -921,6 +1629,7 @@ static int frame_tp_clear(PyFrameObject *f) { Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); /* locals and stack */ PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); @@ -1056,8 +1765,8 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace = NULL; f->f_trace_lines = 1; f->f_trace_opcodes = 0; - f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_extra_locals = NULL; return f; } @@ -1204,103 +1913,54 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) { - /* Merge fast locals into f->f_locals */ - PyObject *locals = frame->f_locals; - if (locals == NULL) { - locals = frame->f_locals = PyDict_New(); - if (locals == NULL) { - return NULL; - } - } - PyObject *hidden = NULL; - - /* If include_hidden, "hidden" fast locals (from inlined comprehensions in - module/class scopes) will be included in the returned dict, but not in - frame->f_locals; the returned dict will be a modified copy. Non-hidden - locals will still be updated in frame->f_locals. */ - if (include_hidden) { - hidden = PyDict_New(); - if (hidden == NULL) { - return NULL; - } - } - - frame_init_get_vars(frame); + /* + * This function returns if there are hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ + PyCodeObject* co = _PyFrame_GetCode(frame); - PyCodeObject *co = _PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value; // borrowed reference - if (!frame_get_var(frame, co, i, &value)) { - continue; - } - - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (kind & CO_FAST_HIDDEN) { - if (include_hidden && value != NULL) { - if (PyObject_SetItem(hidden, name, value) != 0) { - goto error; - } - } - continue; - } - if (value == NULL) { - if (PyObject_DelItem(locals, name) != 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - else { - goto error; - } - } - } - else { - if (PyObject_SetItem(locals, name, value) != 0) { - goto error; + PyObject* value = framelocalsproxy_getval(frame, co, i); + + if (value != NULL) { + return true; } } } - if (include_hidden && PyDict_Size(hidden)) { - PyObject *innerlocals = PyDict_New(); - if (innerlocals == NULL) { - goto error; - } - if (PyDict_Merge(innerlocals, locals, 1) != 0) { - Py_DECREF(innerlocals); - goto error; - } - if (PyDict_Merge(innerlocals, hidden, 1) != 0) { - Py_DECREF(innerlocals); - goto error; - } - locals = innerlocals; - } - else { - Py_INCREF(locals); - } - Py_CLEAR(hidden); - - return locals; - - error: - Py_XDECREF(hidden); - return NULL; + return false; } -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame) { - PyObject *locals = _PyFrame_GetLocals(frame, 0); - if (locals == NULL) { - return -1; + // We should try to avoid creating the FrameObject if possible. + // So we check if the frame is a module or class level scope + PyCodeObject *co = _PyFrame_GetCode(frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { + if (frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + frame->f_locals = PyDict_New(); + if (frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(frame->f_locals); } - Py_DECREF(locals); - return 0; + + PyFrameObject* f = _PyFrame_GetFrameObject(frame); + + return _PyFrameLocalsProxy_New(f); } @@ -1354,112 +2014,19 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - if (f == NULL) { - PyErr_BadInternalCall(); - return -1; - } - assert(!_PyFrame_IsIncomplete(f->f_frame)); - int err = _PyFrame_FastToLocalsWithError(f->f_frame); - if (err == 0) { - f->f_fast_as_locals = 1; - } - return err; + return 0; } void PyFrame_FastToLocals(PyFrameObject *f) { - int res; - assert(!_PyFrame_IsIncomplete(f->f_frame)); - assert(!PyErr_Occurred()); - - res = PyFrame_FastToLocalsWithError(f); - if (res < 0) - PyErr_Clear(); -} - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) -{ - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; - PyCodeObject *co; - locals = frame->f_locals; - if (locals == NULL) { - return; - } - fast = _PyFrame_GetLocalsArray(frame); - co = _PyFrame_GetCode(frame); - - PyObject *exc = PyErr_GetRaisedException(); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) { - continue; - } - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; - } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } - else if (value != oldvalue) { - if (value == NULL) { - // Probably can't delete this, since the compiler's flow - // analysis may have already "proven" that it exists here: - const char *e = "assigning None to unbound local %R"; - if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, name)) { - // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame->frame_obj); - } - value = Py_NewRef(Py_None); - } - Py_XSETREF(fast[i], Py_NewRef(value)); - } - Py_XDECREF(value); - } - PyErr_SetRaisedException(exc); + return; } void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); - if (f && f->f_fast_as_locals && !frame_is_cleared(f)) { - _PyFrame_LocalsToFast(f->f_frame, clear); - f->f_fast_as_locals = 0; - } + return; } int diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 276b3db2970371..8a30213888ef87 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -287,6 +287,7 @@ functions is running. void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { +#ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); if (func->func_version != 0) { struct _func_version_cache_item *slot = @@ -297,7 +298,9 @@ _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) // Leave slot->code alone, there may be use for it. } } +#endif func->func_version = version; +#ifndef Py_GIL_DISABLED if (version != 0) { struct _func_version_cache_item *slot = interp->func_state.func_version_cache @@ -305,11 +308,13 @@ _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) slot->func = func; slot->code = func->func_code; } +#endif } void _PyFunction_ClearCodeByVersion(uint32_t version) { +#ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); struct _func_version_cache_item *slot = interp->func_state.func_version_cache @@ -322,11 +327,15 @@ _PyFunction_ClearCodeByVersion(uint32_t version) slot->func = NULL; } } +#endif } PyFunctionObject * _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) { +#ifdef Py_GIL_DISABLED + return NULL; +#else PyInterpreterState *interp = _PyInterpreterState_GET(); struct _func_version_cache_item *slot = interp->func_state.func_version_cache @@ -346,6 +355,7 @@ _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) return slot->func; } return NULL; +#endif } uint32_t diff --git a/Objects/genobject.c b/Objects/genobject.c index 5d7da49cfca166..92cd8c61e7e9ca 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -11,6 +11,7 @@ #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -329,10 +330,11 @@ gen_close_iter(PyObject *yf) static inline bool is_resume(_Py_CODEUNIT *instr) { + uint8_t code = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); return ( - instr->op.code == RESUME || - instr->op.code == RESUME_CHECK || - instr->op.code == INSTRUMENTED_RESUME + code == RESUME || + code == RESUME_CHECK || + code == INSTRUMENTED_RESUME ); } @@ -380,6 +382,7 @@ gen_close(PyGenObject *gen, PyObject *args) // RESUME after YIELD_VALUE and exception depth is 1 assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); gen->gi_frame_state = FRAME_COMPLETED; + _PyFrame_ClearLocals((_PyInterpreterFrame *)gen->gi_iframe); Py_RETURN_NONE; } } @@ -796,6 +799,7 @@ static PyMethodDef gen_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -1148,6 +1152,7 @@ static PyMethodDef coro_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -1771,6 +1776,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) if (o->ags_state == AWAITABLE_STATE_INIT) { if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, "anext(): asynchronous generator is already running"); @@ -1814,10 +1820,24 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg return NULL; } + if (o->ags_state == AWAITABLE_STATE_INIT) { + if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + + o->ags_state = AWAITABLE_STATE_ITER; + o->ags_gen->ag_running_async = 1; + } + result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { + o->ags_gen->ag_running_async = 0; o->ags_state = AWAITABLE_STATE_CLOSED; } @@ -1828,8 +1848,25 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg static PyObject * async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) { - o->ags_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyObject *result; + if (o->ags_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + result = async_gen_asend_throw(o, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } static void @@ -2206,10 +2243,31 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na return NULL; } + if (o->agt_state == AWAITABLE_STATE_INIT) { + if (o->agt_gen->ag_running_async) { + o->agt_state = AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } + else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + o->agt_state = AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; + } + retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); if (o->agt_args) { retval = async_gen_unwrap_value(o->agt_gen, retval); if (retval == NULL) { + o->agt_gen->ag_running_async = 0; o->agt_state = AWAITABLE_STATE_CLOSED; } return retval; @@ -2223,6 +2281,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na return NULL; } if (retval == NULL) { + o->agt_gen->ag_running_async = 0; o->agt_state = AWAITABLE_STATE_CLOSED; } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || @@ -2251,8 +2310,25 @@ async_gen_athrow_iternext(PyAsyncGenAThrow *o) static PyObject * async_gen_athrow_close(PyAsyncGenAThrow *o, PyObject *args) { - o->agt_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyObject *result; + if (o->agt_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + result = async_gen_athrow_throw(o, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } diff --git a/Objects/listobject.c b/Objects/listobject.c index 4eaf20033fa262..3c4e2d2e6ed7de 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -141,6 +141,9 @@ list_resize(PyListObject *self, Py_ssize_t newsize) target_bytes = allocated * sizeof(PyObject*); } memcpy(array->ob_item, self->ob_item, target_bytes); + } + if (new_allocated > (size_t)allocated) { + memset(array->ob_item + allocated, 0, sizeof(PyObject *) * (new_allocated - allocated)); } _Py_atomic_store_ptr_release(&self->ob_item, &array->ob_item); self->allocated = new_allocated; @@ -502,7 +505,7 @@ _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) Py_DECREF(newitem); return -1; } - PyList_SET_ITEM(self, len, newitem); + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], newitem); return 0; } @@ -1181,7 +1184,7 @@ list_extend_fast(PyListObject *self, PyObject *iterable) PyObject **dest = self->ob_item + m; for (Py_ssize_t i = 0; i < n; i++) { PyObject *o = src[i]; - dest[i] = Py_NewRef(o); + FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(o)); } return 0; } @@ -1238,7 +1241,7 @@ list_extend_iter_lock_held(PyListObject *self, PyObject *iterable) if (Py_SIZE(self) < self->allocated) { Py_ssize_t len = Py_SIZE(self); - PyList_SET_ITEM(self, len, item); // steals item ref + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], item); // steals item ref Py_SET_SIZE(self, len + 1); } else { diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 2f6adb9a2e12be..46995b948a28e7 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -183,6 +183,7 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) if (func == NULL) { return -1; } + _PyObject_SetDeferredRefcount(func); if (PyObject_SetAttrString(module, fdef->ml_name, func) != 0) { Py_DECREF(func); return -1; @@ -249,6 +250,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) } } m->md_def = module; +#ifdef Py_GIL_DISABLE + m->md_gil = Py_MOD_GIL_USED; +#endif return (PyObject*)m; } @@ -261,6 +265,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *m = NULL; int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; + int has_gil_slot = 0; + void *gil_slot = Py_MOD_GIL_USED; int has_execution_slots = 0; const char *name; int ret; @@ -315,6 +321,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio multiple_interpreters = cur_slot->value; has_multiple_interpreters_slot = 1; break; + case Py_mod_gil: + if (has_gil_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'gil' slot", + name); + goto error; + } + gil_slot = cur_slot->value; + has_gil_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -374,6 +391,11 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio if (PyModule_Check(m)) { ((PyModuleObject*)m)->md_state = NULL; ((PyModuleObject*)m)->md_def = def; +#ifdef Py_GIL_DISABLED + ((PyModuleObject*)m)->md_gil = gil_slot; +#else + (void)gil_slot; +#endif } else { if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { PyErr_Format( @@ -415,6 +437,19 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +#ifdef Py_GIL_DISABLED +int +PyUnstable_Module_SetGIL(PyObject *module, void *gil) +{ + if (!PyModule_Check(module)) { + PyErr_BadInternalCall(); + return -1; + } + ((PyModuleObject *)module)->md_gil = gil; + return 0; +} +#endif + int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { @@ -470,6 +505,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) } break; case Py_mod_multiple_interpreters: + case Py_mod_gil: /* handled in PyModule_FromDefAndSpec2 */ break; default: diff --git a/Objects/object.c b/Objects/object.c index 8d856939254080..8ad0389cbc7626 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1132,8 +1132,8 @@ _PyObject_SetAttrId(PyObject *v, _Py_Identifier *name, PyObject *w) return result; } -static inline int -set_attribute_error_context(PyObject* v, PyObject* name) +int +_PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name) { assert(PyErr_Occurred()); if (!PyErr_ExceptionMatches(PyExc_AttributeError)){ @@ -1188,7 +1188,7 @@ PyObject_GetAttr(PyObject *v, PyObject *name) } if (result == NULL) { - set_attribute_error_context(v, name); + _PyObject_SetAttributeErrorContext(v, name); } return result; } @@ -1466,13 +1466,13 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } - PyObject *descr = _PyType_Lookup(tp, name); + PyObject *descr = _PyType_LookupRef(tp, name); descrgetfunc f = NULL; if (descr != NULL) { - Py_INCREF(descr); if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { meth_found = 1; - } else { + } + else { f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { *method = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1535,7 +1535,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); return 0; } @@ -1569,11 +1569,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, goto done; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); f = NULL; if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1647,7 +1646,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1670,6 +1669,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, descrsetfunc f; int res = -1; + assert(!PyType_IsSubtype(tp, &PyType_Type)); if (!PyUnicode_Check(name)){ PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", @@ -1683,10 +1683,9 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, Py_INCREF(name); Py_INCREF(tp); - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_set; if (f != NULL) { res = f(descr, obj, value); @@ -1722,7 +1721,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); } - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } else { PyErr_Format(PyExc_AttributeError, @@ -1732,7 +1731,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto done; } else { - res = _PyObjectDict_SetItem(tp, dictptr, name, value); + res = _PyObjectDict_SetItem(tp, obj, dictptr, name, value); } } else { @@ -1745,17 +1744,10 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } error_check: if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - if (PyType_IsSubtype(tp, &PyType_Type)) { - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - ((PyTypeObject*)obj)->tp_name, name); - } - else { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); - } - set_attribute_error_context(obj, name); + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1797,7 +1789,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } + Py_BEGIN_CRITICAL_SECTION(obj); Py_XSETREF(*dictptr, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -2235,6 +2229,7 @@ static PyTypeObject* static_types[] = { &PyFilter_Type, &PyFloat_Type, &PyFrame_Type, + &PyFrameLocalsProxy_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, @@ -2281,9 +2276,11 @@ static PyTypeObject* static_types[] = { &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, +#ifdef _Py_TIER2 &_PyCounterExecutor_Type, &_PyCounterOptimizer_Type, &_PyDefaultOptimizer_Type, +#endif &_Py_GenericAliasIterType, &_PyHamtItems_Type, &_PyHamtKeys_Type, @@ -2304,8 +2301,10 @@ static PyTypeObject* static_types[] = { &_PyPositionsIterator, &_PyUnicodeASCIIIter_Type, &_PyUnion_Type, +#ifdef _Py_TIER2 &_PyUOpExecutor_Type, &_PyUOpOptimizer_Type, +#endif &_PyWeakref_CallableProxyType, &_PyWeakref_ProxyType, &_PyWeakref_RefType, @@ -2368,9 +2367,6 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) static inline void new_reference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 #if !defined(Py_GIL_DISABLED) op->ob_refcnt = 1; @@ -2385,6 +2381,11 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_CREATE, data); + } } void @@ -2446,12 +2447,13 @@ _PyObject_SetDeferredRefcount(PyObject *op) void _Py_ResurrectReference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + void* data = _PyRuntime.ref_tracer.tracer_data; + _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); + } } @@ -2841,6 +2843,12 @@ _Py_Dealloc(PyObject *op) Py_INCREF(type); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_DESTROY, data); + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif @@ -2929,6 +2937,22 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) Py_SET_REFCNT(ob, refcnt); } +int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { + assert(PyGILState_Check()); + _PyRuntime.ref_tracer.tracer_func = tracer; + _PyRuntime.ref_tracer.tracer_data = data; + return 0; +} + +PyRefTracer PyRefTracer_GetTracer(void** data) { + assert(PyGILState_Check()); + if (data != NULL) { + *data = _PyRuntime.ref_tracer.tracer_data; + } + return _PyRuntime.ref_tracer.tracer_func; +} + + static PyObject* constants[] = { &_Py_NoneStruct, // Py_CONSTANT_NONE diff --git a/Objects/setobject.c b/Objects/setobject.c index 19975e3d4d18e2..68986bb6a6b557 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2621,6 +2621,12 @@ PySet_Clear(PyObject *set) return 0; } +void +_PySet_ClearInternal(PySetObject *so) +{ + (void)set_clear_internal(so); +} + int PySet_Contains(PyObject *anyset, PyObject *key) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 50efbb6e182df0..4b144fab5de8f1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -43,7 +43,8 @@ class object "PyObject *" "&PyBaseObject_Type" & ((1 << MCACHE_SIZE_EXP) - 1)) #define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH((type)->tp_version_tag, ((Py_ssize_t)(name)) >> 3) + MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ + ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ PyUnicode_IS_READY(name) && \ @@ -70,6 +71,16 @@ class object "PyObject *" "&PyBaseObject_Type" _PyCriticalSection_End(&_cs); \ } +#define BEGIN_TYPE_DICT_LOCK(d) \ + { \ + _PyCriticalSection2 _cs; \ + _PyCriticalSection2_Begin(&_cs, TYPE_LOCK, \ + &_PyObject_CAST(d)->ob_mutex); \ + +#define END_TYPE_DICT_LOCK() \ + _PyCriticalSection2_End(&_cs); \ + } + #define ASSERT_TYPE_LOCK_HELD() \ _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) @@ -77,6 +88,8 @@ class object "PyObject *" "&PyBaseObject_Type" #define BEGIN_TYPE_LOCK() #define END_TYPE_LOCK() +#define BEGIN_TYPE_DICT_LOCK(d) +#define END_TYPE_DICT_LOCK() #define ASSERT_TYPE_LOCK_HELD() #endif @@ -727,7 +740,7 @@ _PyType_InitCache(PyInterpreterState *interp) assert(entry->name == NULL); entry->version = 0; - // Set to None so _PyType_Lookup() can use Py_SETREF(), + // Set to None so _PyType_LookupRef() can use Py_SETREF(), // rather than using slower Py_XSETREF(). entry->name = Py_None; entry->value = NULL; @@ -739,7 +752,7 @@ static unsigned int _PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->types.type_cache; - // Set to None, rather than NULL, so _PyType_Lookup() can + // Set to None, rather than NULL, so _PyType_LookupRef() can // use Py_SETREF() rather than using slower Py_XSETREF(). type_cache_clear(cache, Py_None); @@ -848,6 +861,44 @@ PyType_Unwatch(int watcher_id, PyObject* obj) return 0; } +#ifdef Py_GIL_DISABLED + +static void +type_modification_starting_unlocked(PyTypeObject *type) +{ + ASSERT_TYPE_LOCK_HELD(); + + /* Clear version tags on all types, but leave the valid + version tag intact. This prepares for a modification so that + any concurrent readers of the type cache will not see invalid + values. + */ + if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + return; + } + + PyObject *subclasses = lookup_tp_subclasses(type); + if (subclasses != NULL) { + assert(PyDict_CheckExact(subclasses)); + + Py_ssize_t i = 0; + PyObject *ref; + while (PyDict_Next(subclasses, &i, NULL, &ref)) { + PyTypeObject *subclass = type_from_ref(ref); + if (subclass == NULL) { + continue; + } + type_modification_starting_unlocked(subclass); + Py_DECREF(subclass); + } + } + + /* 0 is not a valid version tag */ + _Py_atomic_store_uint32_release(&type->tp_version_tag, 0); +} + +#endif + static void type_modified_unlocked(PyTypeObject *type) { @@ -881,7 +932,7 @@ type_modified_unlocked(PyTypeObject *type) if (subclass == NULL) { continue; } - PyType_Modified(subclass); + type_modified_unlocked(subclass); Py_DECREF(subclass); } } @@ -907,7 +958,7 @@ type_modified_unlocked(PyTypeObject *type) } type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -984,7 +1035,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1020,7 +1071,8 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, + NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1029,7 +1081,8 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_VERSION_TAG(interp)++; + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, + NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } @@ -2349,7 +2402,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) Variants: - _PyObject_LookupSpecial() returns NULL without raising an exception - when the _PyType_Lookup() call fails; + when the _PyType_LookupRef() call fails; - lookup_maybe_method() and lookup_method() are internal routines similar to _PyObject_LookupSpecial(), but can return unbound PyFunction @@ -2362,13 +2415,12 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr) { PyObject *res; - res = _PyType_Lookup(Py_TYPE(self), attr); + res = _PyType_LookupRef(Py_TYPE(self), attr); if (res != NULL) { descrgetfunc f; - if ((f = Py_TYPE(res)->tp_descr_get) == NULL) - Py_INCREF(res); - else - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if ((f = Py_TYPE(res)->tp_descr_get) != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } } return res; } @@ -2385,7 +2437,7 @@ _PyObject_LookupSpecialId(PyObject *self, _Py_Identifier *attrid) static PyObject * lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) { - PyObject *res = _PyType_Lookup(Py_TYPE(self), attr); + PyObject *res = _PyType_LookupRef(Py_TYPE(self), attr); if (res == NULL) { return NULL; } @@ -2393,16 +2445,12 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { /* Avoid temporary PyMethodObject */ *unbound = 1; - Py_INCREF(res); } else { *unbound = 0; descrgetfunc f = Py_TYPE(res)->tp_descr_get; - if (f == NULL) { - Py_INCREF(res); - } - else { - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); } } return res; @@ -4978,14 +5026,13 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); - res = _PyDict_GetItem_KnownHash(dict, name, hash); - if (res != NULL) { - break; - } - if (PyErr_Occurred()) { + if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { *error = -1; goto done; } + if (res != NULL) { + break; + } } *error = 0; done: @@ -5027,8 +5074,6 @@ update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int versio #if Py_GIL_DISABLED -#define TYPE_CACHE_IS_UPDATING(sequence) (sequence & 0x01) - static void update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) @@ -5072,7 +5117,7 @@ _PyTypes_AfterFork(void) /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ PyObject * -_PyType_Lookup(PyTypeObject *type, PyObject *name) +_PyType_LookupRef(PyTypeObject *type, PyObject *name) { PyObject *res; int error; @@ -5085,18 +5130,26 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) // synchronize-with other writing threads by doing an acquire load on the sequence while (1) { int sequence = _PySeqLock_BeginRead(&entry->sequence); - if (_Py_atomic_load_uint32_relaxed(&entry->version) == type->tp_version_tag && + uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); + uint32_t type_version = _Py_atomic_load_uint32_acquire(&type->tp_version_tag); + if (entry_version == type_version && _Py_atomic_load_ptr_relaxed(&entry->name) == name) { - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); PyObject *value = _Py_atomic_load_ptr_relaxed(&entry->value); - // If the sequence is still valid then we're done - if (_PySeqLock_EndRead(&entry->sequence, sequence)) { - return value; + if (value == NULL || _Py_TryIncref(value)) { + if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + return value; + } + Py_XDECREF(value); + } + else { + // If we can't incref the object we need to fallback to locking + break; } - } else { + } + else { // cache miss break; } @@ -5107,6 +5160,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + Py_XINCREF(entry->value); return entry->value; } #endif @@ -5156,6 +5210,14 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return res; } +PyObject * +_PyType_Lookup(PyTypeObject *type, PyObject *name) +{ + PyObject *res = _PyType_LookupRef(type, name); + Py_XDECREF(res); + return res; +} + PyObject * _PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) { @@ -5213,7 +5275,7 @@ _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. The argument suppress_missing_attribute is used to provide a fast path for hasattr. The possible values are: @@ -5249,10 +5311,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin meta_get = NULL; /* Look for the attribute in the metatype */ - meta_attribute = _PyType_Lookup(metatype, name); + meta_attribute = _PyType_LookupRef(metatype, name); if (meta_attribute != NULL) { - Py_INCREF(meta_attribute); meta_get = Py_TYPE(meta_attribute)->tp_descr_get; if (meta_get != NULL && PyDescr_IsData(meta_attribute)) { @@ -5269,10 +5330,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin /* No data descriptor found on metatype. Look in tp_dict of this * type and its bases */ - attribute = _PyType_Lookup(type, name); + attribute = _PyType_LookupRef(type, name); if (attribute != NULL) { /* Implement descriptor functionality, if any */ - Py_INCREF(attribute); descrgetfunc local_get = Py_TYPE(attribute)->tp_descr_get; Py_XDECREF(meta_attribute); @@ -5317,7 +5377,7 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. */ PyObject * _Py_type_getattro(PyObject *type, PyObject *name) { @@ -5336,49 +5396,110 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) name, type->tp_name); return -1; } - if (PyUnicode_Check(name)) { - if (PyUnicode_CheckExact(name)) { - Py_INCREF(name); + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "attribute name must be string, not '%.200s'", + Py_TYPE(name)->tp_name); + return -1; + } + + if (PyUnicode_CheckExact(name)) { + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + /* bpo-40521: Interned strings are shared by all subinterpreters */ + if (!PyUnicode_CHECK_INTERNED(name)) { + PyUnicode_InternInPlace(&name); + if (!PyUnicode_CHECK_INTERNED(name)) { + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; } - else { - name = _PyUnicode_Copy(name); - if (name == NULL) - return -1; + } + + PyTypeObject *metatype = Py_TYPE(type); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); + + PyObject *old_value; + PyObject *descr = _PyType_LookupRef(metatype, name); + if (descr != NULL) { + descrsetfunc f = Py_TYPE(descr)->tp_descr_set; + if (f != NULL) { + old_value = NULL; + res = f(descr, (PyObject *)type, value); + goto done; } - /* bpo-40521: Interned strings are shared by all subinterpreters */ - if (!PyUnicode_CHECK_INTERNED(name)) { - PyUnicode_InternInPlace(&name); - if (!PyUnicode_CHECK_INTERNED(name)) { - PyErr_SetString(PyExc_MemoryError, - "Out of memory interning an attribute name"); - Py_DECREF(name); - return -1; - } + } + + PyObject *dict = type->tp_dict; + if (dict == NULL) { + // We don't just do PyType_Ready because we could already be readying + BEGIN_TYPE_LOCK(); + dict = type->tp_dict; + if (dict == NULL) { + dict = type->tp_dict = PyDict_New(); + } + END_TYPE_LOCK(); + if (dict == NULL) { + return -1; } } - else { - /* Will fail in _PyObject_GenericSetAttrWithDict. */ - Py_INCREF(name); + + // We don't want any re-entrancy between when we update the dict + // and call type_modified_unlocked, including running the destructor + // of the current value as it can observe the cache in an inconsistent + // state. Because we have an exact unicode and our dict has exact + // unicodes we know that this will all complete without releasing + // the locks. + BEGIN_TYPE_DICT_LOCK(dict); + + if (_PyDict_GetItemRef_Unicode_LockHeld((PyDictObject *)dict, name, &old_value) < 0) { + return -1; } - BEGIN_TYPE_LOCK() - res = _PyObject_GenericSetAttrWithDict((PyObject *)type, name, value, NULL); - if (res == 0) { - /* Clear the VALID_VERSION flag of 'type' and all its - subclasses. This could possibly be unified with the - update_subclasses() recursion in update_slot(), but carefully: - they each have their own conditions on which to stop - recursing into subclasses. */ - type_modified_unlocked(type); +#ifdef Py_GIL_DISABLED + // In free-threaded builds readers can race with the lock-free portion + // of the type cache and the assignment into the dict. We clear all of the + // versions initially so no readers will succeed in the lock-free case. + // They'll then block on the type lock until the update below is done. + type_modification_starting_unlocked(type); +#endif + + res = _PyDict_SetItem_LockHeld((PyDictObject *)dict, name, value); + /* Clear the VALID_VERSION flag of 'type' and all its + subclasses. This could possibly be unified with the + update_subclasses() recursion in update_slot(), but carefully: + they each have their own conditions on which to stop + recursing into subclasses. */ + type_modified_unlocked(type); + + if (res == 0) { if (is_dunder_name(name)) { res = update_slot(type, name); } - assert(_PyType_CheckConsistency(type)); } - END_TYPE_LOCK() + else if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Format(PyExc_AttributeError, + "type object '%.50s' has no attribute '%U'", + ((PyTypeObject*)type)->tp_name, name); + + _PyObject_SetAttributeErrorContext((PyObject *)type, name); + } + + assert(_PyType_CheckConsistency(type)); + END_TYPE_DICT_LOCK(); +done: Py_DECREF(name); + Py_XDECREF(descr); + Py_XDECREF(old_value); return res; } @@ -9338,33 +9459,33 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when the attribute is present. So we use - _PyType_Lookup and create the method only when needed, with + _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattr = _PyType_Lookup(tp, &_Py_ID(__getattr__)); + getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ tp->tp_getattro = _Py_slot_tp_getattro; return _Py_slot_tp_getattro(self, name); } - Py_INCREF(getattr); /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when self has the default __getattribute__ - method. So we use _PyType_Lookup and create the method only when + method. So we use _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattribute = _PyType_Lookup(tp, &_Py_ID(__getattribute__)); + getattribute = _PyType_LookupRef(tp, &_Py_ID(__getattribute__)); if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) { + Py_XDECREF(getattribute); res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); /* if res == NULL with no exception set, then it must be an AttributeError suppressed by us. */ if (res == NULL && !PyErr_Occurred()) { res = call_attribute(self, getattr, name); } - } else { - Py_INCREF(getattribute); + } + else { res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -9471,7 +9592,7 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyTypeObject *tp = Py_TYPE(self); PyObject *get; - get = _PyType_Lookup(tp, &_Py_ID(__get__)); + get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -9483,7 +9604,9 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) if (type == NULL) type = Py_None; PyObject *stack[3] = {self, obj, type}; - return PyObject_Vectorcall(get, stack, 3, NULL); + PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); + Py_DECREF(get); + return res; } static int @@ -10378,6 +10501,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) type->tp_flags &= ~Py_TPFLAGS_HAVE_VECTORCALL; } } + Py_DECREF(descr); } while ((++p)->offset == offset); if (specific && !use_generic) *ptr = specific; diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 7f80c9c61b8abc..cc916045266aea 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -23,6 +23,8 @@ typedef struct { PyObject *evaluate_bound; PyObject *constraints; PyObject *evaluate_constraints; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -31,12 +33,16 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *default_value; + PyObject *evaluate_default; } typevartupleobject; typedef struct { PyObject_HEAD PyObject *name; PyObject *bound; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -53,6 +59,64 @@ typedef struct { #include "clinic/typevarobject.c.h" +/* NoDefault is a marker object to indicate that a parameter has no default. */ + +static PyObject * +NoDefault_repr(PyObject *op) +{ + return PyUnicode_FromString("typing.NoDefault"); +} + +static PyObject * +NoDefault_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("NoDefault"); +} + +static PyMethodDef notimplemented_methods[] = { + {"__reduce__", NoDefault_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static PyObject * +nodefault_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "NoDefaultType takes no arguments"); + return NULL; + } + return &_Py_NoDefaultStruct; +} + +static void +nodefault_dealloc(PyObject *nodefault) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref NoDefault out of existence. Instead, + * since NoDefault is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(nodefault); +} + +PyDoc_STRVAR(notimplemented_doc, +"NoDefaultType()\n" +"--\n\n" +"The type of the NoDefault singleton."); + +PyTypeObject _PyNoDefault_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "NoDefaultType", + .tp_dealloc = nodefault_dealloc, + .tp_repr = NoDefault_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = notimplemented_doc, + .tp_methods = notimplemented_methods, + .tp_new = nodefault_new, +}; + +PyObject _Py_NoDefaultStruct = _PyObject_HEAD_INIT(&_PyNoDefault_Type); + + static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) { @@ -200,6 +264,8 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); Py_XDECREF(tv->evaluate_constraints); + Py_XDECREF(tv->default_value); + Py_XDECREF(tv->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -216,6 +282,8 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); Py_VISIT(tv->evaluate_constraints); + Py_VISIT(tv->default_value); + Py_VISIT(tv->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -227,6 +295,8 @@ typevar_clear(typevarobject *self) Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); Py_CLEAR(self->evaluate_constraints); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -266,6 +336,20 @@ typevar_bound(typevarobject *self, void *Py_UNUSED(ignored)) return bound; } +static PyObject * +typevar_default(typevarobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyObject * typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) { @@ -283,12 +367,14 @@ typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) static PyGetSetDef typevar_getset[] = { {"__bound__", (getter)typevar_bound, NULL, NULL, NULL}, {"__constraints__", (getter)typevar_constraints, NULL, NULL, NULL}, + {"__default__", (getter)typevar_default, NULL, NULL, NULL}, {0} }; static typevarobject * typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, PyObject *constraints, PyObject *evaluate_constraints, + PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { @@ -305,6 +391,8 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, tv->evaluate_bound = Py_XNewRef(evaluate_bound); tv->constraints = Py_XNewRef(constraints); tv->evaluate_constraints = Py_XNewRef(evaluate_constraints); + tv->default_value = Py_XNewRef(default_value); + tv->evaluate_default = NULL; tv->covariant = covariant; tv->contravariant = contravariant; @@ -328,6 +416,7 @@ typevar.__new__ as typevar_new name: object(subclass_of="&PyUnicode_Type") *constraints: object bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -337,9 +426,9 @@ Create a TypeVar. static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance) -/*[clinic end generated code: output=1d200450ee99226d input=41ae33a916bfe76f]*/ + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance) +/*[clinic end generated code: output=d2b248ff074eaab6 input=836f97f631d7293a]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, @@ -386,6 +475,7 @@ typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, PyObject *tv = (PyObject *)typevar_alloc(name, bound, NULL, constraints, NULL, + default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); @@ -410,6 +500,66 @@ typevar_typing_subst(typevarobject *self, PyObject *arg) return result; } +/*[clinic input] +typevar.__typing_prepare_subst__ as typevar_typing_prepare_subst + + alias: object + args: object + / + +[clinic start generated code]*/ + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args) +/*[clinic end generated code: output=82c3f4691e0ded22 input=201a750415d14ffb]*/ +{ + PyObject *params = PyObject_GetAttrString(alias, "__parameters__"); + if (params == NULL) { + return NULL; + } + Py_ssize_t i = PySequence_Index(params, (PyObject *)self); + if (i == -1) { + Py_DECREF(params); + return NULL; + } + Py_ssize_t args_len = PySequence_Length(args); + if (args_len == -1) { + Py_DECREF(params); + return NULL; + } + if (i < args_len) { + // We already have a value for our TypeVar + Py_DECREF(params); + return Py_NewRef(args); + } + else if (i == args_len) { + // If the TypeVar has a default, use it. + PyObject *dflt = typevar_default(self, NULL); + if (dflt == NULL) { + Py_DECREF(params); + return NULL; + } + if (dflt != &_Py_NoDefaultStruct) { + PyObject *new_args = PyTuple_Pack(1, dflt); + Py_DECREF(dflt); + if (new_args == NULL) { + Py_DECREF(params); + return NULL; + } + PyObject *result = PySequence_Concat(args, new_args); + Py_DECREF(params); + Py_DECREF(new_args); + return result; + } + } + Py_DECREF(params); + PyErr_Format(PyExc_TypeError, + "Too few arguments for %S; actual %d, expected at least %d", + alias, args_len, i + 1); + return NULL; +} + /*[clinic input] typevar.__reduce__ as typevar_reduce @@ -422,6 +572,23 @@ typevar_reduce_impl(typevarobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevar.has_default as typevar_has_default + +[clinic start generated code]*/ + +static PyObject * +typevar_has_default_impl(typevarobject *self) +/*[clinic end generated code: output=76bf0b8dc98b97dd input=31024aa030761cf6]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevar_mro_entries(PyObject *self, PyObject *args) { @@ -432,7 +599,9 @@ typevar_mro_entries(PyObject *self, PyObject *args) static PyMethodDef typevar_methods[] = { TYPEVAR_TYPING_SUBST_METHODDEF + TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF TYPEVAR_REDUCE_METHODDEF + TYPEVAR_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevar_mro_entries, METH_O}, {0} }; @@ -457,12 +626,18 @@ variables::\n\ class StrOrBytesSequence[A: (str, bytes)]:\n\ ...\n\ \n\ +Type variables can also have defaults:\n\ +\n\ + class IntDefault[T = int]:\n\ + ...\n\ +\n\ However, if desired, reusable type variables can also be constructed\n\ manually, like so::\n\ \n\ T = TypeVar('T') # Can be anything\n\ S = TypeVar('S', bound=str) # Can be any subtype of str\n\ A = TypeVar('A', str, bytes) # Must be exactly str or bytes\n\ + D = TypeVar('D', default=int) # Defaults to int\n\ \n\ Type variables exist primarily for the benefit of static type\n\ checkers. They serve as the parameters for generic types as well\n\ @@ -739,6 +914,8 @@ paramspec_dealloc(PyObject *self) Py_DECREF(ps->name); Py_XDECREF(ps->bound); + Py_XDECREF(ps->default_value); + Py_XDECREF(ps->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -752,6 +929,8 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); paramspecobject *ps = (paramspecobject *)self; Py_VISIT(ps->bound); + Py_VISIT(ps->default_value); + Py_VISIT(ps->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -760,6 +939,8 @@ static int paramspec_clear(paramspecobject *self) { Py_CLEAR(self->bound); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -800,14 +981,29 @@ paramspec_kwargs(PyObject *self, void *unused) return (PyObject *)paramspecattr_new(tp, self); } +static PyObject * +paramspec_default(paramspecobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyGetSetDef paramspec_getset[] = { {"args", (getter)paramspec_args, NULL, PyDoc_STR("Represents positional arguments."), NULL}, {"kwargs", (getter)paramspec_kwargs, NULL, PyDoc_STR("Represents keyword arguments."), NULL}, + {"__default__", (getter)paramspec_default, NULL, "The default value for this ParamSpec.", NULL}, {0}, }; static paramspecobject * -paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, +paramspec_alloc(PyObject *name, PyObject *bound, PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; @@ -820,6 +1016,8 @@ paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, ps->covariant = covariant; ps->contravariant = contravariant; ps->infer_variance = infer_variance; + ps->default_value = Py_XNewRef(default_value); + ps->evaluate_default = NULL; _PyObject_GC_TRACK(ps); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)ps, "__module__", module) < 0) { @@ -837,6 +1035,7 @@ paramspec.__new__ as paramspec_new name: object(subclass_of="&PyUnicode_Type") * bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -846,8 +1045,9 @@ Create a ParamSpec object. static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance) -/*[clinic end generated code: output=fd2daab79cba62da input=57c49c581979b952]*/ + PyObject *default_value, int covariant, int contravariant, + int infer_variance) +/*[clinic end generated code: output=47ca9d63fa5a094d input=495e1565bc067ab9]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); @@ -869,7 +1069,7 @@ paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, return NULL; } PyObject *ps = (PyObject *)paramspec_alloc( - name, bound, covariant, contravariant, infer_variance, module); + name, bound, default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); Py_DECREF(module); return ps; @@ -925,6 +1125,22 @@ paramspec_reduce_impl(paramspecobject *self) return Py_NewRef(self->name); } +/*[clinic input] +paramspec.has_default as paramspec_has_default + +[clinic start generated code]*/ + +static PyObject * +paramspec_has_default_impl(paramspecobject *self) +/*[clinic end generated code: output=daaae7467a6a4368 input=2112e97eeb76cd59]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * paramspec_mro_entries(PyObject *self, PyObject *args) { @@ -936,6 +1152,7 @@ paramspec_mro_entries(PyObject *self, PyObject *args) static PyMethodDef paramspec_methods[] = { PARAMSPEC_TYPING_SUBST_METHODDEF PARAMSPEC_TYPING_PREPARE_SUBST_METHODDEF + PARAMSPEC_HAS_DEFAULT_METHODDEF PARAMSPEC_REDUCE_METHODDEF {"__mro_entries__", paramspec_mro_entries, METH_O}, {0} @@ -950,10 +1167,17 @@ where the use of '**' creates a parameter specification::\n\ \n\ type IntFunc[**P] = Callable[P, int]\n\ \n\ +The following syntax creates a parameter specification that defaults\n\ +to a callable accepting two positional-only arguments of types int\n\ +and str:\n\ +\n\ + type IntFuncDefault[**P = (int, str)] = Callable[P, int]\n\ +\n\ For compatibility with Python 3.11 and earlier, ParamSpec objects\n\ can also be created as follows::\n\ \n\ P = ParamSpec('P')\n\ + DefaultP = ParamSpec('DefaultP', default=(int, str))\n\ \n\ Parameter specification variables exist primarily for the benefit of\n\ static type checkers. They are used to forward the parameter types of\n\ @@ -1021,6 +1245,8 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = (typevartupleobject *)self; Py_DECREF(tvt->name); + Py_XDECREF(tvt->default_value); + Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -1060,7 +1286,7 @@ static PyMemberDef typevartuple_members[] = { }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module) +typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1068,6 +1294,8 @@ typevartuple_alloc(PyObject *name, PyObject *module) return NULL; } tvt->name = Py_NewRef(name); + tvt->default_value = Py_XNewRef(default_value); + tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)tvt, "__module__", module) < 0) { @@ -1083,19 +1311,22 @@ typevartuple_alloc(PyObject *name, PyObject *module) typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") + * + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name) -/*[clinic end generated code: output=09d417a28f976202 input=00d28abcf1fc96bb]*/ +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ { PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module); + PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); Py_DECREF(module); return result; } @@ -1148,6 +1379,23 @@ typevartuple_reduce_impl(typevartupleobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevartuple.has_default as typevartuple_has_default + +[clinic start generated code]*/ + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self) +/*[clinic end generated code: output=4895f602f56a5e29 input=9ef3250ddb2c1851]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevartuple_mro_entries(PyObject *self, PyObject *args) { @@ -1160,6 +1408,8 @@ static int typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); + Py_VISIT(((typevartupleobject *)self)->default_value); + Py_VISIT(((typevartupleobject *)self)->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -1167,14 +1417,36 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) static int typevartuple_clear(PyObject *self) { + Py_CLEAR(((typevartupleobject *)self)->default_value); + Py_CLEAR(((typevartupleobject *)self)->evaluate_default); PyObject_ClearManagedDict(self); return 0; } +static PyObject * +typevartuple_default(typevartupleobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + +static PyGetSetDef typevartuple_getset[] = { + {"__default__", (getter)typevartuple_default, NULL, "The default value for this TypeVarTuple.", NULL}, + {0}, +}; + static PyMethodDef typevartuple_methods[] = { TYPEVARTUPLE_TYPING_SUBST_METHODDEF TYPEVARTUPLE_TYPING_PREPARE_SUBST_METHODDEF TYPEVARTUPLE_REDUCE_METHODDEF + TYPEVARTUPLE_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevartuple_mro_entries, METH_O}, {0} }; @@ -1190,10 +1462,15 @@ where a single '*' indicates a type variable tuple::\n\ def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]:\n\ return (*tup[1:], tup[0])\n\ \n\ +Type variables tuples can have default values:\n\ +\n\ + type AliasWithDefault[*Ts = (str, int)] = tuple[*Ts]\n\ +\n\ For compatibility with Python 3.11 and earlier, TypeVarTuple objects\n\ can also be created as follows::\n\ \n\ Ts = TypeVarTuple('Ts') # Can be given any name\n\ + DefaultTs = TypeVarTuple('Ts', default=(str, int))\n\ \n\ Just as a TypeVar (type variable) is a placeholder for a single type,\n\ a TypeVarTuple is a placeholder for an *arbitrary* number of types. For\n\ @@ -1218,6 +1495,7 @@ PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, + {Py_tp_getset, typevartuple_getset}, {Py_tp_new, typevartuple}, {Py_tp_iter, typevartuple_iter}, {Py_tp_repr, typevartuple_repr}, @@ -1241,21 +1519,21 @@ PyObject * _Py_make_typevar(PyObject *name, PyObject *evaluate_bound, PyObject *evaluate_constraints) { return (PyObject *)typevar_alloc(name, NULL, evaluate_bound, NULL, evaluate_constraints, - false, false, true, NULL); + NULL, false, false, true, NULL); } PyObject * _Py_make_paramspec(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)paramspec_alloc(v, NULL, false, false, true, NULL); + return (PyObject *)paramspec_alloc(v, NULL, NULL, false, false, true, NULL); } PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL); } static void @@ -1687,3 +1965,24 @@ void _Py_clear_generic_types(PyInterpreterState *interp) Py_CLEAR(interp->cached_objects.paramspecargs_type); Py_CLEAR(interp->cached_objects.paramspeckwargs_type); } + +PyObject * +_Py_set_typeparam_default(PyThreadState *ts, PyObject *typeparam, PyObject *evaluate_default) +{ + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + Py_XSETREF(((typevarobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + Py_XSETREF(((paramspecobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + Py_XSETREF(((typevartupleobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } +} diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2c259b7e869efe..057b417074ebea 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15441,7 +15441,11 @@ init_fs_encoding(PyThreadState *tstate) PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate) { - PyStatus status = init_fs_encoding(tstate); + PyStatus status = _PyCodec_InitRegistry(tstate->interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = init_fs_encoding(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -15537,6 +15541,7 @@ static PyMethodDef _string_methods[] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 93c5fe3aacecfd..88afaec86827ed 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -988,7 +988,7 @@ PyObject_ClearWeakRefs(PyObject *object) } list = GET_WEAKREFS_LISTPTR(object); - if (FT_ATOMIC_LOAD_PTR(list) == NULL) { + if (FT_ATOMIC_LOAD_PTR(*list) == NULL) { // Fast path for the common case return; } diff --git a/PC/_testconsole.c b/PC/_testconsole.c index f1ace003df483b..0dcea866f65d35 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -1,10 +1,10 @@ /* Testing module for multi-phase initialization of extension modules (PEP 489) */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -31,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 22ed05276e6f07..48863b90f4cc27 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -362,12 +362,18 @@ static PyMethodDef wmi_functions[] = { { NULL, NULL, 0, NULL } }; +static PyModuleDef_Slot wmi_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef wmi_def = { PyModuleDef_HEAD_INIT, "_wmi", - NULL, // doc - 0, // m_size - wmi_functions + NULL, // doc + 0, // m_size + wmi_functions, // m_methods + wmi_slots, // m_slots }; extern "C" { diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 5ff703217b421f..b170e06b47dd59 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -656,6 +656,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/python3dll.c b/PC/python3dll.c index c6fdc0bd73b9fe..86c888430891c9 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -253,6 +253,9 @@ EXPORT_FUNC(PyEval_EvalFrame) EXPORT_FUNC(PyEval_EvalFrameEx) EXPORT_FUNC(PyEval_GetBuiltins) EXPORT_FUNC(PyEval_GetFrame) +EXPORT_FUNC(PyEval_GetFrameBuiltins) +EXPORT_FUNC(PyEval_GetFrameGlobals) +EXPORT_FUNC(PyEval_GetFrameLocals) EXPORT_FUNC(PyEval_GetFuncDesc) EXPORT_FUNC(PyEval_GetFuncName) EXPORT_FUNC(PyEval_GetGlobals) diff --git a/PC/winreg.c b/PC/winreg.c index 8096d17e43b7bc..efdf8addc06186 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -2179,6 +2179,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index a6b2dac6ac1466..094c77ae34d678 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,10 +35,10 @@ winsound.PlaySound(None, 0) */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -246,6 +246,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 9717d89b54d828..e5e18de60ec349 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -240,6 +240,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 9b106bea601e34..9630f54ae4ea29 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -94,6 +94,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index cc25b6ebd7c673..44dbf2348137e1 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -125,6 +125,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 28c82254d85d4c..cae44bc955f7f1 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -108,6 +108,9 @@ Source Files + + Source Files + diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index a825cac9138674..87db569423de2a 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -108,6 +108,12 @@ false + + + _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) + + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index abfeeb39630daf..27429ea5833077 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -27,4 +27,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 83b50db4467033..6c76f09a071312 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -36,7 +36,9 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. -echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). +echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -66,6 +68,7 @@ set verbose=/nologo /v:m /clp:summary set kill= set do_pgo= set pgo_job=-m test --pgo +set UseTIER2= :CheckOpts if "%~1"=="-h" goto Usage @@ -86,7 +89,10 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts -if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter" (set UseTIER2=4) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -179,6 +185,7 @@ echo on /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ /p:UseJIT=%UseJIT%^ + /p:UseTIER2=%UseTIER2%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 8f07c414272c47..761d3de93b777d 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 -set libraries=%libraries% mpdecimal-2.5.1 +set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.45.3.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 diff --git a/PCbuild/python.props b/PCbuild/python.props index 9f8b42d71cd2f3..86fe8531d7df55 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,7 +74,7 @@ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)\mpdecimal-2.5.1\ + $(ExternalsDir)\mpdecimal-4.0.0\ $(ExternalsDir)openssl-3.0.13\ $(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\ $(opensslOutDir)include diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 25d52945c1c330..16fb424b11c6a8 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -105,6 +105,7 @@ _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) @@ -291,6 +292,7 @@ + @@ -607,6 +609,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4b1f9aa6538562..cf9bc0f4bc1c70 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -789,6 +789,9 @@ Include\internal + + Include\internal + Include\internal @@ -1400,6 +1403,9 @@ Python + + Python + Python diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index b72ef5b5c6055c..4aa14ed1fad9eb 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -33,11 +33,15 @@ <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> + <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> + <_SbomOutputs Include="$(PySourcePath)Misc\externals.spdx.json;$(PySourcePath)Misc\sbom.spdx.json"> + json + - @@ -126,7 +130,14 @@ DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> - + + + + + diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 0d154867276c36..80776ffe449393 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -145,8 +145,8 @@ module Python type_ignore = TypeIgnore(int lineno, string tag) - type_param = TypeVar(identifier name, expr? bound) - | ParamSpec(identifier name) - | TypeVarTuple(identifier name) + type_param = TypeVar(identifier name, expr? bound, expr? default_value) + | ParamSpec(identifier name, expr? default_value) + | TypeVarTuple(identifier name, expr? default_value) attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) } diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index c4df2c52c032bc..11d59faeb0d42c 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -979,14 +979,9 @@ def visitModule(self, mod): goto cleanup; } if (field_types == NULL) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s provides _fields but not _field_types. " - "This will become an error in Python 3.15.", - Py_TYPE(self)->tp_name - ) < 0) { - res = -1; - } + // Probably a user-defined subclass of AST that lacks _field_types. + // This will continue to work as it did before 3.13; i.e., attributes + // that are not passed in simply do not exist on the instance. goto cleanup; } remaining_list = PySequence_List(remaining_fields); @@ -997,12 +992,21 @@ def visitModule(self, mod): PyObject *name = PyList_GET_ITEM(remaining_list, i); PyObject *type = PyDict_GetItemWithError(field_types, name); if (!type) { - if (!PyErr_Occurred()) { - PyErr_SetObject(PyExc_KeyError, name); + if (PyErr_Occurred()) { + goto set_remaining_cleanup; + } + else { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Field '%U' is missing from %.400s._field_types. " + "This will become an error in Python 3.15.", + name, Py_TYPE(self)->tp_name + ) < 0) { + goto set_remaining_cleanup; + } } - goto set_remaining_cleanup; } - if (_PyUnion_Check(type)) { + else if (_PyUnion_Check(type)) { // optional field // do nothing, we'll have set a None default on the class } @@ -1026,8 +1030,7 @@ def visitModule(self, mod): "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { - res = -1; - goto cleanup; + goto set_remaining_cleanup; } } } @@ -1443,6 +1446,7 @@ def visitModule(self, mod): static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Parser/parser.c b/Parser/parser.c index 0715e9775a8f06..7bfc17a92e29de 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -30,11 +30,11 @@ static KeywordToken *reserved_keywords[] = { }, (KeywordToken[]) { {"del", 616}, - {"def", 675}, + {"def", 677}, {"for", 672}, {"try", 644}, {"and", 582}, - {"not", 679}, + {"not", 681}, {NULL, -1}, }, (KeywordToken[]) { @@ -51,8 +51,8 @@ static KeywordToken *reserved_keywords[] = { {"raise", 525}, {"yield", 580}, {"break", 508}, - {"async", 674}, - {"class", 677}, + {"async", 676}, + {"class", 679}, {"while", 667}, {"False", 615}, {"await", 590}, @@ -186,431 +186,435 @@ static char *soft_keywords[] = { #define type_param_seq_type 1099 #define type_param_type 1100 #define type_param_bound_type 1101 -#define expressions_type 1102 -#define expression_type 1103 -#define yield_expr_type 1104 -#define star_expressions_type 1105 -#define star_expression_type 1106 -#define star_named_expressions_type 1107 -#define star_named_expression_type 1108 -#define assignment_expression_type 1109 -#define named_expression_type 1110 -#define disjunction_type 1111 -#define conjunction_type 1112 -#define inversion_type 1113 -#define comparison_type 1114 -#define compare_op_bitwise_or_pair_type 1115 -#define eq_bitwise_or_type 1116 -#define noteq_bitwise_or_type 1117 -#define lte_bitwise_or_type 1118 -#define lt_bitwise_or_type 1119 -#define gte_bitwise_or_type 1120 -#define gt_bitwise_or_type 1121 -#define notin_bitwise_or_type 1122 -#define in_bitwise_or_type 1123 -#define isnot_bitwise_or_type 1124 -#define is_bitwise_or_type 1125 -#define bitwise_or_type 1126 // Left-recursive -#define bitwise_xor_type 1127 // Left-recursive -#define bitwise_and_type 1128 // Left-recursive -#define shift_expr_type 1129 // Left-recursive -#define sum_type 1130 // Left-recursive -#define term_type 1131 // Left-recursive -#define factor_type 1132 -#define power_type 1133 -#define await_primary_type 1134 -#define primary_type 1135 // Left-recursive -#define slices_type 1136 -#define slice_type 1137 -#define atom_type 1138 -#define group_type 1139 -#define lambdef_type 1140 -#define lambda_params_type 1141 -#define lambda_parameters_type 1142 -#define lambda_slash_no_default_type 1143 -#define lambda_slash_with_default_type 1144 -#define lambda_star_etc_type 1145 -#define lambda_kwds_type 1146 -#define lambda_param_no_default_type 1147 -#define lambda_param_with_default_type 1148 -#define lambda_param_maybe_default_type 1149 -#define lambda_param_type 1150 -#define fstring_middle_type 1151 -#define fstring_replacement_field_type 1152 -#define fstring_conversion_type 1153 -#define fstring_full_format_spec_type 1154 -#define fstring_format_spec_type 1155 -#define fstring_type 1156 -#define string_type 1157 -#define strings_type 1158 -#define list_type 1159 -#define tuple_type 1160 -#define set_type 1161 -#define dict_type 1162 -#define double_starred_kvpairs_type 1163 -#define double_starred_kvpair_type 1164 -#define kvpair_type 1165 -#define for_if_clauses_type 1166 -#define for_if_clause_type 1167 -#define listcomp_type 1168 -#define setcomp_type 1169 -#define genexp_type 1170 -#define dictcomp_type 1171 -#define arguments_type 1172 -#define args_type 1173 -#define kwargs_type 1174 -#define starred_expression_type 1175 -#define kwarg_or_starred_type 1176 -#define kwarg_or_double_starred_type 1177 -#define star_targets_type 1178 -#define star_targets_list_seq_type 1179 -#define star_targets_tuple_seq_type 1180 -#define star_target_type 1181 -#define target_with_star_atom_type 1182 -#define star_atom_type 1183 -#define single_target_type 1184 -#define single_subscript_attribute_target_type 1185 -#define t_primary_type 1186 // Left-recursive -#define t_lookahead_type 1187 -#define del_targets_type 1188 -#define del_target_type 1189 -#define del_t_atom_type 1190 -#define type_expressions_type 1191 -#define func_type_comment_type 1192 -#define invalid_arguments_type 1193 -#define invalid_kwarg_type 1194 -#define expression_without_invalid_type 1195 -#define invalid_legacy_expression_type 1196 -#define invalid_expression_type 1197 -#define invalid_named_expression_type 1198 -#define invalid_assignment_type 1199 -#define invalid_ann_assign_target_type 1200 -#define invalid_del_stmt_type 1201 -#define invalid_block_type 1202 -#define invalid_comprehension_type 1203 -#define invalid_dict_comprehension_type 1204 -#define invalid_parameters_type 1205 -#define invalid_default_type 1206 -#define invalid_star_etc_type 1207 -#define invalid_kwds_type 1208 -#define invalid_parameters_helper_type 1209 -#define invalid_lambda_parameters_type 1210 -#define invalid_lambda_parameters_helper_type 1211 -#define invalid_lambda_star_etc_type 1212 -#define invalid_lambda_kwds_type 1213 -#define invalid_double_type_comments_type 1214 -#define invalid_with_item_type 1215 -#define invalid_for_target_type 1216 -#define invalid_group_type 1217 -#define invalid_import_type 1218 -#define invalid_import_from_targets_type 1219 -#define invalid_compound_stmt_type 1220 -#define invalid_with_stmt_type 1221 -#define invalid_with_stmt_indent_type 1222 -#define invalid_try_stmt_type 1223 -#define invalid_except_stmt_type 1224 -#define invalid_finally_stmt_type 1225 -#define invalid_except_stmt_indent_type 1226 -#define invalid_except_star_stmt_indent_type 1227 -#define invalid_match_stmt_type 1228 -#define invalid_case_block_type 1229 -#define invalid_as_pattern_type 1230 -#define invalid_class_pattern_type 1231 -#define invalid_class_argument_pattern_type 1232 -#define invalid_if_stmt_type 1233 -#define invalid_elif_stmt_type 1234 -#define invalid_else_stmt_type 1235 -#define invalid_while_stmt_type 1236 -#define invalid_for_stmt_type 1237 -#define invalid_def_raw_type 1238 -#define invalid_class_def_raw_type 1239 -#define invalid_double_starred_kvpairs_type 1240 -#define invalid_kvpair_type 1241 -#define invalid_starred_expression_type 1242 -#define invalid_replacement_field_type 1243 -#define invalid_conversion_character_type 1244 -#define invalid_arithmetic_type 1245 -#define invalid_factor_type 1246 -#define _loop0_1_type 1247 -#define _loop0_2_type 1248 -#define _loop1_3_type 1249 -#define _loop0_5_type 1250 -#define _gather_4_type 1251 -#define _tmp_6_type 1252 -#define _tmp_7_type 1253 -#define _tmp_8_type 1254 -#define _tmp_9_type 1255 -#define _tmp_10_type 1256 -#define _tmp_11_type 1257 -#define _tmp_12_type 1258 -#define _tmp_13_type 1259 -#define _loop1_14_type 1260 -#define _tmp_15_type 1261 -#define _tmp_16_type 1262 -#define _tmp_17_type 1263 -#define _loop0_19_type 1264 -#define _gather_18_type 1265 -#define _loop0_21_type 1266 -#define _gather_20_type 1267 -#define _tmp_22_type 1268 -#define _tmp_23_type 1269 -#define _loop0_24_type 1270 -#define _loop1_25_type 1271 -#define _loop0_27_type 1272 -#define _gather_26_type 1273 -#define _tmp_28_type 1274 -#define _loop0_30_type 1275 -#define _gather_29_type 1276 -#define _tmp_31_type 1277 -#define _loop1_32_type 1278 -#define _tmp_33_type 1279 -#define _tmp_34_type 1280 -#define _tmp_35_type 1281 -#define _loop0_36_type 1282 -#define _loop0_37_type 1283 -#define _loop0_38_type 1284 -#define _loop1_39_type 1285 -#define _loop0_40_type 1286 -#define _loop1_41_type 1287 -#define _loop1_42_type 1288 -#define _loop1_43_type 1289 -#define _loop0_44_type 1290 -#define _loop1_45_type 1291 -#define _loop0_46_type 1292 -#define _loop1_47_type 1293 -#define _loop0_48_type 1294 -#define _loop0_49_type 1295 -#define _loop1_50_type 1296 -#define _loop0_52_type 1297 -#define _gather_51_type 1298 -#define _loop0_54_type 1299 -#define _gather_53_type 1300 -#define _loop0_56_type 1301 -#define _gather_55_type 1302 -#define _loop0_58_type 1303 -#define _gather_57_type 1304 -#define _tmp_59_type 1305 -#define _loop1_60_type 1306 -#define _loop1_61_type 1307 -#define _tmp_62_type 1308 -#define _tmp_63_type 1309 -#define _loop1_64_type 1310 -#define _loop0_66_type 1311 -#define _gather_65_type 1312 -#define _tmp_67_type 1313 -#define _tmp_68_type 1314 -#define _tmp_69_type 1315 -#define _tmp_70_type 1316 -#define _loop0_72_type 1317 -#define _gather_71_type 1318 -#define _loop0_74_type 1319 -#define _gather_73_type 1320 -#define _tmp_75_type 1321 -#define _loop0_77_type 1322 -#define _gather_76_type 1323 -#define _loop0_79_type 1324 -#define _gather_78_type 1325 -#define _loop0_81_type 1326 -#define _gather_80_type 1327 -#define _loop1_82_type 1328 -#define _loop1_83_type 1329 -#define _loop0_85_type 1330 -#define _gather_84_type 1331 -#define _loop1_86_type 1332 -#define _loop1_87_type 1333 -#define _loop1_88_type 1334 -#define _tmp_89_type 1335 -#define _loop0_91_type 1336 -#define _gather_90_type 1337 -#define _tmp_92_type 1338 -#define _tmp_93_type 1339 -#define _tmp_94_type 1340 -#define _tmp_95_type 1341 -#define _tmp_96_type 1342 -#define _tmp_97_type 1343 -#define _loop0_98_type 1344 -#define _loop0_99_type 1345 -#define _loop0_100_type 1346 -#define _loop1_101_type 1347 -#define _loop0_102_type 1348 -#define _loop1_103_type 1349 -#define _loop1_104_type 1350 -#define _loop1_105_type 1351 -#define _loop0_106_type 1352 -#define _loop1_107_type 1353 -#define _loop0_108_type 1354 -#define _loop1_109_type 1355 -#define _loop0_110_type 1356 -#define _loop1_111_type 1357 -#define _loop0_112_type 1358 -#define _loop0_113_type 1359 -#define _loop1_114_type 1360 -#define _tmp_115_type 1361 -#define _loop0_117_type 1362 -#define _gather_116_type 1363 -#define _loop1_118_type 1364 -#define _loop0_119_type 1365 -#define _loop0_120_type 1366 -#define _tmp_121_type 1367 -#define _tmp_122_type 1368 -#define _loop0_124_type 1369 -#define _gather_123_type 1370 -#define _tmp_125_type 1371 -#define _loop0_127_type 1372 -#define _gather_126_type 1373 -#define _loop0_129_type 1374 -#define _gather_128_type 1375 -#define _loop0_131_type 1376 -#define _gather_130_type 1377 -#define _loop0_133_type 1378 -#define _gather_132_type 1379 -#define _loop0_134_type 1380 -#define _loop0_136_type 1381 -#define _gather_135_type 1382 -#define _loop1_137_type 1383 -#define _tmp_138_type 1384 -#define _loop0_140_type 1385 -#define _gather_139_type 1386 -#define _loop0_142_type 1387 -#define _gather_141_type 1388 -#define _loop0_144_type 1389 -#define _gather_143_type 1390 -#define _loop0_146_type 1391 -#define _gather_145_type 1392 -#define _loop0_148_type 1393 -#define _gather_147_type 1394 -#define _tmp_149_type 1395 -#define _tmp_150_type 1396 -#define _loop0_152_type 1397 -#define _gather_151_type 1398 -#define _tmp_153_type 1399 -#define _tmp_154_type 1400 -#define _tmp_155_type 1401 -#define _tmp_156_type 1402 -#define _tmp_157_type 1403 -#define _tmp_158_type 1404 -#define _tmp_159_type 1405 -#define _tmp_160_type 1406 -#define _tmp_161_type 1407 -#define _tmp_162_type 1408 -#define _loop0_163_type 1409 -#define _loop0_164_type 1410 -#define _loop0_165_type 1411 -#define _tmp_166_type 1412 -#define _tmp_167_type 1413 -#define _tmp_168_type 1414 -#define _tmp_169_type 1415 -#define _loop0_170_type 1416 -#define _loop0_171_type 1417 -#define _loop0_172_type 1418 -#define _loop1_173_type 1419 -#define _tmp_174_type 1420 -#define _loop0_175_type 1421 -#define _tmp_176_type 1422 -#define _loop0_177_type 1423 -#define _loop1_178_type 1424 -#define _tmp_179_type 1425 -#define _tmp_180_type 1426 -#define _tmp_181_type 1427 -#define _loop0_182_type 1428 -#define _tmp_183_type 1429 -#define _tmp_184_type 1430 -#define _loop1_185_type 1431 -#define _tmp_186_type 1432 -#define _loop0_187_type 1433 -#define _loop0_188_type 1434 -#define _loop0_189_type 1435 -#define _loop0_191_type 1436 -#define _gather_190_type 1437 -#define _tmp_192_type 1438 -#define _loop0_193_type 1439 -#define _tmp_194_type 1440 -#define _loop0_195_type 1441 -#define _loop1_196_type 1442 -#define _loop1_197_type 1443 -#define _tmp_198_type 1444 -#define _tmp_199_type 1445 -#define _loop0_200_type 1446 -#define _tmp_201_type 1447 -#define _tmp_202_type 1448 -#define _tmp_203_type 1449 -#define _loop0_205_type 1450 -#define _gather_204_type 1451 -#define _loop0_207_type 1452 -#define _gather_206_type 1453 -#define _loop0_209_type 1454 -#define _gather_208_type 1455 -#define _loop0_211_type 1456 -#define _gather_210_type 1457 -#define _loop0_213_type 1458 -#define _gather_212_type 1459 -#define _tmp_214_type 1460 -#define _loop0_215_type 1461 -#define _loop1_216_type 1462 -#define _tmp_217_type 1463 -#define _loop0_218_type 1464 -#define _loop1_219_type 1465 -#define _tmp_220_type 1466 -#define _tmp_221_type 1467 -#define _tmp_222_type 1468 -#define _tmp_223_type 1469 -#define _tmp_224_type 1470 -#define _tmp_225_type 1471 -#define _tmp_226_type 1472 -#define _tmp_227_type 1473 -#define _tmp_228_type 1474 -#define _tmp_229_type 1475 -#define _loop0_231_type 1476 -#define _gather_230_type 1477 -#define _tmp_232_type 1478 -#define _tmp_233_type 1479 -#define _tmp_234_type 1480 -#define _tmp_235_type 1481 -#define _tmp_236_type 1482 -#define _tmp_237_type 1483 -#define _tmp_238_type 1484 -#define _loop0_239_type 1485 -#define _tmp_240_type 1486 -#define _tmp_241_type 1487 -#define _tmp_242_type 1488 -#define _tmp_243_type 1489 -#define _tmp_244_type 1490 -#define _tmp_245_type 1491 -#define _tmp_246_type 1492 -#define _tmp_247_type 1493 -#define _tmp_248_type 1494 -#define _tmp_249_type 1495 -#define _tmp_250_type 1496 -#define _tmp_251_type 1497 -#define _tmp_252_type 1498 -#define _tmp_253_type 1499 -#define _tmp_254_type 1500 -#define _tmp_255_type 1501 -#define _loop0_256_type 1502 -#define _tmp_257_type 1503 -#define _tmp_258_type 1504 -#define _tmp_259_type 1505 -#define _tmp_260_type 1506 -#define _tmp_261_type 1507 -#define _tmp_262_type 1508 -#define _tmp_263_type 1509 -#define _tmp_264_type 1510 -#define _tmp_265_type 1511 -#define _tmp_266_type 1512 -#define _tmp_267_type 1513 -#define _tmp_268_type 1514 -#define _tmp_269_type 1515 -#define _tmp_270_type 1516 -#define _tmp_271_type 1517 -#define _tmp_272_type 1518 -#define _loop0_274_type 1519 -#define _gather_273_type 1520 -#define _tmp_275_type 1521 -#define _tmp_276_type 1522 -#define _tmp_277_type 1523 -#define _tmp_278_type 1524 -#define _tmp_279_type 1525 -#define _tmp_280_type 1526 +#define type_param_default_type 1102 +#define type_param_starred_default_type 1103 +#define expressions_type 1104 +#define expression_type 1105 +#define yield_expr_type 1106 +#define star_expressions_type 1107 +#define star_expression_type 1108 +#define star_named_expressions_type 1109 +#define star_named_expression_type 1110 +#define assignment_expression_type 1111 +#define named_expression_type 1112 +#define disjunction_type 1113 +#define conjunction_type 1114 +#define inversion_type 1115 +#define comparison_type 1116 +#define compare_op_bitwise_or_pair_type 1117 +#define eq_bitwise_or_type 1118 +#define noteq_bitwise_or_type 1119 +#define lte_bitwise_or_type 1120 +#define lt_bitwise_or_type 1121 +#define gte_bitwise_or_type 1122 +#define gt_bitwise_or_type 1123 +#define notin_bitwise_or_type 1124 +#define in_bitwise_or_type 1125 +#define isnot_bitwise_or_type 1126 +#define is_bitwise_or_type 1127 +#define bitwise_or_type 1128 // Left-recursive +#define bitwise_xor_type 1129 // Left-recursive +#define bitwise_and_type 1130 // Left-recursive +#define shift_expr_type 1131 // Left-recursive +#define sum_type 1132 // Left-recursive +#define term_type 1133 // Left-recursive +#define factor_type 1134 +#define power_type 1135 +#define await_primary_type 1136 +#define primary_type 1137 // Left-recursive +#define slices_type 1138 +#define slice_type 1139 +#define atom_type 1140 +#define group_type 1141 +#define lambdef_type 1142 +#define lambda_params_type 1143 +#define lambda_parameters_type 1144 +#define lambda_slash_no_default_type 1145 +#define lambda_slash_with_default_type 1146 +#define lambda_star_etc_type 1147 +#define lambda_kwds_type 1148 +#define lambda_param_no_default_type 1149 +#define lambda_param_with_default_type 1150 +#define lambda_param_maybe_default_type 1151 +#define lambda_param_type 1152 +#define fstring_middle_type 1153 +#define fstring_replacement_field_type 1154 +#define fstring_conversion_type 1155 +#define fstring_full_format_spec_type 1156 +#define fstring_format_spec_type 1157 +#define fstring_type 1158 +#define string_type 1159 +#define strings_type 1160 +#define list_type 1161 +#define tuple_type 1162 +#define set_type 1163 +#define dict_type 1164 +#define double_starred_kvpairs_type 1165 +#define double_starred_kvpair_type 1166 +#define kvpair_type 1167 +#define for_if_clauses_type 1168 +#define for_if_clause_type 1169 +#define listcomp_type 1170 +#define setcomp_type 1171 +#define genexp_type 1172 +#define dictcomp_type 1173 +#define arguments_type 1174 +#define args_type 1175 +#define kwargs_type 1176 +#define starred_expression_type 1177 +#define kwarg_or_starred_type 1178 +#define kwarg_or_double_starred_type 1179 +#define star_targets_type 1180 +#define star_targets_list_seq_type 1181 +#define star_targets_tuple_seq_type 1182 +#define star_target_type 1183 +#define target_with_star_atom_type 1184 +#define star_atom_type 1185 +#define single_target_type 1186 +#define single_subscript_attribute_target_type 1187 +#define t_primary_type 1188 // Left-recursive +#define t_lookahead_type 1189 +#define del_targets_type 1190 +#define del_target_type 1191 +#define del_t_atom_type 1192 +#define type_expressions_type 1193 +#define func_type_comment_type 1194 +#define invalid_arguments_type 1195 +#define invalid_kwarg_type 1196 +#define expression_without_invalid_type 1197 +#define invalid_legacy_expression_type 1198 +#define invalid_expression_type 1199 +#define invalid_named_expression_type 1200 +#define invalid_assignment_type 1201 +#define invalid_ann_assign_target_type 1202 +#define invalid_del_stmt_type 1203 +#define invalid_block_type 1204 +#define invalid_comprehension_type 1205 +#define invalid_dict_comprehension_type 1206 +#define invalid_parameters_type 1207 +#define invalid_default_type 1208 +#define invalid_star_etc_type 1209 +#define invalid_kwds_type 1210 +#define invalid_parameters_helper_type 1211 +#define invalid_lambda_parameters_type 1212 +#define invalid_lambda_parameters_helper_type 1213 +#define invalid_lambda_star_etc_type 1214 +#define invalid_lambda_kwds_type 1215 +#define invalid_double_type_comments_type 1216 +#define invalid_with_item_type 1217 +#define invalid_for_target_type 1218 +#define invalid_group_type 1219 +#define invalid_import_type 1220 +#define invalid_import_from_targets_type 1221 +#define invalid_compound_stmt_type 1222 +#define invalid_with_stmt_type 1223 +#define invalid_with_stmt_indent_type 1224 +#define invalid_try_stmt_type 1225 +#define invalid_except_stmt_type 1226 +#define invalid_finally_stmt_type 1227 +#define invalid_except_stmt_indent_type 1228 +#define invalid_except_star_stmt_indent_type 1229 +#define invalid_match_stmt_type 1230 +#define invalid_case_block_type 1231 +#define invalid_as_pattern_type 1232 +#define invalid_class_pattern_type 1233 +#define invalid_class_argument_pattern_type 1234 +#define invalid_if_stmt_type 1235 +#define invalid_elif_stmt_type 1236 +#define invalid_else_stmt_type 1237 +#define invalid_while_stmt_type 1238 +#define invalid_for_stmt_type 1239 +#define invalid_def_raw_type 1240 +#define invalid_class_def_raw_type 1241 +#define invalid_double_starred_kvpairs_type 1242 +#define invalid_kvpair_type 1243 +#define invalid_starred_expression_type 1244 +#define invalid_replacement_field_type 1245 +#define invalid_conversion_character_type 1246 +#define invalid_arithmetic_type 1247 +#define invalid_factor_type 1248 +#define invalid_type_params_type 1249 +#define _loop0_1_type 1250 +#define _loop0_2_type 1251 +#define _loop1_3_type 1252 +#define _loop0_5_type 1253 +#define _gather_4_type 1254 +#define _tmp_6_type 1255 +#define _tmp_7_type 1256 +#define _tmp_8_type 1257 +#define _tmp_9_type 1258 +#define _tmp_10_type 1259 +#define _tmp_11_type 1260 +#define _tmp_12_type 1261 +#define _tmp_13_type 1262 +#define _loop1_14_type 1263 +#define _tmp_15_type 1264 +#define _tmp_16_type 1265 +#define _tmp_17_type 1266 +#define _loop0_19_type 1267 +#define _gather_18_type 1268 +#define _loop0_21_type 1269 +#define _gather_20_type 1270 +#define _tmp_22_type 1271 +#define _tmp_23_type 1272 +#define _loop0_24_type 1273 +#define _loop1_25_type 1274 +#define _loop0_27_type 1275 +#define _gather_26_type 1276 +#define _tmp_28_type 1277 +#define _loop0_30_type 1278 +#define _gather_29_type 1279 +#define _tmp_31_type 1280 +#define _loop1_32_type 1281 +#define _tmp_33_type 1282 +#define _tmp_34_type 1283 +#define _tmp_35_type 1284 +#define _loop0_36_type 1285 +#define _loop0_37_type 1286 +#define _loop0_38_type 1287 +#define _loop1_39_type 1288 +#define _loop0_40_type 1289 +#define _loop1_41_type 1290 +#define _loop1_42_type 1291 +#define _loop1_43_type 1292 +#define _loop0_44_type 1293 +#define _loop1_45_type 1294 +#define _loop0_46_type 1295 +#define _loop1_47_type 1296 +#define _loop0_48_type 1297 +#define _loop0_49_type 1298 +#define _loop1_50_type 1299 +#define _loop0_52_type 1300 +#define _gather_51_type 1301 +#define _loop0_54_type 1302 +#define _gather_53_type 1303 +#define _loop0_56_type 1304 +#define _gather_55_type 1305 +#define _loop0_58_type 1306 +#define _gather_57_type 1307 +#define _tmp_59_type 1308 +#define _loop1_60_type 1309 +#define _loop1_61_type 1310 +#define _tmp_62_type 1311 +#define _tmp_63_type 1312 +#define _loop1_64_type 1313 +#define _loop0_66_type 1314 +#define _gather_65_type 1315 +#define _tmp_67_type 1316 +#define _tmp_68_type 1317 +#define _tmp_69_type 1318 +#define _tmp_70_type 1319 +#define _loop0_72_type 1320 +#define _gather_71_type 1321 +#define _loop0_74_type 1322 +#define _gather_73_type 1323 +#define _tmp_75_type 1324 +#define _loop0_77_type 1325 +#define _gather_76_type 1326 +#define _loop0_79_type 1327 +#define _gather_78_type 1328 +#define _loop0_81_type 1329 +#define _gather_80_type 1330 +#define _loop1_82_type 1331 +#define _loop1_83_type 1332 +#define _loop0_85_type 1333 +#define _gather_84_type 1334 +#define _loop1_86_type 1335 +#define _loop1_87_type 1336 +#define _loop1_88_type 1337 +#define _tmp_89_type 1338 +#define _loop0_91_type 1339 +#define _gather_90_type 1340 +#define _tmp_92_type 1341 +#define _tmp_93_type 1342 +#define _tmp_94_type 1343 +#define _tmp_95_type 1344 +#define _tmp_96_type 1345 +#define _tmp_97_type 1346 +#define _loop0_98_type 1347 +#define _loop0_99_type 1348 +#define _loop0_100_type 1349 +#define _loop1_101_type 1350 +#define _loop0_102_type 1351 +#define _loop1_103_type 1352 +#define _loop1_104_type 1353 +#define _loop1_105_type 1354 +#define _loop0_106_type 1355 +#define _loop1_107_type 1356 +#define _loop0_108_type 1357 +#define _loop1_109_type 1358 +#define _loop0_110_type 1359 +#define _loop1_111_type 1360 +#define _loop0_112_type 1361 +#define _loop0_113_type 1362 +#define _loop1_114_type 1363 +#define _tmp_115_type 1364 +#define _loop0_117_type 1365 +#define _gather_116_type 1366 +#define _loop1_118_type 1367 +#define _loop0_119_type 1368 +#define _loop0_120_type 1369 +#define _tmp_121_type 1370 +#define _tmp_122_type 1371 +#define _loop0_124_type 1372 +#define _gather_123_type 1373 +#define _tmp_125_type 1374 +#define _loop0_127_type 1375 +#define _gather_126_type 1376 +#define _loop0_129_type 1377 +#define _gather_128_type 1378 +#define _loop0_131_type 1379 +#define _gather_130_type 1380 +#define _loop0_133_type 1381 +#define _gather_132_type 1382 +#define _loop0_134_type 1383 +#define _loop0_136_type 1384 +#define _gather_135_type 1385 +#define _loop1_137_type 1386 +#define _tmp_138_type 1387 +#define _loop0_140_type 1388 +#define _gather_139_type 1389 +#define _loop0_142_type 1390 +#define _gather_141_type 1391 +#define _loop0_144_type 1392 +#define _gather_143_type 1393 +#define _loop0_146_type 1394 +#define _gather_145_type 1395 +#define _loop0_148_type 1396 +#define _gather_147_type 1397 +#define _tmp_149_type 1398 +#define _tmp_150_type 1399 +#define _loop0_152_type 1400 +#define _gather_151_type 1401 +#define _tmp_153_type 1402 +#define _tmp_154_type 1403 +#define _tmp_155_type 1404 +#define _tmp_156_type 1405 +#define _tmp_157_type 1406 +#define _tmp_158_type 1407 +#define _tmp_159_type 1408 +#define _tmp_160_type 1409 +#define _tmp_161_type 1410 +#define _tmp_162_type 1411 +#define _loop0_163_type 1412 +#define _loop0_164_type 1413 +#define _loop0_165_type 1414 +#define _tmp_166_type 1415 +#define _tmp_167_type 1416 +#define _tmp_168_type 1417 +#define _tmp_169_type 1418 +#define _loop0_170_type 1419 +#define _loop0_171_type 1420 +#define _loop0_172_type 1421 +#define _loop1_173_type 1422 +#define _tmp_174_type 1423 +#define _loop0_175_type 1424 +#define _tmp_176_type 1425 +#define _loop0_177_type 1426 +#define _loop1_178_type 1427 +#define _tmp_179_type 1428 +#define _tmp_180_type 1429 +#define _tmp_181_type 1430 +#define _loop0_182_type 1431 +#define _tmp_183_type 1432 +#define _tmp_184_type 1433 +#define _loop1_185_type 1434 +#define _tmp_186_type 1435 +#define _loop0_187_type 1436 +#define _loop0_188_type 1437 +#define _loop0_189_type 1438 +#define _loop0_191_type 1439 +#define _gather_190_type 1440 +#define _tmp_192_type 1441 +#define _loop0_193_type 1442 +#define _tmp_194_type 1443 +#define _loop0_195_type 1444 +#define _loop1_196_type 1445 +#define _loop1_197_type 1446 +#define _tmp_198_type 1447 +#define _tmp_199_type 1448 +#define _loop0_200_type 1449 +#define _tmp_201_type 1450 +#define _tmp_202_type 1451 +#define _tmp_203_type 1452 +#define _loop0_205_type 1453 +#define _gather_204_type 1454 +#define _loop0_207_type 1455 +#define _gather_206_type 1456 +#define _loop0_209_type 1457 +#define _gather_208_type 1458 +#define _loop0_211_type 1459 +#define _gather_210_type 1460 +#define _loop0_213_type 1461 +#define _gather_212_type 1462 +#define _tmp_214_type 1463 +#define _loop0_215_type 1464 +#define _loop1_216_type 1465 +#define _tmp_217_type 1466 +#define _loop0_218_type 1467 +#define _loop1_219_type 1468 +#define _tmp_220_type 1469 +#define _tmp_221_type 1470 +#define _tmp_222_type 1471 +#define _tmp_223_type 1472 +#define _tmp_224_type 1473 +#define _tmp_225_type 1474 +#define _tmp_226_type 1475 +#define _tmp_227_type 1476 +#define _tmp_228_type 1477 +#define _tmp_229_type 1478 +#define _tmp_230_type 1479 +#define _loop0_232_type 1480 +#define _gather_231_type 1481 +#define _tmp_233_type 1482 +#define _tmp_234_type 1483 +#define _tmp_235_type 1484 +#define _tmp_236_type 1485 +#define _tmp_237_type 1486 +#define _tmp_238_type 1487 +#define _tmp_239_type 1488 +#define _loop0_240_type 1489 +#define _tmp_241_type 1490 +#define _tmp_242_type 1491 +#define _tmp_243_type 1492 +#define _tmp_244_type 1493 +#define _tmp_245_type 1494 +#define _tmp_246_type 1495 +#define _tmp_247_type 1496 +#define _tmp_248_type 1497 +#define _tmp_249_type 1498 +#define _tmp_250_type 1499 +#define _tmp_251_type 1500 +#define _tmp_252_type 1501 +#define _tmp_253_type 1502 +#define _tmp_254_type 1503 +#define _tmp_255_type 1504 +#define _tmp_256_type 1505 +#define _loop0_257_type 1506 +#define _tmp_258_type 1507 +#define _tmp_259_type 1508 +#define _tmp_260_type 1509 +#define _tmp_261_type 1510 +#define _tmp_262_type 1511 +#define _tmp_263_type 1512 +#define _tmp_264_type 1513 +#define _tmp_265_type 1514 +#define _tmp_266_type 1515 +#define _tmp_267_type 1516 +#define _tmp_268_type 1517 +#define _tmp_269_type 1518 +#define _tmp_270_type 1519 +#define _tmp_271_type 1520 +#define _tmp_272_type 1521 +#define _tmp_273_type 1522 +#define _loop0_275_type 1523 +#define _gather_274_type 1524 +#define _tmp_276_type 1525 +#define _tmp_277_type 1526 +#define _tmp_278_type 1527 +#define _tmp_279_type 1528 +#define _tmp_280_type 1529 +#define _tmp_281_type 1530 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -714,6 +718,8 @@ static asdl_type_param_seq* type_params_rule(Parser *p); static asdl_type_param_seq* type_param_seq_rule(Parser *p); static type_param_ty type_param_rule(Parser *p); static expr_ty type_param_bound_rule(Parser *p); +static expr_ty type_param_default_rule(Parser *p); +static expr_ty type_param_starred_default_rule(Parser *p); static expr_ty expressions_rule(Parser *p); static expr_ty expression_rule(Parser *p); static expr_ty yield_expr_rule(Parser *p); @@ -859,6 +865,7 @@ static void *invalid_replacement_field_rule(Parser *p); static void *invalid_conversion_character_rule(Parser *p); static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); +static void *invalid_type_params_rule(Parser *p); static asdl_seq *_loop0_1_rule(Parser *p); static asdl_seq *_loop0_2_rule(Parser *p); static asdl_seq *_loop1_3_rule(Parser *p); @@ -1088,17 +1095,17 @@ static void *_tmp_226_rule(Parser *p); static void *_tmp_227_rule(Parser *p); static void *_tmp_228_rule(Parser *p); static void *_tmp_229_rule(Parser *p); -static asdl_seq *_loop0_231_rule(Parser *p); -static asdl_seq *_gather_230_rule(Parser *p); -static void *_tmp_232_rule(Parser *p); +static void *_tmp_230_rule(Parser *p); +static asdl_seq *_loop0_232_rule(Parser *p); +static asdl_seq *_gather_231_rule(Parser *p); static void *_tmp_233_rule(Parser *p); static void *_tmp_234_rule(Parser *p); static void *_tmp_235_rule(Parser *p); static void *_tmp_236_rule(Parser *p); static void *_tmp_237_rule(Parser *p); static void *_tmp_238_rule(Parser *p); -static asdl_seq *_loop0_239_rule(Parser *p); -static void *_tmp_240_rule(Parser *p); +static void *_tmp_239_rule(Parser *p); +static asdl_seq *_loop0_240_rule(Parser *p); static void *_tmp_241_rule(Parser *p); static void *_tmp_242_rule(Parser *p); static void *_tmp_243_rule(Parser *p); @@ -1114,8 +1121,8 @@ static void *_tmp_252_rule(Parser *p); static void *_tmp_253_rule(Parser *p); static void *_tmp_254_rule(Parser *p); static void *_tmp_255_rule(Parser *p); -static asdl_seq *_loop0_256_rule(Parser *p); -static void *_tmp_257_rule(Parser *p); +static void *_tmp_256_rule(Parser *p); +static asdl_seq *_loop0_257_rule(Parser *p); static void *_tmp_258_rule(Parser *p); static void *_tmp_259_rule(Parser *p); static void *_tmp_260_rule(Parser *p); @@ -1131,14 +1138,15 @@ static void *_tmp_269_rule(Parser *p); static void *_tmp_270_rule(Parser *p); static void *_tmp_271_rule(Parser *p); static void *_tmp_272_rule(Parser *p); -static asdl_seq *_loop0_274_rule(Parser *p); -static asdl_seq *_gather_273_rule(Parser *p); -static void *_tmp_275_rule(Parser *p); +static void *_tmp_273_rule(Parser *p); +static asdl_seq *_loop0_275_rule(Parser *p); +static asdl_seq *_gather_274_rule(Parser *p); static void *_tmp_276_rule(Parser *p); static void *_tmp_277_rule(Parser *p); static void *_tmp_278_rule(Parser *p); static void *_tmp_279_rule(Parser *p); static void *_tmp_280_rule(Parser *p); +static void *_tmp_281_rule(Parser *p); // file: statements? $ @@ -4362,7 +4370,7 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='class' + (_keyword = _PyPegen_expect_token(p, 679)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && @@ -4470,8 +4478,8 @@ function_def_rule(Parser *p) // function_def_raw: // | invalid_def_raw -// | 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block -// | 'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block +// | 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block +// | 'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block static stmt_ty function_def_raw_rule(Parser *p) { @@ -4512,12 +4520,12 @@ function_def_raw_rule(Parser *p) D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_def_raw")); } - { // 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + { // 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token * _keyword; Token * _literal; Token * _literal_1; @@ -4529,13 +4537,13 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 675)) // token='def' + (_keyword = _PyPegen_expect_token(p, 677)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + (_literal = _PyPegen_expect_token(p, 7)) // token='(' && (params = params_rule(p), !p->error_indicator) // params? && @@ -4543,14 +4551,14 @@ function_def_raw_rule(Parser *p) && (a = _tmp_34_rule(p), !p->error_indicator) // ['->' expression] && - (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -4570,14 +4578,14 @@ function_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); } - { // 'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + { // 'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token * _keyword; Token * _keyword_1; Token * _literal; @@ -4590,15 +4598,15 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 675)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 677)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + (_literal = _PyPegen_expect_token(p, 7)) // token='(' && (params = params_rule(p), !p->error_indicator) // params? && @@ -4606,14 +4614,14 @@ function_def_raw_rule(Parser *p) && (a = _tmp_35_rule(p), !p->error_indicator) // ['->' expression] && - (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -4633,7 +4641,7 @@ function_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); } _res = NULL; done: @@ -6437,7 +6445,7 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='for' && @@ -6672,7 +6680,7 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 635)) // token='with' && @@ -6724,7 +6732,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 635)) // token='with' && @@ -10549,7 +10557,7 @@ type_alias_rule(Parser *p) return _res; } -// type_params: '[' type_param_seq ']' +// type_params: invalid_type_params | '[' type_param_seq ']' static asdl_type_param_seq* type_params_rule(Parser *p) { @@ -10562,6 +10570,25 @@ type_params_rule(Parser *p) } asdl_type_param_seq* _res = NULL; int _mark = p->mark; + if (p->call_invalid_rules) { // invalid_type_params + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_type_params")); + void *invalid_type_params_var; + if ( + (invalid_type_params_var = invalid_type_params_rule(p)) // invalid_type_params + ) + { + D(fprintf(stderr, "%*c+ type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_type_params")); + _res = invalid_type_params_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_params[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_type_params")); + } { // '[' type_param_seq ']' if (p->error_indicator) { p->level--; @@ -10646,11 +10673,11 @@ type_param_seq_rule(Parser *p) } // type_param: -// | NAME type_param_bound? +// | NAME type_param_bound? type_param_default? // | '*' NAME ':' expression -// | '*' NAME +// | '*' NAME type_param_starred_default? // | '**' NAME ':' expression -// | '**' NAME +// | '**' NAME type_param_default? static type_param_ty type_param_rule(Parser *p) { @@ -10676,21 +10703,24 @@ type_param_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // NAME type_param_bound? + { // NAME type_param_bound? type_param_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound?")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound? type_param_default?")); expr_ty a; void *b; + void *c; if ( (a = _PyPegen_name_token(p)) // NAME && (b = type_param_bound_rule(p), !p->error_indicator) // type_param_bound? + && + (c = type_param_default_rule(p), !p->error_indicator) // type_param_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound?")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound? type_param_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10700,7 +10730,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_TypeVar ( a -> v . Name . id , b , EXTRA ); + _res = _PyAST_TypeVar ( a -> v . Name . id , b , c , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10710,7 +10740,7 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME type_param_bound?")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME type_param_bound? type_param_default?")); } { // '*' NAME ':' expression if (p->error_indicator) { @@ -10745,21 +10775,24 @@ type_param_rule(Parser *p) D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME ':' expression")); } - { // '*' NAME + { // '*' NAME type_param_starred_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME type_param_starred_default?")); Token * _literal; expr_ty a; + void *b; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (a = _PyPegen_name_token(p)) // NAME + && + (b = type_param_starred_default_rule(p), !p->error_indicator) // type_param_starred_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME type_param_starred_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10769,7 +10802,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_TypeVarTuple ( a -> v . Name . id , EXTRA ); + _res = _PyAST_TypeVarTuple ( a -> v . Name . id , b , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10779,7 +10812,7 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME type_param_starred_default?")); } { // '**' NAME ':' expression if (p->error_indicator) { @@ -10814,21 +10847,24 @@ type_param_rule(Parser *p) D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME ':' expression")); } - { // '**' NAME + { // '**' NAME type_param_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME type_param_default?")); Token * _literal; expr_ty a; + void *b; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' && (a = _PyPegen_name_token(p)) // NAME + && + (b = type_param_default_rule(p), !p->error_indicator) // type_param_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME type_param_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10838,7 +10874,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_ParamSpec ( a -> v . Name . id , EXTRA ); + _res = _PyAST_ParamSpec ( a -> v . Name . id , b , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10848,7 +10884,7 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME type_param_default?")); } _res = NULL; done: @@ -10903,6 +10939,98 @@ type_param_bound_rule(Parser *p) return _res; } +// type_param_default: '=' expression +static expr_ty +type_param_default_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + { // '=' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_param_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' expression")); + Token * _literal; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (e = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ type_param_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' expression")); + _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_param_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// type_param_starred_default: '=' star_expression +static expr_ty +type_param_starred_default_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + { // '=' star_expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_param_starred_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' star_expression")); + Token * _literal; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (e = star_expression_rule(p)) // star_expression + ) + { + D(fprintf(stderr, "%*c+ type_param_starred_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' star_expression")); + _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_param_starred_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' star_expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // expressions: expression ((',' expression))+ ','? | expression ',' | expression static expr_ty expressions_rule(Parser *p) @@ -11995,7 +12123,7 @@ inversion_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 679)) // token='not' + (_keyword = _PyPegen_expect_token(p, 681)) // token='not' && (a = inversion_rule(p)) // inversion ) @@ -12649,7 +12777,7 @@ notin_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 679)) // token='not' + (_keyword = _PyPegen_expect_token(p, 681)) // token='not' && (_keyword_1 = _PyPegen_expect_token(p, 673)) // token='in' && @@ -12746,7 +12874,7 @@ isnot_bitwise_or_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 589)) // token='is' && - (_keyword_1 = _PyPegen_expect_token(p, 679)) // token='not' + (_keyword_1 = _PyPegen_expect_token(p, 681)) // token='not' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -16917,7 +17045,7 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='for' && @@ -17003,7 +17131,7 @@ for_if_clause_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_tmp_121_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 672)) // token='for' && @@ -22456,7 +22584,7 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 672)) // token='for' && @@ -22819,7 +22947,7 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 635)) // token='with' && @@ -22857,7 +22985,7 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 635)) // token='with' && @@ -22919,7 +23047,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (a = _PyPegen_expect_token(p, 635)) // token='with' && @@ -22962,7 +23090,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (a = _PyPegen_expect_token(p, 635)) // token='with' && @@ -24248,7 +24376,7 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 672)) // token='for' && @@ -24289,7 +24417,7 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (a = _PyPegen_expect_token(p, 672)) // token='for' && @@ -24327,6 +24455,7 @@ invalid_for_stmt_rule(Parser *p) // invalid_def_raw: // | 'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT +// | 'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block static void * invalid_def_raw_rule(Parser *p) { @@ -24360,9 +24489,9 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 675)) // token='def' + (a = _PyPegen_expect_token(p, 677)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24396,6 +24525,60 @@ invalid_def_raw_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_def_raw[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); } + { // 'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + Token * _keyword; + Token * _literal; + Token * _literal_1; + Token * _literal_2; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + void *_opt_var_1; + UNUSED(_opt_var_1); // Silence compiler warnings + void *_opt_var_2; + UNUSED(_opt_var_2); // Silence compiler warnings + void *_opt_var_3; + UNUSED(_opt_var_3); // Silence compiler warnings + void *_opt_var_4; + UNUSED(_opt_var_4); // Silence compiler warnings + asdl_stmt_seq* block_var; + expr_ty name_var; + if ( + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + && + (_keyword = _PyPegen_expect_token(p, 677)) // token='def' + && + (name_var = _PyPegen_name_token(p)) // NAME + && + (_opt_var_1 = type_params_rule(p), !p->error_indicator) // type_params? + && + (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + && + (_opt_var_2 = params_rule(p), !p->error_indicator) // params? + && + (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' + && + (_opt_var_3 = _tmp_228_rule(p), !p->error_indicator) // ['->' expression] + && + (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + && + (_opt_var_4 = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? + && + (block_var = block_rule(p)) // block + ) + { + D(fprintf(stderr, "%*c+ invalid_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + _res = _PyPegen_dummy_name(p, _opt_var, _keyword, name_var, _opt_var_1, _literal, _opt_var_2, _literal_1, _opt_var_3, _literal_2, _opt_var_4, block_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_def_raw[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + } _res = NULL; done: p->level--; @@ -24431,13 +24614,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='class' + (_keyword = _PyPegen_expect_token(p, 679)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_229_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24470,13 +24653,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='class' + (a = _PyPegen_expect_token(p, 679)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_229_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_230_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24526,11 +24709,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_230_var; + asdl_seq * _gather_231_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_230_var = _gather_230_rule(p)) // ','.double_starred_kvpair+ + (_gather_231_var = _gather_231_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24538,7 +24721,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_230_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_231_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24591,7 +24774,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_232_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_233_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24701,7 +24884,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_233_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_234_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24946,7 +25129,7 @@ invalid_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_234_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_235_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -24978,7 +25161,7 @@ invalid_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_235_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_236_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -25042,9 +25225,9 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_236_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_237_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_237_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_238_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -25068,7 +25251,7 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_239_var; + asdl_seq * _loop0_240_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; @@ -25081,11 +25264,11 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_238_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_239_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_239_var = _loop0_239_rule(p)) // fstring_format_spec* + (_loop0_240_var = _loop0_240_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25122,7 +25305,7 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_240_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_241_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25169,7 +25352,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_241_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_242_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -25236,16 +25419,16 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_242_var; + void *_tmp_243_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_242_var = _tmp_242_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_243_var = _tmp_243_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && - (a = _PyPegen_expect_token(p, 679)) // token='not' + (a = _PyPegen_expect_token(p, 681)) // token='not' && (b = inversion_rule(p)) // inversion ) @@ -25288,13 +25471,13 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_243_var; + void *_tmp_244_var; Token * a; expr_ty b; if ( - (_tmp_243_var = _tmp_243_rule(p)) // '+' | '-' | '~' + (_tmp_244_var = _tmp_244_rule(p)) // '+' | '-' | '~' && - (a = _PyPegen_expect_token(p, 679)) // token='not' + (a = _PyPegen_expect_token(p, 681)) // token='not' && (b = factor_rule(p)) // factor ) @@ -25318,6 +25501,52 @@ invalid_factor_rule(Parser *p) return _res; } +// invalid_type_params: '[' ']' +static void * +invalid_type_params_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '[' ']' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'[' ']'")); + Token * _literal; + Token * token; + if ( + (_literal = _PyPegen_expect_token(p, 9)) // token='[' + && + (token = _PyPegen_expect_token(p, 10)) // token=']' + ) + { + D(fprintf(stderr, "%*c+ invalid_type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'[' ']'")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Type parameter list cannot be empty" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_type_params[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'[' ']'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // _loop0_1: NEWLINE static asdl_seq * _loop0_1_rule(Parser *p) @@ -25719,7 +25948,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 675)) // token='def' + (_keyword = _PyPegen_expect_token(p, 677)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -25757,7 +25986,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25795,7 +26024,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='class' + (_keyword = _PyPegen_expect_token(p, 679)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -25871,7 +26100,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25928,7 +26157,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -26133,12 +26362,12 @@ _loop1_14_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_244_var; + void *_tmp_245_var; while ( - (_tmp_244_var = _tmp_244_rule(p)) // star_targets '=' + (_tmp_245_var = _tmp_245_rule(p)) // star_targets '=' ) { - _res = _tmp_244_var; + _res = _tmp_245_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26702,73 +26931,6 @@ _loop0_24_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_245_var; - while ( - (_tmp_245_var = _tmp_245_rule(p)) // '.' | '...' - ) - { - _res = _tmp_245_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_24[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_25: ('.' | '...') -static asdl_seq * -_loop1_25_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ('.' | '...') - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); void *_tmp_246_var; while ( (_tmp_246_var = _tmp_246_rule(p)) // '.' | '...' @@ -26791,6 +26953,73 @@ _loop1_25_rule(Parser *p) _mark = p->mark; } p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_24[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_25: ('.' | '...') +static asdl_seq * +_loop1_25_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ('.' | '...') + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); + void *_tmp_247_var; + while ( + (_tmp_247_var = _tmp_247_rule(p)) // '.' | '...' + ) + { + _res = _tmp_247_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; D(fprintf(stderr, "%*c%s _loop1_25[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); } @@ -27167,12 +27396,12 @@ _loop1_32_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_247_var; + void *_tmp_248_var; while ( - (_tmp_247_var = _tmp_247_rule(p)) // '@' named_expression NEWLINE + (_tmp_248_var = _tmp_248_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_247_var; + _res = _tmp_248_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30297,12 +30526,12 @@ _loop1_82_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_248_var; + void *_tmp_249_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // ',' expression + (_tmp_249_var = _tmp_249_rule(p)) // ',' expression ) { - _res = _tmp_248_var; + _res = _tmp_249_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30369,12 +30598,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_249_var; + void *_tmp_250_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // ',' star_expression + (_tmp_250_var = _tmp_250_rule(p)) // ',' star_expression ) { - _res = _tmp_249_var; + _res = _tmp_250_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30558,12 +30787,12 @@ _loop1_86_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_250_var; + void *_tmp_251_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // 'or' conjunction + (_tmp_251_var = _tmp_251_rule(p)) // 'or' conjunction ) { - _res = _tmp_250_var; + _res = _tmp_251_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30630,12 +30859,12 @@ _loop1_87_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_251_var; + void *_tmp_252_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // 'and' inversion + (_tmp_252_var = _tmp_252_rule(p)) // 'and' inversion ) { - _res = _tmp_251_var; + _res = _tmp_252_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30822,7 +31051,7 @@ _loop0_91_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_252_rule(p)) // slice | starred_expression + (elem = _tmp_253_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30887,7 +31116,7 @@ _gather_90_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_252_rule(p)) // slice | starred_expression + (elem = _tmp_253_rule(p)) // slice | starred_expression && (seq = _loop0_91_rule(p)) // _loop0_91 ) @@ -32429,12 +32658,12 @@ _loop1_114_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_253_var; + void *_tmp_254_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // fstring | string + (_tmp_254_var = _tmp_254_rule(p)) // fstring | string ) { - _res = _tmp_253_var; + _res = _tmp_254_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32739,12 +32968,12 @@ _loop0_119_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_254_var; + void *_tmp_255_var; while ( - (_tmp_254_var = _tmp_254_rule(p)) // 'if' disjunction + (_tmp_255_var = _tmp_255_rule(p)) // 'if' disjunction ) { - _res = _tmp_254_var; + _res = _tmp_255_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32806,12 +33035,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_255_var; + void *_tmp_256_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // 'if' disjunction + (_tmp_256_var = _tmp_256_rule(p)) // 'if' disjunction ) { - _res = _tmp_255_var; + _res = _tmp_256_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32864,20 +33093,20 @@ _tmp_121_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_256_var; + asdl_seq * _loop0_257_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty bitwise_or_var; if ( (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - (_loop0_256_var = _loop0_256_rule(p)) // ((',' bitwise_or))* + (_loop0_257_var = _loop0_257_rule(p)) // ((',' bitwise_or))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_256_var, _opt_var); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_257_var, _opt_var); goto done; } p->mark = _mark; @@ -32982,7 +33211,7 @@ _loop0_124_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -33048,7 +33277,7 @@ _gather_123_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_124_rule(p)) // _loop0_124 ) @@ -33609,12 +33838,12 @@ _loop0_134_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_258_var; + void *_tmp_259_var; while ( - (_tmp_258_var = _tmp_258_rule(p)) // ',' star_target + (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target ) { - _res = _tmp_258_var; + _res = _tmp_259_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33793,12 +34022,12 @@ _loop1_137_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_259_var; + void *_tmp_260_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target + (_tmp_260_var = _tmp_260_rule(p)) // ',' star_target ) { - _res = _tmp_259_var; + _res = _tmp_260_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34524,13 +34753,13 @@ _tmp_150_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_260_var; + void *_tmp_261_var; if ( - (_tmp_260_var = _tmp_260_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_261_var = _tmp_261_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_260_var; + _res = _tmp_261_var; goto done; } p->mark = _mark; @@ -34595,7 +34824,7 @@ _loop0_152_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_261_rule(p)) // starred_expression !'=' + (elem = _tmp_262_rule(p)) // starred_expression !'=' ) { _res = elem; @@ -34660,7 +34889,7 @@ _gather_151_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_261_rule(p)) // starred_expression !'=' + (elem = _tmp_262_rule(p)) // starred_expression !'=' && (seq = _loop0_152_rule(p)) // _loop0_152 ) @@ -35413,12 +35642,12 @@ _loop0_164_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_262_var; + void *_tmp_263_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // star_targets '=' + (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' ) { - _res = _tmp_262_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35480,12 +35709,12 @@ _loop0_165_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_263_var; + void *_tmp_264_var; while ( - (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' + (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' ) { - _res = _tmp_263_var; + _res = _tmp_264_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -36455,15 +36684,15 @@ _tmp_180_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_264_var; + void *_tmp_265_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_264_var = _tmp_264_rule(p)) // ')' | '**' + (_tmp_265_var = _tmp_265_rule(p)) // ')' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_264_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); goto done; } p->mark = _mark; @@ -37611,15 +37840,15 @@ _tmp_198_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_265_var; + void *_tmp_266_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_265_var = _tmp_265_rule(p)) // ':' | '**' + (_tmp_266_var = _tmp_266_rule(p)) // ':' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); goto done; } p->mark = _mark; @@ -38115,7 +38344,7 @@ _loop0_207_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_266_rule(p)) // expression ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38180,7 +38409,7 @@ _gather_206_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_266_rule(p)) // expression ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] && (seq = _loop0_207_rule(p)) // _loop0_207 ) @@ -38232,7 +38461,7 @@ _loop0_209_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_267_rule(p)) // expressions ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38297,7 +38526,7 @@ _gather_208_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_267_rule(p)) // expressions ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] && (seq = _loop0_209_rule(p)) // _loop0_209 ) @@ -38349,7 +38578,7 @@ _loop0_211_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38414,7 +38643,7 @@ _gather_210_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] && (seq = _loop0_211_rule(p)) // _loop0_211 ) @@ -38466,7 +38695,7 @@ _loop0_213_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38531,7 +38760,7 @@ _gather_212_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] && (seq = _loop0_213_rule(p)) // _loop0_213 ) @@ -38951,7 +39180,7 @@ _tmp_220_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_270_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_271_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); @@ -39271,9 +39500,50 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: '(' arguments? ')' +// _tmp_228: '->' expression static void * _tmp_228_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '->' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + Token * _literal; + expr_ty expression_var; + if ( + (_literal = _PyPegen_expect_token(p, 51)) // token='->' + && + (expression_var = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + _res = _PyPegen_dummy_name(p, _literal, expression_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_229: '(' arguments? ')' +static void * +_tmp_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39289,7 +39559,7 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39302,12 +39572,12 @@ _tmp_228_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39316,9 +39586,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _tmp_229: '(' arguments? ')' +// _tmp_230: '(' arguments? ')' static void * -_tmp_229_rule(Parser *p) +_tmp_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39334,7 +39604,7 @@ _tmp_229_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39347,12 +39617,12 @@ _tmp_229_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39361,9 +39631,9 @@ _tmp_229_rule(Parser *p) return _res; } -// _loop0_231: ',' double_starred_kvpair +// _loop0_232: ',' double_starred_kvpair static asdl_seq * -_loop0_231_rule(Parser *p) +_loop0_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39388,7 +39658,7 @@ _loop0_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -39420,7 +39690,7 @@ _loop0_231_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39437,9 +39707,9 @@ _loop0_231_rule(Parser *p) return _seq; } -// _gather_230: double_starred_kvpair _loop0_231 +// _gather_231: double_starred_kvpair _loop0_232 static asdl_seq * -_gather_230_rule(Parser *p) +_gather_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39450,27 +39720,27 @@ _gather_230_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_231 + { // double_starred_kvpair _loop0_232 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c> _gather_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_232")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_231_rule(p)) // _loop0_231 + (seq = _loop0_232_rule(p)) // _loop0_232 ) { - D(fprintf(stderr, "%*c+ _gather_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c+ _gather_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_232")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_230[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c%s _gather_231[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_232")); } _res = NULL; done: @@ -39478,9 +39748,9 @@ _gather_230_rule(Parser *p) return _res; } -// _tmp_232: '}' | ',' +// _tmp_233: '}' | ',' static void * -_tmp_232_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39496,18 +39766,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39515,18 +39785,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39535,9 +39805,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: '}' | ',' +// _tmp_234: '}' | ',' static void * -_tmp_233_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39553,18 +39823,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39572,18 +39842,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39592,9 +39862,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: '=' | '!' | ':' | '}' +// _tmp_235: '=' | '!' | ':' | '}' static void * -_tmp_234_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39610,18 +39880,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -39629,18 +39899,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39648,18 +39918,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39667,18 +39937,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39687,9 +39957,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: '!' | ':' | '}' +// _tmp_236: '!' | ':' | '}' static void * -_tmp_235_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39705,18 +39975,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39724,18 +39994,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39743,18 +40013,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39763,9 +40033,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: '!' NAME +// _tmp_237: '!' NAME static void * -_tmp_236_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39781,7 +40051,7 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39790,12 +40060,12 @@ _tmp_236_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39804,9 +40074,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: ':' | '}' +// _tmp_238: ':' | '}' static void * -_tmp_237_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39822,18 +40092,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39841,18 +40111,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39861,9 +40131,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: '!' NAME +// _tmp_239: '!' NAME static void * -_tmp_238_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39879,7 +40149,7 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39888,12 +40158,12 @@ _tmp_238_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39902,9 +40172,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _loop0_239: fstring_format_spec +// _loop0_240: fstring_format_spec static asdl_seq * -_loop0_239_rule(Parser *p) +_loop0_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39929,7 +40199,7 @@ _loop0_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -39952,7 +40222,7 @@ _loop0_239_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39969,9 +40239,9 @@ _loop0_239_rule(Parser *p) return _seq; } -// _tmp_240: '!' NAME +// _tmp_241: '!' NAME static void * -_tmp_240_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39987,7 +40257,7 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39996,12 +40266,12 @@ _tmp_240_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40010,9 +40280,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: ':' | '}' +// _tmp_242: ':' | '}' static void * -_tmp_241_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40028,18 +40298,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40047,18 +40317,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40067,9 +40337,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _tmp_242: '+' | '-' | '*' | '/' | '%' | '//' | '@' +// _tmp_243: '+' | '-' | '*' | '/' | '%' | '//' | '@' static void * -_tmp_242_rule(Parser *p) +_tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40085,18 +40355,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -40104,18 +40374,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '*' @@ -40123,18 +40393,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '/' @@ -40142,18 +40412,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } { // '%' @@ -40161,18 +40431,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 24)) // token='%' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } { // '//' @@ -40180,18 +40450,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 47)) // token='//' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } { // '@' @@ -40199,18 +40469,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; @@ -40219,9 +40489,9 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: '+' | '-' | '~' +// _tmp_244: '+' | '-' | '~' static void * -_tmp_243_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40237,18 +40507,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -40256,18 +40526,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '~' @@ -40275,18 +40545,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 31)) // token='~' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; @@ -40295,9 +40565,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _tmp_244: star_targets '=' +// _tmp_245: star_targets '=' static void * -_tmp_244_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40313,7 +40583,7 @@ _tmp_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40322,7 +40592,7 @@ _tmp_244_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40332,7 +40602,7 @@ _tmp_244_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40341,9 +40611,9 @@ _tmp_244_rule(Parser *p) return _res; } -// _tmp_245: '.' | '...' +// _tmp_246: '.' | '...' static void * -_tmp_245_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40359,18 +40629,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40378,18 +40648,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40398,9 +40668,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: '.' | '...' +// _tmp_247: '.' | '...' static void * -_tmp_246_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40416,18 +40686,18 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40435,18 +40705,18 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40455,9 +40725,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: '@' named_expression NEWLINE +// _tmp_248: '@' named_expression NEWLINE static void * -_tmp_247_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40473,7 +40743,7 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40485,7 +40755,7 @@ _tmp_247_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40495,7 +40765,7 @@ _tmp_247_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40504,9 +40774,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: ',' expression +// _tmp_249: ',' expression static void * -_tmp_248_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40522,7 +40792,7 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40531,7 +40801,7 @@ _tmp_248_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40541,7 +40811,7 @@ _tmp_248_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40550,9 +40820,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: ',' star_expression +// _tmp_250: ',' star_expression static void * -_tmp_249_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40568,7 +40838,7 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40577,7 +40847,7 @@ _tmp_249_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40587,7 +40857,7 @@ _tmp_249_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40596,9 +40866,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: 'or' conjunction +// _tmp_251: 'or' conjunction static void * -_tmp_250_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40614,7 +40884,7 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40623,7 +40893,7 @@ _tmp_250_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40633,7 +40903,7 @@ _tmp_250_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40642,9 +40912,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: 'and' inversion +// _tmp_252: 'and' inversion static void * -_tmp_251_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40660,7 +40930,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -40669,7 +40939,7 @@ _tmp_251_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40679,7 +40949,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -40688,9 +40958,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: slice | starred_expression +// _tmp_253: slice | starred_expression static void * -_tmp_252_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40706,18 +40976,18 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -40725,18 +40995,18 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -40745,9 +41015,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: fstring | string +// _tmp_254: fstring | string static void * -_tmp_253_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40763,18 +41033,18 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -40782,18 +41052,18 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -40802,9 +41072,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: 'if' disjunction +// _tmp_255: 'if' disjunction static void * -_tmp_254_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40820,7 +41090,7 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40829,7 +41099,7 @@ _tmp_254_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40839,7 +41109,7 @@ _tmp_254_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40848,9 +41118,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: 'if' disjunction +// _tmp_256: 'if' disjunction static void * -_tmp_255_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40866,7 +41136,7 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40875,7 +41145,7 @@ _tmp_255_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40885,7 +41155,7 @@ _tmp_255_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40894,9 +41164,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _loop0_256: (',' bitwise_or) +// _loop0_257: (',' bitwise_or) static asdl_seq * -_loop0_256_rule(Parser *p) +_loop0_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40921,13 +41191,13 @@ _loop0_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_271_var; + D(fprintf(stderr, "%*c> _loop0_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_272_var; while ( - (_tmp_271_var = _tmp_271_rule(p)) // ',' bitwise_or + (_tmp_272_var = _tmp_272_rule(p)) // ',' bitwise_or ) { - _res = _tmp_271_var; + _res = _tmp_272_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -40944,7 +41214,7 @@ _loop0_256_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -40961,9 +41231,9 @@ _loop0_256_rule(Parser *p) return _seq; } -// _tmp_257: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_258: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_257_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40979,18 +41249,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -40998,20 +41268,20 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_272_var; + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_273_var; if ( - (_tmp_272_var = _tmp_272_rule(p)) // assignment_expression | expression !':=' + (_tmp_273_var = _tmp_273_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_272_var; + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_273_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -41020,9 +41290,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: ',' star_target +// _tmp_259: ',' star_target static void * -_tmp_258_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41038,7 +41308,7 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41047,7 +41317,7 @@ _tmp_258_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41057,7 +41327,7 @@ _tmp_258_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41066,9 +41336,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: ',' star_target +// _tmp_260: ',' star_target static void * -_tmp_259_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41084,7 +41354,7 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41093,7 +41363,7 @@ _tmp_259_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41103,7 +41373,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41112,10 +41382,10 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: +// _tmp_261: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_260_rule(Parser *p) +_tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41131,24 +41401,24 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_273_var; + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_274_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_273_var = _gather_273_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_274_var = _gather_274_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_273_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_274_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -41157,9 +41427,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: starred_expression !'=' +// _tmp_262: starred_expression !'=' static void * -_tmp_261_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41175,7 +41445,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression @@ -41183,12 +41453,12 @@ _tmp_261_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); } _res = NULL; @@ -41197,9 +41467,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: star_targets '=' +// _tmp_263: star_targets '=' static void * -_tmp_262_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41215,7 +41485,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41224,12 +41494,12 @@ _tmp_262_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41238,9 +41508,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: star_targets '=' +// _tmp_264: star_targets '=' static void * -_tmp_263_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41256,7 +41526,7 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41265,12 +41535,12 @@ _tmp_263_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41279,9 +41549,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: ')' | '**' +// _tmp_265: ')' | '**' static void * -_tmp_264_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41297,18 +41567,18 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -41316,18 +41586,18 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41336,9 +41606,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: ':' | '**' +// _tmp_266: ':' | '**' static void * -_tmp_265_rule(Parser *p) +_tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41354,18 +41624,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -41373,18 +41643,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41393,9 +41663,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: expression ['as' star_target] +// _tmp_267: expression ['as' star_target] static void * -_tmp_266_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41411,22 +41681,22 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_275_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41435,9 +41705,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: expressions ['as' star_target] +// _tmp_268: expressions ['as' star_target] static void * -_tmp_267_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41453,22 +41723,22 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_277_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41477,9 +41747,9 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: expression ['as' star_target] +// _tmp_269: expression ['as' star_target] static void * -_tmp_268_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41495,22 +41765,22 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_277_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_278_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41519,9 +41789,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: expressions ['as' star_target] +// _tmp_270: expressions ['as' star_target] static void * -_tmp_269_rule(Parser *p) +_tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41537,22 +41807,22 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_278_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_279_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41561,9 +41831,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: 'as' NAME +// _tmp_271: 'as' NAME static void * -_tmp_270_rule(Parser *p) +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41579,7 +41849,7 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -41588,12 +41858,12 @@ _tmp_270_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41602,9 +41872,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: ',' bitwise_or +// _tmp_272: ',' bitwise_or static void * -_tmp_271_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41620,7 +41890,7 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); Token * _literal; expr_ty bitwise_or_var; if ( @@ -41629,12 +41899,12 @@ _tmp_271_rule(Parser *p) (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; @@ -41643,9 +41913,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: assignment_expression | expression !':=' +// _tmp_273: assignment_expression | expression !':=' static void * -_tmp_272_rule(Parser *p) +_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41661,18 +41931,18 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41680,7 +41950,7 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41688,12 +41958,12 @@ _tmp_272_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41702,9 +41972,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _loop0_274: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_275: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_274_rule(Parser *p) +_loop0_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41729,13 +41999,13 @@ _loop0_274_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_279_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -41761,7 +42031,7 @@ _loop0_274_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_274[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_275[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -41778,10 +42048,10 @@ _loop0_274_rule(Parser *p) return _seq; } -// _gather_273: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 +// _gather_274: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 static asdl_seq * -_gather_273_rule(Parser *p) +_gather_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41792,27 +42062,27 @@ _gather_273_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + D(fprintf(stderr, "%*c> _gather_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_279_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_274_rule(p)) // _loop0_274 + (seq = _loop0_275_rule(p)) // _loop0_275 ) { - D(fprintf(stderr, "%*c+ _gather_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + D(fprintf(stderr, "%*c+ _gather_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_273[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + D(fprintf(stderr, "%*c%s _gather_274[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); } _res = NULL; done: @@ -41820,9 +42090,9 @@ _gather_273_rule(Parser *p) return _res; } -// _tmp_275: 'as' star_target +// _tmp_276: 'as' star_target static void * -_tmp_275_rule(Parser *p) +_tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41838,7 +42108,7 @@ _tmp_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41847,12 +42117,12 @@ _tmp_275_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41861,9 +42131,9 @@ _tmp_275_rule(Parser *p) return _res; } -// _tmp_276: 'as' star_target +// _tmp_277: 'as' star_target static void * -_tmp_276_rule(Parser *p) +_tmp_277_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41879,7 +42149,7 @@ _tmp_276_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41888,12 +42158,12 @@ _tmp_276_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41902,9 +42172,9 @@ _tmp_276_rule(Parser *p) return _res; } -// _tmp_277: 'as' star_target +// _tmp_278: 'as' star_target static void * -_tmp_277_rule(Parser *p) +_tmp_278_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41920,7 +42190,7 @@ _tmp_277_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41929,12 +42199,12 @@ _tmp_277_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41943,9 +42213,9 @@ _tmp_277_rule(Parser *p) return _res; } -// _tmp_278: 'as' star_target +// _tmp_279: 'as' star_target static void * -_tmp_278_rule(Parser *p) +_tmp_279_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41961,7 +42231,7 @@ _tmp_278_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41970,12 +42240,12 @@ _tmp_278_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41984,9 +42254,9 @@ _tmp_278_rule(Parser *p) return _res; } -// _tmp_279: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_280: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_279_rule(Parser *p) +_tmp_280_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42002,18 +42272,18 @@ _tmp_279_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -42021,20 +42291,20 @@ _tmp_279_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_280_var; + D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_281_var; if ( - (_tmp_280_var = _tmp_280_rule(p)) // assignment_expression | expression !':=' + (_tmp_281_var = _tmp_281_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_280_var; + D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_281_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -42043,9 +42313,9 @@ _tmp_279_rule(Parser *p) return _res; } -// _tmp_280: assignment_expression | expression !':=' +// _tmp_281: assignment_expression | expression !':=' static void * -_tmp_280_rule(Parser *p) +_tmp_281_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42061,18 +42331,18 @@ _tmp_280_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -42080,7 +42350,7 @@ _tmp_280_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -42088,12 +42358,12 @@ _tmp_280_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 657e9345cf5ab7..3e7ab7bffb158e 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -25,8 +25,8 @@ unsigned char M_test_frozenmain[] = { 108,99,97,112,105,218,5,112,114,105,110,116,218,4,97,114, 103,118,218,11,103,101,116,95,99,111,110,102,105,103,115,114, 3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0, - 250,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, - 110,46,112,121,250,8,60,109,111,100,117,108,101,62,114,18, + 218,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, + 110,46,112,121,218,8,60,109,111,100,117,108,101,62,114,18, 0,0,0,1,0,0,0,115,99,0,0,0,240,3,1,1, 1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,212, 0,27,217,0,5,128,106,144,35,151,40,145,40,212,0,27, diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 60b46263a0d329..4956d04f719de9 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -203,6 +203,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->conversion); Py_CLEAR(state->ctx); Py_CLEAR(state->decorator_list); + Py_CLEAR(state->default_value); Py_CLEAR(state->defaults); Py_CLEAR(state->elt); Py_CLEAR(state->elts); @@ -311,6 +312,7 @@ static int init_identifiers(struct ast_state *state) if ((state->conversion = PyUnicode_InternFromString("conversion")) == NULL) return -1; if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return -1; if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return -1; + if ((state->default_value = PyUnicode_InternFromString("default_value")) == NULL) return -1; if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return -1; if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return -1; if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return -1; @@ -809,12 +811,15 @@ static PyObject* ast2obj_type_param(struct ast_state *state, struct validator static const char * const TypeVar_fields[]={ "name", "bound", + "default_value", }; static const char * const ParamSpec_fields[]={ "name", + "default_value", }; static const char * const TypeVarTuple_fields[]={ "name", + "default_value", }; @@ -4913,6 +4918,22 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVar_annotations, "default_value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->TypeVar_type, "_field_types", TypeVar_annotations) == 0; if (!cond) { @@ -4938,6 +4959,22 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + cond = PyDict_SetItemString(ParamSpec_annotations, "default_value", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->ParamSpec_type, "_field_types", ParamSpec_annotations) == 0; if (!cond) { @@ -4964,6 +5001,22 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVarTuple_annotations, "default_value", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->TypeVarTuple_type, "_field_types", TypeVarTuple_annotations) == 0; if (!cond) { @@ -5125,14 +5178,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } if (field_types == NULL) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s provides _fields but not _field_types. " - "This will become an error in Python 3.15.", - Py_TYPE(self)->tp_name - ) < 0) { - res = -1; - } + // Probably a user-defined subclass of AST that lacks _field_types. + // This will continue to work as it did before 3.13; i.e., attributes + // that are not passed in simply do not exist on the instance. goto cleanup; } remaining_list = PySequence_List(remaining_fields); @@ -5143,12 +5191,21 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) PyObject *name = PyList_GET_ITEM(remaining_list, i); PyObject *type = PyDict_GetItemWithError(field_types, name); if (!type) { - if (!PyErr_Occurred()) { - PyErr_SetObject(PyExc_KeyError, name); + if (PyErr_Occurred()) { + goto set_remaining_cleanup; + } + else { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Field '%U' is missing from %.400s._field_types. " + "This will become an error in Python 3.15.", + name, Py_TYPE(self)->tp_name + ) < 0) { + goto set_remaining_cleanup; + } } - goto set_remaining_cleanup; } - if (_PyUnion_Check(type)) { + else if (_PyUnion_Check(type)) { // optional field // do nothing, we'll have set a None default on the class } @@ -5172,8 +5229,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { - res = -1; - goto cleanup; + goto set_remaining_cleanup; } } } @@ -6243,28 +6299,37 @@ init_types(struct ast_state *state) if (!state->TypeIgnore_type) return -1; state->type_param_type = make_type(state, "type_param", state->AST_type, NULL, 0, - "type_param = TypeVar(identifier name, expr? bound)\n" - " | ParamSpec(identifier name)\n" - " | TypeVarTuple(identifier name)"); + "type_param = TypeVar(identifier name, expr? bound, expr? default_value)\n" + " | ParamSpec(identifier name, expr? default_value)\n" + " | TypeVarTuple(identifier name, expr? default_value)"); if (!state->type_param_type) return -1; if (add_attributes(state, state->type_param_type, type_param_attributes, 4) < 0) return -1; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, - TypeVar_fields, 2, - "TypeVar(identifier name, expr? bound)"); + TypeVar_fields, 3, + "TypeVar(identifier name, expr? bound, expr? default_value)"); if (!state->TypeVar_type) return -1; if (PyObject_SetAttr(state->TypeVar_type, state->bound, Py_None) == -1) return -1; + if (PyObject_SetAttr(state->TypeVar_type, state->default_value, Py_None) == + -1) + return -1; state->ParamSpec_type = make_type(state, "ParamSpec", state->type_param_type, ParamSpec_fields, - 1, - "ParamSpec(identifier name)"); + 2, + "ParamSpec(identifier name, expr? default_value)"); if (!state->ParamSpec_type) return -1; + if (PyObject_SetAttr(state->ParamSpec_type, state->default_value, Py_None) + == -1) + return -1; state->TypeVarTuple_type = make_type(state, "TypeVarTuple", state->type_param_type, - TypeVarTuple_fields, 1, - "TypeVarTuple(identifier name)"); + TypeVarTuple_fields, 2, + "TypeVarTuple(identifier name, expr? default_value)"); if (!state->TypeVarTuple_type) return -1; + if (PyObject_SetAttr(state->TypeVarTuple_type, state->default_value, + Py_None) == -1) + return -1; if (!add_ast_annotations(state)) { return -1; @@ -8055,8 +8120,9 @@ _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena) } type_param_ty -_PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_TypeVar(identifier name, expr_ty bound, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { type_param_ty p; if (!name) { @@ -8070,6 +8136,7 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int p->kind = TypeVar_kind; p->v.TypeVar.name = name; p->v.TypeVar.bound = bound; + p->v.TypeVar.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -8078,8 +8145,8 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int } type_param_ty -_PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena) +_PyAST_ParamSpec(identifier name, expr_ty default_value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { type_param_ty p; if (!name) { @@ -8092,6 +8159,7 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, return NULL; p->kind = ParamSpec_kind; p->v.ParamSpec.name = name; + p->v.ParamSpec.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -8100,8 +8168,9 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, } type_param_ty -_PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_TypeVarTuple(identifier name, expr_ty default_value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { type_param_ty p; if (!name) { @@ -8114,6 +8183,7 @@ _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int return NULL; p->kind = TypeVarTuple_kind; p->v.TypeVarTuple.name = name; + p->v.TypeVarTuple.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -10079,6 +10149,11 @@ ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) if (PyObject_SetAttr(result, state->bound, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.TypeVar.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; case ParamSpec_kind: tp = (PyTypeObject *)state->ParamSpec_type; @@ -10089,6 +10164,11 @@ ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.ParamSpec.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; case TypeVarTuple_kind: tp = (PyTypeObject *)state->TypeVarTuple_type; @@ -10099,6 +10179,11 @@ ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.TypeVarTuple.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; } value = ast2obj_int(state, vstate, o->lineno); @@ -16935,6 +17020,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (isinstance) { identifier name; expr_ty bound; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { return -1; @@ -16970,8 +17056,25 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeVar(name, bound, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'TypeVar' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_TypeVar(name, bound, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -16982,6 +17085,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, } if (isinstance) { identifier name; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { return -1; @@ -17000,8 +17104,25 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_ParamSpec(name, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'ParamSpec' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_ParamSpec(name, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -17012,6 +17133,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, } if (isinstance) { identifier name; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { return -1; @@ -17030,8 +17152,25 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeVarTuple(name, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'TypeVarTuple' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_TypeVarTuple(name, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -17452,6 +17591,7 @@ astmodule_exec(PyObject *m) static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index a7891709b3b44a..41e8107e205b46 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -322,6 +322,7 @@ static PyMethodDef tokenize_methods[] = { static PyModuleDef_Slot tokenizemodule_slots[] = { {Py_mod_exec, tokenizemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/_warnings.c b/Python/_warnings.c index 2ba704dcaa79b2..793cbc657f3184 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1498,6 +1498,7 @@ warnings_module_exec(PyObject *module) static PyModuleDef_Slot warnings_slots[] = { {Py_mod_exec, warnings_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/ast.c b/Python/ast.c index 71b09d889f17c1..1d1a48ec885686 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1011,13 +1011,19 @@ validate_typeparam(struct validator *state, type_param_ty tp) case TypeVar_kind: ret = validate_name(tp->v.TypeVar.name) && (!tp->v.TypeVar.bound || - validate_expr(state, tp->v.TypeVar.bound, Load)); + validate_expr(state, tp->v.TypeVar.bound, Load)) && + (!tp->v.TypeVar.default_value || + validate_expr(state, tp->v.TypeVar.default_value, Load)); break; case ParamSpec_kind: - ret = validate_name(tp->v.ParamSpec.name); + ret = validate_name(tp->v.ParamSpec.name) && + (!tp->v.ParamSpec.default_value || + validate_expr(state, tp->v.ParamSpec.default_value, Load)); break; case TypeVarTuple_kind: - ret = validate_name(tp->v.TypeVarTuple.name); + ret = validate_name(tp->v.TypeVarTuple.name) && + (!tp->v.TypeVarTuple.default_value || + validate_expr(state, tp->v.TypeVarTuple.default_value, Load)); break; } return ret; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 7af3ac9c5158d6..d192d5be751cfc 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -866,8 +866,21 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (str == NULL) goto error; +#ifdef Py_GIL_DISABLED + // gh-118527: Disable immortalization of code constants for explicit + // compile() calls to get consistent frozen outputs between the default + // and free-threaded builds. + PyInterpreterState *interp = _PyInterpreterState_GET(); + int old_value = interp->gc.immortalize.enable_on_thread_created; + interp->gc.immortalize.enable_on_thread_created = 0; +#endif + result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); +#ifdef Py_GIL_DISABLED + interp->gc.immortalize.enable_on_thread_created = old_value; +#endif + Py_XDECREF(source_copy); goto finally; @@ -925,9 +938,9 @@ builtin_divmod_impl(PyObject *module, PyObject *x, PyObject *y) eval as builtin_eval source: object + / globals: object = None locals: object = None - / Evaluate the given source in the context of globals and locals. @@ -941,7 +954,7 @@ If only globals is given, locals defaults to it. static PyObject * builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) -/*[clinic end generated code: output=0a0824aa70093116 input=11ee718a8640e527]*/ +/*[clinic end generated code: output=0a0824aa70093116 input=7c7bce5299a89062]*/ { PyObject *result = NULL, *source_copy; const char *str; @@ -1024,9 +1037,9 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, exec as builtin_exec source: object + / globals: object = None locals: object = None - / * closure: object(c_default="NULL") = None @@ -1044,7 +1057,7 @@ when source is a code object requiring exactly that many cellvars. static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) -/*[clinic end generated code: output=7579eb4e7646743d input=f13a7e2b503d1d9a]*/ +/*[clinic end generated code: output=7579eb4e7646743d input=25e989b6d87a3a21]*/ { PyObject *v; @@ -3124,6 +3137,9 @@ _PyBuiltin_Init(PyInterpreterState *interp) mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION); if (mod == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif dict = PyModule_GetDict(mod); #ifdef Py_TRACE_REFS diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f688856d6909ca..55eda9711dea1f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -20,7 +20,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -148,22 +148,31 @@ dummy_func( tier1 inst(RESUME, (--)) { assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - PyCodeObject *code = _PyFrame_GetCode(frame); - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(code, tstate->interp); - ERROR_IF(err, error); - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + PyCodeObject* code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + ERROR_IF(err, error); + next_instr = this_instr; + DISPATCH(); } - this_instr->op.code = RESUME_CHECK; + assert(this_instr->op.code == RESUME || + this_instr->op.code == RESUME_CHECK || + this_instr->op.code == INSTRUMENTED_RESUME || + this_instr->op.code == ENTER_EXECUTOR); + if (this_instr->op.code == RESUME) { + #if ENABLE_SPECIALIZATION + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ + } + } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } } @@ -181,7 +190,7 @@ dummy_func( inst(INSTRUMENTED_RESUME, (--)) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { ERROR_NO_POP(); } @@ -1089,32 +1098,40 @@ dummy_func( goto resume_frame; } - tier1 inst(YIELD_VALUE, (retval -- unused)) { + inst(YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + #if TIER_ONE assert(frame != &entry_frame); - frame->instr_ptr = next_instr; + #endif + frame->instr_ptr++; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); } inst(POP_EXCEPT, (exc_value -- )) { @@ -2355,6 +2372,7 @@ dummy_func( CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); + #ifdef _Py_TIER2 #if ENABLE_SPECIALIZATION _Py_BackoffCounter counter = this_instr[1].counter; if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { @@ -2380,6 +2398,7 @@ dummy_func( ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ } pseudo(JUMP) = { @@ -2393,23 +2412,31 @@ dummy_func( }; tier1 inst(ENTER_EXECUTOR, (--)) { - int prevoparg = oparg; - CHECK_EVAL_BREAKER(); - if (this_instr->op.code != ENTER_EXECUTOR || - this_instr->op.arg != prevoparg) { - next_instr = this_instr; - DISPATCH(); - } - + #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } tstate->previous_executor = Py_None; Py_INCREF(executor); GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { @@ -2589,9 +2616,7 @@ dummy_func( _PyErr_Clear(tstate); } /* iterator ended normally */ - Py_DECREF(iter); - STACK_SHRINK(1); - /* The translator sets the deopt target just past END_FOR */ + /* The translator sets the deopt target just past the matching END_FOR */ DEOPT_IF(true); } // Common case: no jump, leave it to the code generator @@ -3022,7 +3047,6 @@ dummy_func( family(CALL, INLINE_CACHE_ENTRIES_CALL) = { CALL_BOUND_METHOD_EXACT_ARGS, CALL_PY_EXACT_ARGS, - CALL_PY_WITH_DEFAULTS, CALL_TYPE_1, CALL_STR_1, CALL_TUPLE_1, @@ -3038,6 +3062,9 @@ dummy_func( CALL_METHOD_DESCRIPTOR_NOARGS, CALL_METHOD_DESCRIPTOR_FAST, CALL_ALLOC_AND_ENTER_INIT, + CALL_PY_GENERAL, + CALL_BOUND_METHOD_GENERAL, + CALL_NON_PY_GENERAL, }; specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -3127,9 +3154,108 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + SYNC_SP(); + if (new_frame == NULL) { + ERROR_NO_POP(); + } + } + + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + EXIT_IF(!PyFunction_Check(callable)); + PyFunctionObject *func = (PyFunctionObject *)callable; + EXIT_IF(func->func_version != func_version); + } + + macro(CALL_PY_GENERAL) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION + + _PY_FRAME_GENERAL + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable)->im_func; + EXIT_IF(!PyFunction_Check(func)); + EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); + EXIT_IF(null != NULL); + } + + op(_EXPAND_METHOD, (callable, null, unused[oparg] -- method, self, unused[oparg])) { + assert(null == NULL); + assert(Py_TYPE(callable) == &PyMethod_Type); + self = ((PyMethodObject *)callable)->im_self; + Py_INCREF(self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = ((PyMethodObject *)callable)->im_func; + assert(PyFunction_Check(method)); + Py_INCREF(method); + Py_DECREF(callable); + } + + macro(CALL_BOUND_METHOD_GENERAL) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION + + _EXPAND_METHOD + + _PY_FRAME_GENERAL + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + EXIT_IF(PyFunction_Check(callable)); + EXIT_IF(Py_TYPE(callable) == &PyMethod_Type); + } + + op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { +#if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); +#endif + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + ERROR_IF(res == NULL, error); + } + + macro(CALL_NON_PY_GENERAL) = + unused/1 + // Skip over the counter + unused/2 + + _CHECK_IS_NOT_PY_CALLABLE + + _CALL_NON_PY_GENERAL + + _CHECK_PERIODIC; + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - DEOPT_IF(null != NULL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); + EXIT_IF(null != NULL); + EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { @@ -3207,40 +3333,6 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - DEOPT_IF(tstate->interp->eval_frame); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount); - DEOPT_IF(argcount < min_args); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); - } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); @@ -4091,7 +4183,6 @@ dummy_func( #ifndef _Py_JIT next_uop = ¤t_executor->trace[1]; #endif - CHECK_EVAL_BREAKER(); } tier2 op(_SET_IP, (instr_ptr/4 --)) { @@ -4114,7 +4205,7 @@ dummy_func( } tier2 op(_EXIT_TRACE, (--)) { - EXIT_IF(1); + EXIT_TO_TRACE(); } tier2 op(_CHECK_VALIDITY, (--)) { @@ -4247,38 +4338,22 @@ dummy_func( EXIT_TO_TIER1(); } - tier2 op(_SIDE_EXIT, (--)) { - EXIT_TO_TRACE(); - } - tier2 op(_ERROR_POP_N, (target/2, unused[oparg] --)) { frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target; SYNC_SP(); GOTO_UNWIND(); } - - /* Special version of RESUME_CHECK that (when paired with _EVAL_BREAKER_EXIT) - * is safe for tier 2. Progress is guaranteed because _EVAL_BREAKER_EXIT calls - * _Py_HandlePending which clears the eval_breaker so that _TIER2_RESUME_CHECK - * will not exit if it is immediately executed again. */ + /* Progress is guaranteed if we DEOPT on the eval breaker, because + * ENTER_EXECUTOR will not re-enter tier 2 with the eval breaker set. */ tier2 op(_TIER2_RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - EXIT_IF(_Py_emscripten_signal_clock == 0); + DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - EXIT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); - assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); - } - - tier2 op(_EVAL_BREAKER_EXIT, (--)) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_HandlePending(tstate) != 0) { - GOTO_UNWIND(); - } - EXIT_TO_TRACE(); + DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index d130c734a67144..128e0417a9fd63 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -20,7 +20,7 @@ #include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -247,10 +247,6 @@ static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); -static _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames); static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); @@ -755,7 +751,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; PyObject **stack_pointer; -#ifndef _Py_JIT +#if defined(_Py_TIER2) && !defined(_Py_JIT) /* Tier 2 interpreter state */ _PyExecutorObject *current_executor = NULL; const _PyUOpInstruction *next_uop = NULL; @@ -808,17 +804,23 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int { _Py_CODEUNIT *prev = frame->instr_ptr; _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); + } } if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); @@ -959,6 +961,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; +#ifdef _Py_TIER2 // Tier 2 is also here! enter_tier_two: @@ -1113,6 +1116,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif // _Py_JIT +#endif // _Py_TIER2 + } #if defined(__GNUC__) @@ -1707,7 +1712,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) } /* Consumes references to func, locals and all the args */ -static _PyInterpreterFrame * +_PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames) @@ -1727,6 +1732,8 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return frame; fail: /* Consume the references */ + Py_DECREF(func); + Py_XDECREF(locals); for (size_t i = 0; i < argcount; i++) { Py_DECREF(args[i]); } @@ -2468,12 +2475,7 @@ PyEval_GetLocals(void) return NULL; } - if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { - return NULL; - } - - PyObject *locals = current_frame->f_locals; - assert(locals != NULL); + PyObject *locals = _PyEval_GetFrameLocals(); return locals; } @@ -2487,7 +2489,28 @@ _PyEval_GetFrameLocals(void) return NULL; } - return _PyFrame_GetLocals(current_frame, 1); + PyObject *locals = _PyFrame_GetLocals(current_frame); + if (locals == NULL) { + return NULL; + } + + if (PyFrameLocalsProxy_Check(locals)) { + PyObject* ret = PyDict_New(); + if (ret == NULL) { + Py_DECREF(locals); + return NULL; + } + if (PyDict_Update(ret, locals) < 0) { + Py_DECREF(ret); + Py_DECREF(locals); + return NULL; + } + Py_DECREF(locals); + return ret; + } + + assert(PyMapping_Check(locals)); + return locals; } PyObject * @@ -2501,6 +2524,28 @@ PyEval_GetGlobals(void) return current_frame->f_globals; } +PyObject* +PyEval_GetFrameLocals(void) +{ + return _PyEval_GetFrameLocals(); +} + +PyObject* PyEval_GetFrameGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + return NULL; + } + return Py_XNewRef(current_frame->f_globals); +} + +PyObject* PyEval_GetFrameBuiltins(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return Py_XNewRef(_PyEval_GetBuiltins(tstate)); +} + int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index fdbb4882c3d711..7a54c185303cd1 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -205,6 +205,16 @@ static void recreate_gil(struct _gil_runtime_state *gil) } #endif +static void +drop_gil_impl(struct _gil_runtime_state *gil) +{ + MUTEX_LOCK(gil->mutex); + _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); + _Py_atomic_store_int_relaxed(&gil->locked, 0); + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); +} + static void drop_gil(PyInterpreterState *interp, PyThreadState *tstate) { @@ -220,11 +230,11 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) struct _gil_runtime_state *gil = ceval->gil; #ifdef Py_GIL_DISABLED - if (!gil->enabled) { + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { return; } #endif - if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) { + if (!_Py_atomic_load_int_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -236,11 +246,7 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) _Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate); } - MUTEX_LOCK(gil->mutex); - _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); - _Py_atomic_store_int_relaxed(&gil->locked, 0); - COND_SIGNAL(gil->cond); - MUTEX_UNLOCK(gil->mutex); + drop_gil_impl(gil); #ifdef FORCE_SWITCHING /* We check tstate first in case we might be releasing the GIL for @@ -275,8 +281,10 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) The function saves errno at entry and restores its value at exit. - tstate must be non-NULL. */ -static void + tstate must be non-NULL. + + Returns 1 if the GIL was acquired, or 0 if not. */ +static int take_gil(PyThreadState *tstate) { int err = errno; @@ -300,8 +308,8 @@ take_gil(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; struct _gil_runtime_state *gil = interp->ceval.gil; #ifdef Py_GIL_DISABLED - if (!gil->enabled) { - return; + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + return 0; } #endif @@ -346,6 +354,17 @@ take_gil(PyThreadState *tstate) } } +#ifdef Py_GIL_DISABLED + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + // Another thread disabled the GIL between our check above and + // now. Don't take the GIL, signal any other waiting threads, and + // return 0. + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); + return 0; + } +#endif + #ifdef FORCE_SWITCHING /* This mutex must be taken before modifying gil->last_holder: see drop_gil(). */ @@ -387,6 +406,7 @@ take_gil(PyThreadState *tstate) MUTEX_UNLOCK(gil->mutex); errno = err; + return 1; } void _PyEval_SetSwitchInterval(unsigned long microseconds) @@ -451,7 +471,8 @@ init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { assert(!gil_created(gil)); #ifdef Py_GIL_DISABLED - gil->enabled = _PyInterpreterState_GetConfig(interp)->enable_gil == _PyConfig_GIL_ENABLE; + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + gil->enabled = config->enable_gil == _PyConfig_GIL_ENABLE ? INT_MAX : 0; #endif create_gil(gil); assert(gil_created(gil)); @@ -545,11 +566,11 @@ PyEval_ReleaseLock(void) drop_gil(tstate->interp, tstate); } -void +int _PyEval_AcquireLock(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - take_gil(tstate); + return take_gil(tstate); } void @@ -877,21 +898,20 @@ make_pending_calls(PyThreadState *tstate) /* Only one thread (per interpreter) may run the pending calls at once. In the same way, we don't do recursive pending calls. */ PyMutex_Lock(&pending->mutex); - if (pending->busy) { + if (pending->handling_thread != NULL) { /* A pending call was added after another thread was already handling the pending calls (and had already "unsignaled"). Once that thread is done, it may have taken care of all the pending calls, or there might be some still waiting. - Regardless, this interpreter's pending calls will stay - "signaled" until that first thread has finished. At that - point the next thread to trip the eval breaker will take - care of any remaining pending calls. Until then, though, - all the interpreter's threads will be tripping the eval - breaker every time it's checked. */ + To avoid all threads constantly stopping on the eval breaker, + we clear the bit for this thread and make sure it is set + for the thread currently handling the pending call. */ + _Py_set_eval_breaker_bit(pending->handling_thread, _PY_CALLS_TO_DO_BIT); + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); PyMutex_Unlock(&pending->mutex); return 0; } - pending->busy = 1; + pending->handling_thread = tstate; PyMutex_Unlock(&pending->mutex); /* unsignal before starting to call callbacks, so that any callback @@ -900,7 +920,7 @@ make_pending_calls(PyThreadState *tstate) int32_t npending; if (_make_pending_calls(pending, &npending) != 0) { - pending->busy = 0; + pending->handling_thread = NULL; /* There might not be more calls to make, but we play it safe. */ signal_pending_calls(tstate, interp); return -1; @@ -912,7 +932,7 @@ make_pending_calls(PyThreadState *tstate) if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { if (_make_pending_calls(pending_main, &npending) != 0) { - pending->busy = 0; + pending->handling_thread = NULL; /* There might not be more calls to make, but we play it safe. */ signal_pending_calls(tstate, interp); return -1; @@ -923,7 +943,7 @@ make_pending_calls(PyThreadState *tstate) } } - pending->busy = 0; + pending->handling_thread = NULL; return 0; } @@ -1012,6 +1032,117 @@ _PyEval_InitState(PyInterpreterState *interp) _gil_initialize(&interp->_gil); } +#ifdef Py_GIL_DISABLED +int +_PyEval_EnableGILTransient(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + // The GIL is already enabled permanently. + return 0; + } + if (enabled == INT_MAX - 1) { + Py_FatalError("Too many transient requests to enable the GIL"); + } + if (enabled > 0) { + // If enabled is nonzero, we know we hold the GIL. This means that no + // other threads are attached, and nobody else can be concurrently + // mutating it. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + return 0; + } + + // Enabling the GIL changes what it means to be an "attached" thread. To + // safely make this transition, we: + // 1. Detach the current thread. + // 2. Stop the world to detach (and suspend) all other threads. + // 3. Enable the GIL, if nobody else did between our check above and when + // our stop-the-world begins. + // 4. Start the world. + // 5. Attach the current thread. Other threads may attach and hold the GIL + // before this thread, which is harmless. + _PyThreadState_Detach(tstate); + + // This could be an interpreter-local stop-the-world in situations where we + // know that this interpreter's GIL is not shared, and that it won't become + // shared before the stop-the-world begins. For now, we always stop all + // interpreters for simplicity. + _PyEval_StopTheWorldAll(&_PyRuntime); + + enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + int this_thread_enabled = enabled == 0; + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_Attach(tstate); + + return this_thread_enabled; +} + +int +_PyEval_EnableGILPermanent(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; + } + + _Py_atomic_store_int_relaxed(&gil->enabled, INT_MAX); + return 1; +} + +int +_PyEval_DisableGIL(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; + } + + assert(enabled >= 1); + enabled--; + + // Disabling the GIL is much simpler than enabling it, since we know we are + // the only attached thread. Other threads may start free-threading as soon + // as this store is complete, if it sets gil->enabled to 0. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled); + + if (enabled == 0) { + // We're attached, so we know the GIL will remain disabled until at + // least the next time we detach, which must be after this function + // returns. + // + // Drop the GIL, which will wake up any threads waiting in take_gil() + // and let them resume execution without the GIL. + drop_gil_impl(gil); + return 1; + } + return 0; +} +#endif + /* Do periodic things, like check for signals and async I/0. * We need to do reasonably frequently, but not too frequently. diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1a8554ab72269f..50941e4ec473e8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -162,7 +162,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* The integer overflow is checked by an assertion below. */ #define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame)))) #define NEXTOPARG() do { \ - _Py_CODEUNIT word = *next_instr; \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ opcode = word.op.code; \ oparg = word.op.arg; \ } while (0) @@ -358,12 +358,16 @@ do { \ // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (next_instr == NULL) { \ - next_instr = (dest)+1; \ - goto error; \ + if (tstate->tracing) {\ + next_instr = dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ } \ } while (0); diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 3f005bcbfb6a1a..f75a8d4ac0ccd4 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -395,7 +395,7 @@ builtin_divmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_eval__doc__, -"eval($module, source, globals=None, locals=None, /)\n" +"eval($module, source, /, globals=None, locals=None)\n" "--\n" "\n" "Evaluate the given source in the context of globals and locals.\n" @@ -407,33 +407,63 @@ PyDoc_STRVAR(builtin_eval__doc__, "If only globals is given, locals defaults to it."); #define BUILTIN_EVAL_METHODDEF \ - {"eval", _PyCFunction_CAST(builtin_eval), METH_FASTCALL, builtin_eval__doc__}, + {"eval", _PyCFunction_CAST(builtin_eval), METH_FASTCALL|METH_KEYWORDS, builtin_eval__doc__}, static PyObject * builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals); static PyObject * -builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(globals), &_Py_ID(locals), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "globals", "locals", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "eval", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *source; PyObject *globals = Py_None; PyObject *locals = Py_None; - if (!_PyArg_CheckPositional("eval", nargs, 1, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { goto exit; } source = args[0]; - if (nargs < 2) { - goto skip_optional; + if (!noptargs) { + goto skip_optional_pos; } - globals = args[1]; - if (nargs < 3) { - goto skip_optional; + if (args[1]) { + globals = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } } locals = args[2]; -skip_optional: +skip_optional_pos: return_value = builtin_eval_impl(module, source, globals, locals); exit: @@ -441,7 +471,7 @@ builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_exec__doc__, -"exec($module, source, globals=None, locals=None, /, *, closure=None)\n" +"exec($module, source, /, globals=None, locals=None, *, closure=None)\n" "--\n" "\n" "Execute the given source in the context of globals and locals.\n" @@ -467,14 +497,14 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(closure), }, + .ob_item = { &_Py_ID(globals), &_Py_ID(locals), &_Py_ID(closure), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -483,7 +513,7 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "", "", "closure", NULL}; + static const char * const _keywords[] = {"", "globals", "locals", "closure", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "exec", @@ -502,17 +532,22 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } source = args[0]; - if (nargs < 2) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - globals = args[1]; - if (nargs < 3) { - goto skip_optional_posonly; + if (args[1]) { + globals = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } } - noptargs--; - locals = args[2]; -skip_optional_posonly: + if (args[2]) { + locals = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -1193,4 +1228,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=6d15edfc194b2c08 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=435d3f286a863c49 input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 31f66e807a8547..56a831eb2ea06e 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1485,6 +1485,52 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(sys__baserepl__doc__, +"_baserepl($module, /)\n" +"--\n" +"\n" +"Private function for getting the base REPL"); + +#define SYS__BASEREPL_METHODDEF \ + {"_baserepl", (PyCFunction)sys__baserepl, METH_NOARGS, sys__baserepl__doc__}, + +static PyObject * +sys__baserepl_impl(PyObject *module); + +static PyObject * +sys__baserepl(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__baserepl_impl(module); +} + +PyDoc_STRVAR(sys__is_gil_enabled__doc__, +"_is_gil_enabled($module, /)\n" +"--\n" +"\n" +"Return True if the GIL is currently enabled and False otherwise."); + +#define SYS__IS_GIL_ENABLED_METHODDEF \ + {"_is_gil_enabled", (PyCFunction)sys__is_gil_enabled, METH_NOARGS, sys__is_gil_enabled__doc__}, + +static int +sys__is_gil_enabled_impl(PyObject *module); + +static PyObject * +sys__is_gil_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = sys__is_gil_enabled_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + #ifndef SYS_GETWINDOWSVERSION_METHODDEF #define SYS_GETWINDOWSVERSION_METHODDEF #endif /* !defined(SYS_GETWINDOWSVERSION_METHODDEF) */ @@ -1528,4 +1574,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=518424ee03e353b0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef7c35945443d300 input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index d8fe7b22063a80..bed245366f9234 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -11,6 +11,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.codec_search_path +#include "pycore_lock.h" // PyMutex #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI @@ -19,24 +20,10 @@ const char *Py_hexdigits = "0123456789abcdef"; /* --- Codec Registry ----------------------------------------------------- */ -/* Import the standard encodings package which will register the first - codec search function. - - This is done in a lazy way so that the Unicode implementation does - not downgrade startup time of scripts not needing it. - - ImportErrors are silently ignored by this function. Only one try is - made. - -*/ - -static int _PyCodecRegistry_Init(void); /* Forward */ - int PyCodec_Register(PyObject *search_function) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - goto onError; + assert(interp->codecs.initialized); if (search_function == NULL) { PyErr_BadArgument(); goto onError; @@ -45,7 +32,14 @@ int PyCodec_Register(PyObject *search_function) PyErr_SetString(PyExc_TypeError, "argument must be callable"); goto onError; } - return PyList_Append(interp->codec_search_path, search_function); +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->codecs.search_path_mutex); +#endif + int ret = PyList_Append(interp->codecs.search_path, search_function); +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->codecs.search_path_mutex); +#endif + return ret; onError: return -1; @@ -55,22 +49,34 @@ int PyCodec_Unregister(PyObject *search_function) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *codec_search_path = interp->codec_search_path; - /* Do nothing if codec_search_path is not created yet or was cleared. */ - if (codec_search_path == NULL) { + if (interp->codecs.initialized != 1) { + /* Do nothing if codecs state was cleared (only possible during + interpreter shutdown). */ return 0; } + PyObject *codec_search_path = interp->codecs.search_path; assert(PyList_CheckExact(codec_search_path)); - Py_ssize_t n = PyList_GET_SIZE(codec_search_path); - for (Py_ssize_t i = 0; i < n; i++) { - PyObject *item = PyList_GET_ITEM(codec_search_path, i); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(codec_search_path); i++) { +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->codecs.search_path_mutex); +#endif + PyObject *item = PyList_GetItemRef(codec_search_path, i); + int ret = 1; if (item == search_function) { - if (interp->codec_search_cache != NULL) { - assert(PyDict_CheckExact(interp->codec_search_cache)); - PyDict_Clear(interp->codec_search_cache); - } - return PyList_SetSlice(codec_search_path, i, i+1, NULL); + // We hold a reference to the item, so its destructor can't run + // while we hold search_path_mutex. + ret = PyList_SetSlice(codec_search_path, i, i+1, NULL); + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->codecs.search_path_mutex); +#endif + Py_DECREF(item); + if (ret != 1) { + assert(interp->codecs.search_cache != NULL); + assert(PyDict_CheckExact(interp->codecs.search_cache)); + PyDict_Clear(interp->codecs.search_cache); + return ret; } } return 0; @@ -132,9 +138,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) { - return NULL; - } + assert(interp->codecs.initialized); /* Convert the encoding to a normalized Python string: all characters are converted to lower case, spaces and hyphens are @@ -147,7 +151,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) /* First, try to lookup the name in the registry dictionary */ PyObject *result; - if (PyDict_GetItemRef(interp->codec_search_cache, v, &result) < 0) { + if (PyDict_GetItemRef(interp->codecs.search_cache, v, &result) < 0) { goto onError; } if (result != NULL) { @@ -156,7 +160,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } /* Next, scan the search functions in order of registration */ - const Py_ssize_t len = PyList_Size(interp->codec_search_path); + const Py_ssize_t len = PyList_Size(interp->codecs.search_path); if (len < 0) goto onError; if (len == 0) { @@ -170,14 +174,15 @@ PyObject *_PyCodec_Lookup(const char *encoding) for (i = 0; i < len; i++) { PyObject *func; - func = PyList_GetItem(interp->codec_search_path, i); + func = PyList_GetItemRef(interp->codecs.search_path, i); if (func == NULL) goto onError; result = PyObject_CallOneArg(func, v); + Py_DECREF(func); if (result == NULL) goto onError; if (result == Py_None) { - Py_DECREF(result); + Py_CLEAR(result); continue; } if (!PyTuple_Check(result) || PyTuple_GET_SIZE(result) != 4) { @@ -188,7 +193,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } break; } - if (i == len) { + if (result == NULL) { /* XXX Perhaps we should cache misses too ? */ PyErr_Format(PyExc_LookupError, "unknown encoding: %s", encoding); @@ -196,7 +201,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } /* Cache and return the result */ - if (PyDict_SetItem(interp->codec_search_cache, v, result) < 0) { + if (PyDict_SetItem(interp->codecs.search_cache, v, result) < 0) { Py_DECREF(result); goto onError; } @@ -600,13 +605,12 @@ PyObject *_PyCodec_DecodeText(PyObject *object, int PyCodec_RegisterError(const char *name, PyObject *error) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - return -1; + assert(interp->codecs.initialized); if (!PyCallable_Check(error)) { PyErr_SetString(PyExc_TypeError, "handler must be callable"); return -1; } - return PyDict_SetItemString(interp->codec_error_registry, + return PyDict_SetItemString(interp->codecs.error_registry, name, error); } @@ -616,13 +620,12 @@ int PyCodec_RegisterError(const char *name, PyObject *error) PyObject *PyCodec_LookupError(const char *name) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - return NULL; + assert(interp->codecs.initialized); if (name==NULL) name = "strict"; PyObject *handler; - if (PyDict_GetItemStringRef(interp->codec_error_registry, name, &handler) < 0) { + if (PyDict_GetItemStringRef(interp->codecs.error_registry, name, &handler) < 0) { return NULL; } if (handler == NULL) { @@ -1375,7 +1378,8 @@ static PyObject *surrogateescape_errors(PyObject *self, PyObject *exc) return PyCodec_SurrogateEscapeErrors(exc); } -static int _PyCodecRegistry_Init(void) +PyStatus +_PyCodec_InitRegistry(PyInterpreterState *interp) { static struct { const char *name; @@ -1463,45 +1467,51 @@ static int _PyCodecRegistry_Init(void) } }; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *mod; - - if (interp->codec_search_path != NULL) - return 0; - - interp->codec_search_path = PyList_New(0); - if (interp->codec_search_path == NULL) { - return -1; + assert(interp->codecs.initialized == 0); + interp->codecs.search_path = PyList_New(0); + if (interp->codecs.search_path == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_search_cache = PyDict_New(); - if (interp->codec_search_cache == NULL) { - return -1; + interp->codecs.search_cache = PyDict_New(); + if (interp->codecs.search_cache == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_error_registry = PyDict_New(); - if (interp->codec_error_registry == NULL) { - return -1; + interp->codecs.error_registry = PyDict_New(); + if (interp->codecs.error_registry == NULL) { + return PyStatus_NoMemory(); } - for (size_t i = 0; i < Py_ARRAY_LENGTH(methods); ++i) { PyObject *func = PyCFunction_NewEx(&methods[i].def, NULL, NULL); - if (!func) { - return -1; + if (func == NULL) { + return PyStatus_NoMemory(); } - int res = PyCodec_RegisterError(methods[i].name, func); + int res = PyDict_SetItemString(interp->codecs.error_registry, + methods[i].name, func); Py_DECREF(func); - if (res) { - return -1; + if (res < 0) { + return PyStatus_Error("Failed to insert into codec error registry"); } } - mod = PyImport_ImportModule("encodings"); + interp->codecs.initialized = 1; + + // Importing `encodings' will call back into this module to register codec + // search functions, so this is done after everything else is initialized. + PyObject *mod = PyImport_ImportModule("encodings"); if (mod == NULL) { - return -1; + return PyStatus_Error("Failed to import encodings module"); } Py_DECREF(mod); - interp->codecs_initialized = 1; - return 0; + + return PyStatus_Ok(); +} + +void +_PyCodec_Fini(PyInterpreterState *interp) +{ + Py_CLEAR(interp->codecs.search_path); + Py_CLEAR(interp->codecs.search_cache); + Py_CLEAR(interp->codecs.error_registry); + interp->codecs.initialized = 0; } diff --git a/Python/compile.c b/Python/compile.c index ca5551a8e64ab0..79f3baadca6b4a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -113,7 +113,8 @@ compiler IR. enum fblocktype { WHILE_LOOP, FOR_LOOP, TRY_EXCEPT, FINALLY_TRY, FINALLY_END, WITH, ASYNC_WITH, HANDLER_CLEANUP, POP_VALUE, EXCEPTION_HANDLER, - EXCEPTION_GROUP_HANDLER, ASYNC_COMPREHENSION_GENERATOR }; + EXCEPTION_GROUP_HANDLER, ASYNC_COMPREHENSION_GENERATOR, + STOP_ITERATION }; struct fblockinfo { enum fblocktype fb_type; @@ -1503,6 +1504,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, case EXCEPTION_HANDLER: case EXCEPTION_GROUP_HANDLER: case ASYNC_COMPREHENSION_GENERATOR: + case STOP_ITERATION: return SUCCESS; case FOR_LOOP: @@ -2114,6 +2116,36 @@ wrap_in_stopiteration_handler(struct compiler *c) return SUCCESS; } +static int +compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, + identifier name, void *key, + bool allow_starred) +{ + if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS, + key, e->lineno) == -1) { + return ERROR; + } + if (allow_starred && e->kind == Starred_kind) { + VISIT(c, expr, e->v.Starred.value); + ADDOP_I(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); + } + else { + VISIT(c, expr, e); + } + ADDOP_IN_SCOPE(c, LOC(e), RETURN_VALUE); + PyCodeObject *co = optimize_and_assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) { + return ERROR; + } + if (compiler_make_closure(c, LOC(e), co, 0) < 0) { + Py_DECREF(co); + return ERROR; + } + Py_DECREF(co); + return SUCCESS; +} + static int compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) { @@ -2121,6 +2153,7 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) return SUCCESS; } Py_ssize_t n = asdl_seq_LEN(type_params); + bool seen_default = false; for (Py_ssize_t i = 0; i < n; i++) { type_param_ty typeparam = asdl_seq_GET(type_params, i); @@ -2130,22 +2163,10 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVar.name); if (typeparam->v.TypeVar.bound) { expr_ty bound = typeparam->v.TypeVar.bound; - if (compiler_enter_scope(c, typeparam->v.TypeVar.name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparam, bound->lineno) == -1) { + if (compiler_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, + (void *)typeparam, false) < 0) { return ERROR; } - VISIT_IN_SCOPE(c, expr, bound); - ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); - PyCodeObject *co = optimize_and_assemble(c, 1); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); int intrinsic = bound->kind == Tuple_kind ? INTRINSIC_TYPEVAR_WITH_CONSTRAINTS @@ -2155,18 +2176,60 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) else { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVAR); } + if (typeparam->v.TypeVar.default_value) { + seen_default = true; + expr_ty default_ = typeparam->v.TypeVar.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, + (void *)((uintptr_t)typeparam + 1), false) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVar.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVar.name, Store)); break; case TypeVarTuple_kind: ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVarTuple.name); ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVARTUPLE); + if (typeparam->v.TypeVarTuple.default_value) { + expr_ty default_ = typeparam->v.TypeVarTuple.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, + (void *)typeparam, true) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVarTuple.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVarTuple.name, Store)); break; case ParamSpec_kind: ADDOP_LOAD_CONST(c, loc, typeparam->v.ParamSpec.name); ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PARAMSPEC); + if (typeparam->v.ParamSpec.default_value) { + expr_ty default_ = typeparam->v.ParamSpec.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, + (void *)typeparam, false) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.ParamSpec.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.ParamSpec.name, Store)); break; @@ -2232,14 +2295,26 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); + + NEW_JUMP_TARGET_LABEL(c, start); + USE_LABEL(c, start); + bool add_stopiteration_handler = c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator; + if (add_stopiteration_handler) { + /* wrap_in_stopiteration_handler will push a block, so we need to account for that */ + RETURN_IF_ERROR( + compiler_push_fblock(c, NO_LOCATION, STOP_ITERATION, + start, NO_LABEL, NULL)); + } + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(body); i++) { VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } - if (c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator) { + if (add_stopiteration_handler) { if (wrap_in_stopiteration_handler(c) < 0) { compiler_exit_scope(c); return ERROR; } + compiler_pop_fblock(c, STOP_ITERATION, start); } PyCodeObject *co = optimize_and_assemble(c, 1); compiler_exit_scope(c); @@ -2427,6 +2502,11 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromLong(c->u->u_metadata.u_firstlineno)); + if (compiler_nameop(c, loc, &_Py_ID(__firstlineno__), Store) < 0) { + compiler_exit_scope(c); + return ERROR; + } asdl_type_param_seq *type_params = s->v.ClassDef.type_params; if (asdl_seq_LEN(type_params) > 0) { if (!compiler_set_type_params_in_class(c, loc)) { @@ -3835,7 +3915,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) } if (location_is_after(LOC(s), c->c_future.ff_location) && - s->v.ImportFrom.module && + s->v.ImportFrom.module && s->v.ImportFrom.level == 0 && _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) { Py_DECREF(names); @@ -5486,10 +5566,48 @@ push_inlined_comprehension_state(struct compiler *c, location loc, while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { assert(PyLong_Check(v)); long symbol = PyLong_AS_LONG(v); - // only values bound in the comprehension (DEF_LOCAL) need to be handled - // at all; DEF_LOCAL | DEF_NONLOCAL can occur in the case of an - // assignment expression to a nonlocal in the comprehension, these don't - // need handling here since they shouldn't be isolated + long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; + PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); + if (outv == NULL) { + if (PyErr_Occurred()) { + return ERROR; + } + outv = _PyLong_GetZero(); + } + assert(PyLong_CheckExact(outv)); + long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; + // If a name has different scope inside than outside the comprehension, + // we need to temporarily handle it with the right scope while + // compiling the comprehension. If it's free in the comprehension + // scope, no special handling; it should be handled the same as the + // enclosing scope. (If it's free in outer scope and cell in inner + // scope, we can't treat it as both cell and free in the same function, + // but treating it as free throughout is fine; it's *_DEREF + // either way.) + if ((scope != outsc && scope != FREE && !(scope == CELL && outsc == FREE)) + || in_class_block) { + if (state->temp_symbols == NULL) { + state->temp_symbols = PyDict_New(); + if (state->temp_symbols == NULL) { + return ERROR; + } + } + // update the symbol to the in-comprehension version and save + // the outer version; we'll restore it after running the + // comprehension + Py_INCREF(outv); + if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { + Py_DECREF(outv); + return ERROR; + } + if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { + Py_DECREF(outv); + return ERROR; + } + Py_DECREF(outv); + } + // locals handling for names bound in comprehension (DEF_LOCAL | + // DEF_NONLOCAL occurs in assignment expression to nonlocal) if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { if (!_PyST_IsFunctionLike(c->u->u_ste)) { // non-function scope: override this name to use fast locals @@ -5514,41 +5632,6 @@ push_inlined_comprehension_state(struct compiler *c, location loc, } } } - long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; - PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); - if (outv == NULL) { - outv = _PyLong_GetZero(); - } - assert(PyLong_Check(outv)); - long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; - if (scope != outsc && !(scope == CELL && outsc == FREE)) { - // If a name has different scope inside than outside the - // comprehension, we need to temporarily handle it with the - // right scope while compiling the comprehension. (If it's free - // in outer scope and cell in inner scope, we can't treat it as - // both cell and free in the same function, but treating it as - // free throughout is fine; it's *_DEREF either way.) - - if (state->temp_symbols == NULL) { - state->temp_symbols = PyDict_New(); - if (state->temp_symbols == NULL) { - return ERROR; - } - } - // update the symbol to the in-comprehension version and save - // the outer version; we'll restore it after running the - // comprehension - Py_INCREF(outv); - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { - Py_DECREF(outv); - return ERROR; - } - if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { - Py_DECREF(outv); - return ERROR; - } - Py_DECREF(outv); - } // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack diff --git a/Python/errors.c b/Python/errors.c index e5f176a5dd208e..433253b8f9aada 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -632,8 +632,8 @@ _PyErr_StackItemToExcInfoTuple(_PyErr_StackItem *err_info) PyObject *exc_type = get_exc_type(exc_value); PyObject *exc_traceback = get_exc_traceback(exc_value); - return Py_BuildValue( - "(OOO)", + return PyTuple_Pack( + 3, exc_type ? exc_type : Py_None, exc_value ? exc_value : Py_None, exc_traceback ? exc_traceback : Py_None); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2d9acfeea432bc..347a1e677a0832 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1110,6 +1110,49 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ + case _YIELD_VALUE: { + PyObject *retval; + PyObject *value; + oparg = CURRENT_OPARG(); + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _POP_EXCEPT: { PyObject *exc_value; exc_value = stack_pointer[-1]; @@ -2623,9 +2666,7 @@ _PyErr_Clear(tstate); } /* iterator ended normally */ - Py_DECREF(iter); - STACK_SHRINK(1); - /* The translator sets the deopt target just past END_FOR */ + /* The translator sets the deopt target just past the matching END_FOR */ if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2993,6 +3034,153 @@ break; } + case _PY_FRAME_GENERAL: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { + JUMP_TO_ERROR(); + } + stack_pointer[0] = (PyObject *)new_frame; + stack_pointer += 1; + break; + } + + case _CHECK_FUNCTION_VERSION: { + PyObject *callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + if (!PyFunction_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_METHOD_VERSION: { + PyObject *null; + PyObject *callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + if (Py_TYPE(callable) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EXPAND_METHOD: { + PyObject *null; + PyObject *callable; + PyObject *method; + PyObject *self; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(null == NULL); + assert(Py_TYPE(callable) == &PyMethod_Type); + self = ((PyMethodObject *)callable)->im_self; + Py_INCREF(self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = ((PyMethodObject *)callable)->im_func; + assert(PyFunction_Check(method)); + Py_INCREF(method); + Py_DECREF(callable); + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + PyObject *callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + if (PyFunction_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_NON_PY_GENERAL: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *null; PyObject *callable; @@ -3237,8 +3425,6 @@ break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _CALL_TYPE_1: { PyObject *arg; PyObject *null; @@ -4053,7 +4239,6 @@ #ifndef _Py_JIT next_uop = ¤t_executor->trace[1]; #endif - CHECK_EVAL_BREAKER(); break; } @@ -4089,10 +4274,7 @@ } case _EXIT_TRACE: { - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + EXIT_TO_TRACE(); break; } @@ -4281,11 +4463,6 @@ break; } - case _SIDE_EXIT: { - EXIT_TO_TRACE(); - break; - } - case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND(); @@ -4308,17 +4485,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); - break; - } - - case _EVAL_BREAKER_EXIT: { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_HandlePending(tstate) != 0) { - GOTO_UNWIND(); - } - EXIT_TO_TRACE(); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); break; } diff --git a/Python/fileutils.c b/Python/fileutils.c index 54853ba2f75d9d..e6a5391a3a28b5 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -3050,3 +3050,52 @@ _Py_GetTicksPerSecond(long *ticks_per_second) return 0; } #endif + + +/* Check if a file descriptor is valid or not. + Return 0 if the file descriptor is invalid, return non-zero otherwise. */ +int +_Py_IsValidFD(int fd) +{ +/* dup() is faster than fstat(): fstat() can require input/output operations, + whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python + startup. Problem: dup() doesn't check if the file descriptor is valid on + some platforms. + + fcntl(fd, F_GETFD) is even faster, because it only checks the process table. + It is preferred over dup() when available, since it cannot fail with the + "too many open files" error (EMFILE). + + bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other + side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with + EBADF. FreeBSD has similar issue (bpo-32849). + + Only use dup() on Linux where dup() is enough to detect invalid FD + (bpo-32849). +*/ + if (fd < 0) { + return 0; + } +#if defined(F_GETFD) && ( \ + defined(__linux__) || \ + defined(__APPLE__) || \ + (defined(__wasm__) && !defined(__wasi__))) + return fcntl(fd, F_GETFD) >= 0; +#elif defined(__linux__) + int fd2 = dup(fd); + if (fd2 >= 0) { + close(fd2); + } + return (fd2 >= 0); +#elif defined(MS_WINDOWS) + HANDLE hfile; + _Py_BEGIN_SUPPRESS_IPH + hfile = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + return (hfile != INVALID_HANDLE_VALUE + && GetFileType(hfile) != FILE_TYPE_UNKNOWN); +#else + struct stat st; + return (fstat(fd, &st) == 0); +#endif +} diff --git a/Python/frame.c b/Python/frame.c index db9d13359a23ca..2bb12823572028 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -94,6 +94,18 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) } } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame) +{ + assert(frame->stacktop >= 0); + int stacktop = frame->stacktop; + frame->stacktop = 0; + for (int i = 0; i < stacktop; i++) { + Py_XDECREF(frame->localsplus[i]); + } + Py_CLEAR(frame->f_locals); +} + void _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) { @@ -114,11 +126,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) } Py_DECREF(f); } - assert(frame->stacktop >= 0); - for (int i = 0; i < frame->stacktop; i++) { - Py_XDECREF(frame->localsplus[i]); - } - Py_XDECREF(frame->f_locals); + _PyFrame_ClearLocals(frame); Py_DECREF(frame->f_funcobj); } diff --git a/Python/future.c b/Python/future.c index 399345bd8fcbd9..8d94d515605dcd 100644 --- a/Python/future.c +++ b/Python/future.c @@ -77,7 +77,7 @@ future_parse(_PyFutureFeatures *ff, mod_ty mod, PyObject *filename) * are another future statement and a doc string. */ - if (s->kind == ImportFrom_kind) { + if (s->kind == ImportFrom_kind && s->v.ImportFrom.level == 0) { identifier modname = s->v.ImportFrom.module; if (modname && _PyUnicode_EqualToASCIIString(modname, "__future__")) { diff --git a/Python/gc.c b/Python/gc.c index a48738835fface..aa8b216124c36a 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -12,7 +12,6 @@ #include "pycore_object_alloc.h" // _PyObject_MallocWithType() #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index ef6aaad1252311..ee006bb4aa12b7 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -11,7 +11,6 @@ #include "pycore_object_stack.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_GetPerfCounter() #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" @@ -1164,7 +1163,8 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_PerfCounterUnchecked(); + // ignore error: don't interrupt the GC if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); } if (PyDTrace_GC_START_ENABLED()) { @@ -1184,7 +1184,9 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) n = state.uncollectable; if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + double d = PyTime_AsSecondsDouble(t2 - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c27505fde3d9fa..8b8112209cc78a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1002,6 +1002,97 @@ DISPATCH(); } + TARGET(CALL_BOUND_METHOD_GENERAL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *null; + PyObject *callable; + PyObject *method; + PyObject *self; + PyObject **args; + PyObject *self_or_null; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_METHOD_VERSION + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(null != NULL, CALL); + } + // _EXPAND_METHOD + { + assert(null == NULL); + assert(Py_TYPE(callable) == &PyMethod_Type); + self = ((PyMethodObject *)callable)->im_self; + Py_INCREF(self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = ((PyMethodObject *)callable)->im_func; + assert(PyFunction_Check(method)); + Py_INCREF(method); + Py_DECREF(callable); + } + // _PY_FRAME_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = self; + callable = method; + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1713,6 +1804,56 @@ DISPATCH(); } + TARGET(CALL_NON_PY_GENERAL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *callable; + PyObject **args; + PyObject *self_or_null; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(PyFunction_Check(callable), CALL); + DEOPT_IF(Py_TYPE(callable) == &PyMethod_Type, CALL); + } + // _CALL_NON_PY_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + TARGET(CALL_PY_EXACT_ARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -1786,50 +1927,76 @@ DISPATCH(); } - TARGET(CALL_PY_WITH_DEFAULTS) { + TARGET(CALL_PY_GENERAL) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *callable; PyObject **args; PyObject *self_or_null; - PyObject *callable; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { + goto error; + } } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(CALL_STR_1) { @@ -2492,22 +2659,31 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - int prevoparg = oparg; - CHECK_EVAL_BREAKER(); - if (this_instr->op.code != ENTER_EXECUTOR || - this_instr->op.arg != prevoparg) { - next_instr = this_instr; - DISPATCH(); - } + #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } tstate->previous_executor = Py_None; Py_INCREF(executor); GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ DISPATCH(); } @@ -3286,7 +3462,7 @@ INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } @@ -3432,6 +3608,7 @@ CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); + #ifdef _Py_TIER2 #if ENABLE_SPECIALIZATION _Py_BackoffCounter counter = this_instr[1].counter; if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { @@ -3457,6 +3634,7 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ DISPATCH(); } @@ -4940,22 +5118,31 @@ _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - PyCodeObject *code = _PyFrame_GetCode(frame); - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(code, tstate->interp); - if (err) goto error; - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + PyCodeObject* code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + DISPATCH(); + } + assert(this_instr->op.code == RESUME || + this_instr->op.code == RESUME_CHECK || + this_instr->op.code == INSTRUMENTED_RESUME || + this_instr->op.code == ENTER_EXECUTOR); + if (this_instr->op.code == RESUME) { + #if ENABLE_SPECIALIZATION + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ } - this_instr->op.code = RESUME_CHECK; + } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } DISPATCH(); } @@ -6012,31 +6199,43 @@ next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; + PyObject *value; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + #if TIER_ONE assert(frame != &entry_frame); - frame->instr_ptr = next_instr; + #endif + frame->instr_ptr++; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); } #undef TIER_ONE diff --git a/Python/import.c b/Python/import.c index 447114ab115bc7..ba44477318d473 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,6 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" +#include "pycore_ceval.h" #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -13,7 +14,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() -#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() +#include "pycore_time.h" // _PyTime_AsMicroseconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -34,6 +35,17 @@ module _imp #include "clinic/import.c.h" +#ifndef NDEBUG +static bool +is_interpreter_isolated(PyInterpreterState *interp) +{ + return !_Py_IsMainInterpreter(interp) + && !(interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) + && interp->ceval.own_gil; +} +#endif + + /*******************************/ /* process-global import state */ /*******************************/ @@ -435,6 +447,45 @@ _PyImport_GetNextModuleIndex(void) return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1; } +#ifndef NDEBUG +struct extensions_cache_value; +static struct extensions_cache_value * _find_cached_def(PyModuleDef *); +static Py_ssize_t _get_cached_module_index(struct extensions_cache_value *); +#endif + +static Py_ssize_t +_get_module_index_from_def(PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + assert(index > 0); +#ifndef NDEBUG + struct extensions_cache_value *cached = _find_cached_def(def); + assert(cached == NULL || index == _get_cached_module_index(cached)); +#endif + return index; +} + +static void +_set_module_index(PyModuleDef *def, Py_ssize_t index) +{ + assert(index > 0); + if (index == def->m_base.m_index) { + /* There's nothing to do. */ + } + else if (def->m_base.m_index == 0) { + /* It should have been initialized by PyModuleDef_Init(). + * We assert here to catch this in dev, but keep going otherwise. */ + assert(def->m_base.m_index != 0); + def->m_base.m_index = index; + } + else { + /* It was already set for a different module. + * We replace the old value. */ + assert(def->m_base.m_index > 0); + def->m_base.m_index = index; + } +} + static const char * _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) { @@ -451,9 +502,8 @@ _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) } static PyObject * -_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_get(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; if (_modules_by_index_check(interp, index) != NULL) { return NULL; } @@ -463,11 +513,9 @@ _modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) static int _modules_by_index_set(PyInterpreterState *interp, - PyModuleDef *def, PyObject *module) + Py_ssize_t index, PyObject *module) { - assert(def != NULL); - assert(def->m_slots == NULL); - assert(def->m_base.m_index > 0); + assert(index > 0); if (MODULES_BY_INDEX(interp) == NULL) { MODULES_BY_INDEX(interp) = PyList_New(0); @@ -476,7 +524,6 @@ _modules_by_index_set(PyInterpreterState *interp, } } - Py_ssize_t index = def->m_base.m_index; while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) { if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) { return -1; @@ -487,9 +534,8 @@ _modules_by_index_set(PyInterpreterState *interp, } static int -_modules_by_index_clear_one(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_clear_one(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; const char *err = _modules_by_index_check(interp, index); if (err != NULL) { Py_FatalError(err); @@ -506,7 +552,8 @@ PyState_FindModule(PyModuleDef* module) if (module->m_slots) { return NULL; } - return _modules_by_index_get(interp, module); + Py_ssize_t index = _get_module_index_from_def(module); + return _modules_by_index_get(interp, index); } /* _PyState_AddModule() has been completely removed from the C-API @@ -526,7 +573,9 @@ _PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) "PyState_AddModule called on module with slots"); return -1; } - return _modules_by_index_set(tstate->interp, def, module); + assert(def->m_slots == NULL); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_set(tstate->interp, index, module); } int @@ -546,7 +595,7 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) } PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; + Py_ssize_t index = _get_module_index_from_def(def); if (MODULES_BY_INDEX(interp) && index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) && module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index)) @@ -555,7 +604,8 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) return -1; } - return _modules_by_index_set(interp, def, module); + assert(def->m_slots == NULL); + return _modules_by_index_set(interp, index, module); } int @@ -568,7 +618,8 @@ PyState_RemoveModule(PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - return _modules_by_index_clear_one(tstate->interp, def); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_clear_one(tstate->interp, index); } @@ -587,6 +638,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) /* cleanup the saved copy of module dicts */ PyModuleDef *md = PyModule_GetDef(m); if (md) { + // XXX Do this more carefully. The dict might be owned + // by another interpreter. Py_CLEAR(md->m_base.m_copy); } } @@ -645,33 +698,33 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) K. PyModule_CreateInitialized() -> PyModule_SetDocString() L. PyModule_CreateInitialized(): set mod->md_def M. : initialize the module, etc. - N. _PyImport_RunModInitFunc(): set def->m_base.m_init - O. import_run_extension() + N. import_run_extension() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - P. import_run_extension(): set __file__ - Q. import_run_extension() -> update_global_state_for_extension() - R. update_global_state_for_extension(): + O. import_run_extension(): set __file__ + P. import_run_extension() -> update_global_state_for_extension() + Q. update_global_state_for_extension(): copy __dict__ into def->m_base.m_copy - S. update_global_state_for_extension(): + R. update_global_state_for_extension(): add it to _PyRuntime.imports.extensions - T. import_run_extension() -> finish_singlephase_extension() - U. finish_singlephase_extension(): + S. import_run_extension() -> finish_singlephase_extension() + T. finish_singlephase_extension(): add it to interp->imports.modules_by_index - V. finish_singlephase_extension(): add it to sys.modules + U. finish_singlephase_extension(): add it to sys.modules Step (Q) is skipped for core modules (sys/builtins). (6). subsequent times (found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. import_find_extension() - -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - C. import_find_extension() -> import_add_module() - D. if name in sys.modules: use that module - E. else: - 1. import_add_module() -> PyModule_NewObject() - 2. import_add_module(): set it on sys.modules - F. import_find_extension(): copy the "m_copy" dict into __dict__ - G. import_find_extension(): add to modules_by_index + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension() -> import_add_module() + E. if name in sys.modules: use that module + F. else: + 1. import_add_module() -> PyModule_NewObject() + 2. import_add_module(): set it on sys.modules + G. reload_singlephase_extension(): copy the "m_copy" dict into __dict__ + H. reload_singlephase_extension(): add to modules_by_index (10). (every time): A. noop @@ -681,21 +734,23 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions): A-P. (same as for m_size == -1) - Q-S. (skipped) - T-V. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R. (skipped) + S-U. (same as for m_size == -1) (6). main interpreter - first time (not found in _PyRuntime.imports.extensions): - A-R. (same as for m_size == -1) - S. (skipped) - T-V. (same as for m_size == -1) + A-P. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R-U. (same as for m_size == -1) (6). subsequent times (found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. import_find_extension() - -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - C. import_find_extension(): call def->m_base.m_init (see above) - D. import_find_extension(): add the module to sys.modules - E. import_find_extension(): add to modules_by_index + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension(): call def->m_base.m_init (see above) + E. reload_singlephase_extension(): add the module to sys.modules + F. reload_singlephase_extension(): add to modules_by_index (10). every time: A. noop @@ -922,6 +977,7 @@ extensions_lock_release(void) PyMutex_Unlock(&_PyRuntime.imports.extensions.mutex); } + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by the tuple @@ -938,6 +994,220 @@ extensions_lock_release(void) dictionary, to avoid loading shared libraries twice. */ +typedef struct cached_m_dict { + /* A shallow copy of the original module's __dict__. */ + PyObject *copied; + /* The interpreter that owns the copy. */ + int64_t interpid; +} *cached_m_dict_t; + +struct extensions_cache_value { + PyModuleDef *def; + + /* The function used to re-initialize the module. + This is only set for legacy (single-phase init) extension modules + and only used for those that support multiple initializations + (m_size >= 0). + It is set by update_global_state_for_extension(). */ + PyModInitFunction m_init; + + /* The module's index into its interpreter's modules_by_index cache. + This is set for all extension modules but only used for legacy ones. + (See PyInterpreterState.modules_by_index for more info.) */ + Py_ssize_t m_index; + + /* A copy of the module's __dict__ after the first time it was loaded. + This is only set/used for legacy modules that do not support + multiple initializations. + It is set exclusively by fixup_cached_def(). */ + cached_m_dict_t m_dict; + struct cached_m_dict _m_dict; + + _Py_ext_module_origin origin; + +#ifdef Py_GIL_DISABLED + /* The module's md_gil slot, for legacy modules that are reinitialized from + m_dict rather than calling their initialization function again. */ + void *md_gil; +#endif +}; + +static struct extensions_cache_value * +alloc_extensions_cache_value(void) +{ + struct extensions_cache_value *value + = PyMem_RawMalloc(sizeof(struct extensions_cache_value)); + if (value == NULL) { + PyErr_NoMemory(); + return NULL; + } + *value = (struct extensions_cache_value){0}; + return value; +} + +static void +free_extensions_cache_value(struct extensions_cache_value *value) +{ + PyMem_RawFree(value); +} + +static Py_ssize_t +_get_cached_module_index(struct extensions_cache_value *cached) +{ + assert(cached->m_index > 0); + return cached->m_index; +} + +static void +fixup_cached_def(struct extensions_cache_value *value) +{ + /* For the moment, the values in the def's m_base may belong + * to another module, and we're replacing them here. This can + * cause problems later if the old module is reloaded. + * + * Also, we don't decref any old cached values first when we + * replace them here, in case we need to restore them in the + * near future. Instead, the caller is responsible for wrapping + * this up by calling cleanup_old_cached_def() or + * restore_old_cached_def() if there was an error. */ + PyModuleDef *def = value->def; + assert(def != NULL); + + /* We assume that all module defs are statically allocated + and will never be freed. Otherwise, we would incref here. */ + _Py_SetImmortalUntracked((PyObject *)def); + + def->m_base.m_init = value->m_init; + + assert(value->m_index > 0); + _set_module_index(def, value->m_index); + + /* Different modules can share the same def, so we can't just + * expect m_copy to be NULL. */ + assert(def->m_base.m_copy == NULL + || def->m_base.m_init == NULL + || value->m_dict != NULL); + if (value->m_dict != NULL) { + assert(value->m_dict->copied != NULL); + /* As noted above, we don't first decref the old value, if any. */ + def->m_base.m_copy = Py_NewRef(value->m_dict->copied); + } +} + +static void +restore_old_cached_def(PyModuleDef *def, PyModuleDef_Base *oldbase) +{ + def->m_base = *oldbase; +} + +static void +cleanup_old_cached_def(PyModuleDef_Base *oldbase) +{ + Py_XDECREF(oldbase->m_copy); +} + +static void +del_cached_def(struct extensions_cache_value *value) +{ + /* If we hadn't made the stored defs immortal, we would decref here. + However, this decref would be problematic if the module def were + dynamically allocated, it were the last ref, and this function + were called with an interpreter other than the def's owner. */ + assert(value->def == NULL || _Py_IsImmortal(value->def)); + + Py_XDECREF(value->def->m_base.m_copy); + value->def->m_base.m_copy = NULL; +} + +static int +init_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict) +{ + assert(value != NULL); + /* This should only have been called without an m_dict already set. */ + assert(value->m_dict == NULL); + if (m_dict == NULL) { + return 0; + } + assert(PyDict_Check(m_dict)); + assert(value->origin != _Py_ext_module_origin_CORE); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!is_interpreter_isolated(interp)); + + /* XXX gh-88216: The copied dict is owned by the current + * interpreter. That's a problem if the interpreter has + * its own obmalloc state or if the module is successfully + * imported into such an interpreter. If the interpreter + * has its own GIL then there may be data races and + * PyImport_ClearModulesByIndex() can crash. Normally, + * a single-phase init module cannot be imported in an + * isolated interpreter, but there are ways around that. + * Hence, heere be dragons! Ideally we would instead do + * something like make a read-only, immortal copy of the + * dict using PyMem_RawMalloc() and store *that* in m_copy. + * Then we'd need to make sure to clear that when the + * runtime is finalized, rather than in + * PyImport_ClearModulesByIndex(). */ + PyObject *copied = PyDict_Copy(m_dict); + if (copied == NULL) { + /* We expect this can only be "out of memory". */ + return -1; + } + // XXX We may want to make the copy immortal. + + value->_m_dict = (struct cached_m_dict){ + .copied=copied, + .interpid=PyInterpreterState_GetID(interp), + }; + + value->m_dict = &value->_m_dict; + return 0; +} + +static void +del_cached_m_dict(struct extensions_cache_value *value) +{ + if (value->m_dict != NULL) { + assert(value->m_dict == &value->_m_dict); + assert(value->m_dict->copied != NULL); + /* In the future we can take advantage of m_dict->interpid + * to decref the dict using the owning interpreter. */ + Py_XDECREF(value->m_dict->copied); + value->m_dict = NULL; + } +} + +static PyObject * get_core_module_dict( + PyInterpreterState *interp, PyObject *name, PyObject *path); + +static PyObject * +get_cached_m_dict(struct extensions_cache_value *value, + PyObject *name, PyObject *path) +{ + assert(value != NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + /* It might be a core module (e.g. sys & builtins), + for which we don't cache m_dict. */ + if (value->origin == _Py_ext_module_origin_CORE) { + return get_core_module_dict(interp, name, path); + } + assert(value->def != NULL); + // XXX Switch to value->m_dict. + PyObject *m_dict = value->def->m_base.m_copy; + Py_XINCREF(m_dict); + return m_dict; +} + +static void +del_extensions_cache_value(struct extensions_cache_value *value) +{ + if (value != NULL) { + del_cached_m_dict(value); + del_cached_def(value); + free_extensions_cache_value(value); + } +} + static void * hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) { @@ -951,6 +1221,7 @@ hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) assert(SIZE_MAX - str1_len - str2_len > 2); size_t size = str1_len + 1 + str2_len + 1; + // XXX Use a buffer if it's a temp value (every case but "set"). char *key = PyMem_RawMalloc(size); if (key == NULL) { PyErr_NoMemory(); @@ -982,102 +1253,220 @@ hashtable_destroy_str(void *ptr) PyMem_RawFree(ptr); } -#define HTSEP ':' +#ifndef NDEBUG +struct hashtable_next_match_def_data { + PyModuleDef *def; + struct extensions_cache_value *matched; +}; -static PyModuleDef * -_extensions_cache_get(PyObject *filename, PyObject *name) +static int +hashtable_next_match_def(_Py_hashtable_t *ht, + const void *key, const void *value, void *user_data) { - PyModuleDef *def = NULL; - void *key = NULL; - extensions_lock_acquire(); + if (value == NULL) { + /* It was previously deleted. */ + return 0; + } + struct hashtable_next_match_def_data *data + = (struct hashtable_next_match_def_data *)user_data; + struct extensions_cache_value *cur + = (struct extensions_cache_value *)value; + if (cur->def == data->def) { + data->matched = cur; + return 1; + } + return 0; +} + +static struct extensions_cache_value * +_find_cached_def(PyModuleDef *def) +{ + struct hashtable_next_match_def_data data = {0}; + (void)_Py_hashtable_foreach( + EXTENSIONS.hashtable, hashtable_next_match_def, &data); + return data.matched; +} +#endif +#define HTSEP ':' + +static int +_extensions_cache_init(void) +{ + _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; + EXTENSIONS.hashtable = _Py_hashtable_new_full( + hashtable_hash_str, + hashtable_compare_str, + hashtable_destroy_str, // key + (_Py_hashtable_destroy_func)del_extensions_cache_value, // value + &alloc + ); if (EXTENSIONS.hashtable == NULL) { - goto finally; + PyErr_NoMemory(); + return -1; } + return 0; +} - key = hashtable_key_from_2_strings(filename, name, HTSEP); +static _Py_hashtable_entry_t * +_extensions_cache_find_unlocked(PyObject *path, PyObject *name, + void **p_key) +{ + if (EXTENSIONS.hashtable == NULL) { + return NULL; + } + void *key = hashtable_key_from_2_strings(path, name, HTSEP); if (key == NULL) { - goto finally; + return NULL; + } + _Py_hashtable_entry_t *entry = + _Py_hashtable_get_entry(EXTENSIONS.hashtable, key); + if (p_key != NULL) { + *p_key = key; } - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + else { + hashtable_destroy_str(key); + } + return entry; +} + +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * +_extensions_cache_get(PyObject *path, PyObject *name) +{ + struct extensions_cache_value *value = NULL; + extensions_lock_acquire(); + + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { + /* It was never added. */ goto finally; } - def = (PyModuleDef *)entry->value; + value = (struct extensions_cache_value *)entry->value; finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } - return def; + return value; } -static int -_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * +_extensions_cache_set(PyObject *path, PyObject *name, + PyModuleDef *def, PyModInitFunction m_init, + Py_ssize_t m_index, PyObject *m_dict, + _Py_ext_module_origin origin, void *md_gil) { - int res = -1; + struct extensions_cache_value *value = NULL; + void *key = NULL; + struct extensions_cache_value *newvalue = NULL; + PyModuleDef_Base olddefbase = def->m_base; + + assert(def != NULL); + assert(m_init == NULL || m_dict == NULL); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(def->m_base.m_init == NULL || def->m_base.m_init == m_init); + /* For now we don't worry about comparing value->m_copy. */ + assert(def->m_base.m_copy == NULL || m_dict != NULL); + assert((origin == _Py_ext_module_origin_DYNAMIC) == (name != path)); + assert(origin != _Py_ext_module_origin_CORE || m_dict == NULL); + extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { - _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; - EXTENSIONS.hashtable = _Py_hashtable_new_full( - hashtable_hash_str, - hashtable_compare_str, - hashtable_destroy_str, // key - /* There's no need to decref the def since it's immortal. */ - NULL, // value - &alloc - ); - if (EXTENSIONS.hashtable == NULL) { - PyErr_NoMemory(); + if (_extensions_cache_init() < 0) { goto finally; } } - void *key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { + /* Create a cached value to populate for the module. */ + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, &key); + value = entry == NULL + ? NULL + : (struct extensions_cache_value *)entry->value; + /* We should never be updating an existing cache value. */ + assert(value == NULL); + if (value != NULL) { + PyErr_Format(PyExc_SystemError, + "extension module %R is already cached", name); goto finally; } + newvalue = alloc_extensions_cache_value(); + if (newvalue == NULL) { + goto finally; + } + + /* Populate the new cache value data. */ + *newvalue = (struct extensions_cache_value){ + .def=def, + .m_init=m_init, + .m_index=m_index, + /* m_dict is set by set_cached_m_dict(). */ + .origin=origin, +#ifdef Py_GIL_DISABLED + .md_gil=md_gil, +#endif + }; +#ifndef Py_GIL_DISABLED + (void)md_gil; +#endif + if (init_cached_m_dict(newvalue, m_dict) < 0) { + goto finally; + } + fixup_cached_def(newvalue); - int already_set = 0; - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); if (entry == NULL) { - if (_Py_hashtable_set(EXTENSIONS.hashtable, key, def) < 0) { - PyMem_RawFree(key); + /* It was never added. */ + if (_Py_hashtable_set(EXTENSIONS.hashtable, key, newvalue) < 0) { PyErr_NoMemory(); goto finally; } + /* The hashtable owns the key now. */ + key = NULL; + } + else if (value == NULL) { + /* It was previously deleted. */ + entry->value = newvalue; } else { - if (entry->value == NULL) { - entry->value = def; - } - else { - /* We expect it to be static, so it must be the same pointer. */ - assert((PyModuleDef *)entry->value == def); - already_set = 1; + /* We are updating the entry for an existing module. */ + /* We expect def to be static, so it must be the same pointer. */ + assert(value->def == def); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(value->m_init == m_init); + /* The same module can't switch between caching __dict__ and not. */ + assert((value->m_dict == NULL) == (m_dict == NULL)); + /* This shouldn't ever happen. */ + Py_UNREACHABLE(); + } + + value = newvalue; + +finally: + if (value == NULL) { + restore_old_cached_def(def, &olddefbase); + if (newvalue != NULL) { + del_extensions_cache_value(newvalue); } - PyMem_RawFree(key); } - if (!already_set) { - /* We assume that all module defs are statically allocated - and will never be freed. Otherwise, we would incref here. */ - _Py_SetImmortal((PyObject *)def); + else { + cleanup_old_cached_def(&olddefbase); } - res = 0; -finally: extensions_lock_release(); - return res; + if (key != NULL) { + hashtable_destroy_str(key); + } + + return value; } static void -_extensions_cache_delete(PyObject *filename, PyObject *name) +_extensions_cache_delete(PyObject *path, PyObject *name) { - void *key = NULL; extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { @@ -1085,13 +1474,8 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) goto finally; } - key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { - goto finally; - } - - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { /* It was never added. */ goto finally; @@ -1100,18 +1484,13 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) /* It was already removed. */ goto finally; } - /* If we hadn't made the stored defs immortal, we would decref here. - However, this decref would be problematic if the module def were - dynamically allocated, it were the last ref, and this function - were called with an interpreter other than the def's owner. */ - assert(_Py_IsImmortal(entry->value)); + struct extensions_cache_value *value = entry->value; entry->value = NULL; + del_extensions_cache_value(value); + finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } } static void @@ -1157,6 +1536,67 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) return 0; } +#ifdef Py_GIL_DISABLED +int +_PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (module == NULL) { + _PyEval_DisableGIL(tstate); + return 0; + } + + if (!PyModule_Check(module) || + ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { + if (_PyEval_EnableGILPermanent(tstate)) { + int warn_result = PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "The global interpreter lock (GIL) has been enabled to load " + "module '%U', which has not declared that it can run safely " + "without the GIL. To override this behavior and keep the GIL " + "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.", + module_name + ); + if (warn_result < 0) { + return warn_result; + } + } + + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil == _PyConfig_GIL_DEFAULT && config->verbose) { + PySys_FormatStderr("# loading module '%U', which requires the GIL\n", + module_name); + } + } + else { + _PyEval_DisableGIL(tstate); + } + + return 0; +} +#endif + +static PyThreadState * +switch_to_main_interpreter(PyThreadState *tstate) +{ + if (_Py_IsMainInterpreter(tstate->interp)) { + return tstate; + } + PyThreadState *main_tstate = PyThreadState_New(_PyInterpreterState_Main()); + if (main_tstate == NULL) { + return NULL; + } + main_tstate->_whence = _PyThreadState_WHENCE_EXEC; +#ifndef NDEBUG + PyThreadState *old_tstate = PyThreadState_Swap(main_tstate); + assert(old_tstate == tstate); +#else + (void)PyThreadState_Swap(main_tstate); +#endif + return main_tstate; +} + static PyObject * get_core_module_dict(PyInterpreterState *interp, PyObject *name, PyObject *path) @@ -1165,11 +1605,11 @@ get_core_module_dict(PyInterpreterState *interp, if (path == name) { assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { - return interp->sysdict_copy; + return Py_NewRef(interp->sysdict_copy); } assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "builtins") == 0) { - return interp->builtins_copy; + return Py_NewRef(interp->builtins_copy); } assert(!PyErr_Occurred()); } @@ -1192,44 +1632,107 @@ is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *path) return 0; } + #ifndef NDEBUG -static bool -is_singlephase(PyModuleDef *def) +static _Py_ext_module_kind +_get_extension_kind(PyModuleDef *def, bool check_size) { + _Py_ext_module_kind kind; if (def == NULL) { /* It must be a module created by reload_singlephase_extension() * from m_copy. Ideally we'd do away with this case. */ - return true; + kind = _Py_ext_module_kind_SINGLEPHASE; } - else if (def->m_slots == NULL) { - return true; + else if (def->m_slots != NULL) { + kind = _Py_ext_module_kind_MULTIPHASE; } - else { - return false; + else if (check_size && def->m_size == -1) { + kind = _Py_ext_module_kind_SINGLEPHASE; } -} + else if (def->m_base.m_init != NULL) { + kind = _Py_ext_module_kind_SINGLEPHASE; + } + else { + // This is probably single-phase init, but a multi-phase + // module *can* have NULL m_slots. + kind = _Py_ext_module_kind_UNKNOWN; + } + return kind; +} + +/* The module might not be fully initialized yet + * and PyModule_FromDefAndSpec() checks m_size + * so we skip m_size. */ +#define assert_multiphase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, false); \ + assert(kind == _Py_ext_module_kind_MULTIPHASE \ + /* m_slots can be NULL. */ \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase(cached) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(cached->def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE); \ + } while (0) + +#else /* defined(NDEBUG) */ +#define assert_multiphase_def(def) +#define assert_singlephase_def(def) +#define assert_singlephase(cached) #endif struct singlephase_global_update { + PyModInitFunction m_init; + Py_ssize_t m_index; PyObject *m_dict; + _Py_ext_module_origin origin; + void *md_gil; }; -static int +static struct extensions_cache_value * update_global_state_for_extension(PyThreadState *tstate, PyObject *path, PyObject *name, PyModuleDef *def, struct singlephase_global_update *singlephase) { - /* Copy the module's __dict__, if applicable. */ + struct extensions_cache_value *cached = NULL; + PyModInitFunction m_init = NULL; + PyObject *m_dict = NULL; + + /* Set up for _extensions_cache_set(). */ if (singlephase == NULL) { + assert(def->m_base.m_init == NULL); assert(def->m_base.m_copy == NULL); } else { - assert(def->m_base.m_init != NULL - || is_core_module(tstate->interp, name, path)); - if (singlephase->m_dict == NULL) { + if (singlephase->m_init != NULL) { + assert(singlephase->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + assert(def->m_size >= 0); + /* Remember pointer to module init function. */ + // XXX If two modules share a def then def->m_base will + // reflect the last one added (here) to the global cache. + // We should prevent this somehow. The simplest solution + // is probably to store m_copy/m_init in the cache along + // with the def, rather than within the def. + m_init = singlephase->m_init; + } + else if (singlephase->m_dict == NULL) { + /* It must be a core builtin module. */ + assert(is_core_module(tstate->interp, name, path)); + assert(def->m_size == -1); assert(def->m_base.m_copy == NULL); + assert(def->m_base.m_init == NULL); } else { assert(PyDict_Check(singlephase->m_dict)); @@ -1239,32 +1742,7 @@ update_global_state_for_extension(PyThreadState *tstate, assert(!is_core_module(tstate->interp, name, path)); assert(PyUnicode_CompareWithASCIIString(name, "sys") != 0); assert(PyUnicode_CompareWithASCIIString(name, "builtins") != 0); - /* XXX gh-88216: The copied dict is owned by the current - * interpreter. That's a problem if the interpreter has - * its own obmalloc state or if the module is successfully - * imported into such an interpreter. If the interpreter - * has its own GIL then there may be data races and - * PyImport_ClearModulesByIndex() can crash. Normally, - * a single-phase init module cannot be imported in an - * isolated interpreter, but there are ways around that. - * Hence, heere be dragons! Ideally we would instead do - * something like make a read-only, immortal copy of the - * dict using PyMem_RawMalloc() and store *that* in m_copy. - * Then we'd need to make sure to clear that when the - * runtime is finalized, rather than in - * PyImport_ClearModulesByIndex(). */ - if (def->m_base.m_copy) { - /* Somebody already imported the module, - likely under a different name. - XXX this should really not happen. */ - Py_CLEAR(def->m_base.m_copy); - } - def->m_base.m_copy = PyDict_Copy(singlephase->m_dict); - if (def->m_base.m_copy == NULL) { - // XXX Ignore this error? Doing so would effectively - // mark the module as not loadable. */ - return -1; - } + m_dict = singlephase->m_dict; } } @@ -1272,28 +1750,34 @@ update_global_state_for_extension(PyThreadState *tstate, // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { #ifndef NDEBUG - PyModuleDef *cached = _extensions_cache_get(path, name); - assert(cached == NULL || cached == def); + cached = _extensions_cache_get(path, name); + assert(cached == NULL || cached->def == def); #endif - if (_extensions_cache_set(path, name, def) < 0) { - return -1; + cached = _extensions_cache_set( + path, name, def, m_init, singlephase->m_index, m_dict, + singlephase->origin, singlephase->md_gil); + if (cached == NULL) { + // XXX Ignore this error? Doing so would effectively + // mark the module as not loadable. + return NULL; } } - return 0; + return cached; } /* For multi-phase init modules, the module is finished * by PyModule_FromDefAndSpec(). */ static int -finish_singlephase_extension(PyThreadState *tstate, - PyObject *mod, PyModuleDef *def, +finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, + struct extensions_cache_value *cached, PyObject *name, PyObject *modules) { assert(mod != NULL && PyModule_Check(mod)); - assert(def == _PyModule_GetDef(mod)); + assert(cached->def == _PyModule_GetDef(mod)); - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { return -1; } @@ -1308,15 +1792,14 @@ finish_singlephase_extension(PyThreadState *tstate, static PyObject * -import_find_extension(PyThreadState *tstate, - struct _Py_ext_module_loader_info *info) +reload_singlephase_extension(PyThreadState *tstate, + struct extensions_cache_value *cached, + struct _Py_ext_module_loader_info *info) { - /* Only single-phase init modules will be in the cache. */ - PyModuleDef *def = _extensions_cache_get(info->path, info->name); - if (def == NULL) { - return NULL; - } - assert(is_singlephase(def)); + PyModuleDef *def = cached->def; + assert(def != NULL); + assert_singlephase(cached); + PyObject *mod = NULL; /* It may have been successfully imported previously in an interpreter that allows legacy modules @@ -1327,69 +1810,133 @@ import_find_extension(PyThreadState *tstate, return NULL; } - PyObject *mod, *mdict; PyObject *modules = get_modules_dict(tstate, true); - if (def->m_size == -1) { - PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ + assert(cached->m_init == NULL); + assert(def->m_base.m_init == NULL); + // XXX Copying the cached dict may break interpreter isolation. + // We could solve this by temporarily acquiring the original + // interpreter's GIL. + PyObject *m_copy = get_cached_m_dict(cached, info->name, info->path); if (m_copy == NULL) { - /* It might be a core module (e.g. sys & builtins), - for which we don't set m_copy. */ - m_copy = get_core_module_dict( - tstate->interp, info->name, info->path); - if (m_copy == NULL) { - return NULL; - } + assert(!PyErr_Occurred()); + return NULL; } mod = import_add_module(tstate, info->name); if (mod == NULL) { + Py_DECREF(m_copy); return NULL; } - mdict = PyModule_GetDict(mod); + PyObject *mdict = PyModule_GetDict(mod); if (mdict == NULL) { + Py_DECREF(m_copy); Py_DECREF(mod); return NULL; } - if (PyDict_Update(mdict, m_copy)) { + int rc = PyDict_Update(mdict, m_copy); + Py_DECREF(m_copy); + if (rc < 0) { Py_DECREF(mod); return NULL; } +#ifdef Py_GIL_DISABLED + if (def->m_base.m_copy != NULL) { + // For non-core modules, fetch the GIL slot that was stored by + // import_run_extension(). + ((PyModuleObject *)mod)->md_gil = cached->md_gil; + } +#endif /* We can't set mod->md_def if it's missing, * because _PyImport_ClearModulesByIndex() might break - * due to violating interpreter isolation. See the note - * in fix_up_extension_for_interpreter(). Until that - * is solved, we leave md_def set to NULL. */ + * due to violating interpreter isolation. + * See the note in set_cached_m_dict(). + * Until that is solved, we leave md_def set to NULL. */ assert(_PyModule_GetDef(mod) == NULL || _PyModule_GetDef(mod) == def); } else { - if (def->m_base.m_init == NULL) { + assert(cached->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + // XXX Use cached->m_init. + PyModInitFunction p0 = def->m_base.m_init; + if (p0 == NULL) { + assert(!PyErr_Occurred()); return NULL; } struct _Py_ext_module_loader_result res; - if (_PyImport_RunModInitFunc(def->m_base.m_init, info, &res) < 0) { + if (_PyImport_RunModInitFunc(p0, info, &res) < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); return NULL; } assert(!PyErr_Occurred()); + assert(res.err == NULL); + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); mod = res.module; - // XXX __file__ doesn't get set! + /* Tchnically, the init function could return a different module def. + * Then we would probably need to update the global cache. + * However, we don't expect anyone to change the def. */ + assert(res.def == def); + _Py_ext_module_loader_result_clear(&res); + + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + if (PyModule_AddObjectRef(mod, "__file__", info->filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + if (PyObject_SetItem(modules, info->name, mod) == -1) { Py_DECREF(mod); return NULL; } } - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { + + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { PyMapping_DelItem(modules, info->name); Py_DECREF(mod); return NULL; } + return mod; +} + +static PyObject * +import_find_extension(PyThreadState *tstate, + struct _Py_ext_module_loader_info *info, + struct extensions_cache_value **p_cached) +{ + /* Only single-phase init modules will be in the cache. */ + struct extensions_cache_value *cached + = _extensions_cache_get(info->path, info->name); + if (cached == NULL) { + return NULL; + } + assert(cached->def != NULL); + assert_singlephase(cached); + *p_cached = cached; + + /* It may have been successfully imported previously + in an interpreter that allows legacy modules + but is not allowed in the current interpreter. */ + const char *name_buf = PyUnicode_AsUTF8(info->name); + assert(name_buf != NULL); + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + return NULL; + } + + PyObject *mod = reload_singlephase_extension(tstate, cached, info); + if (mod == NULL) { + return NULL; + } + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; if (verbose) { PySys_FormatStderr("import %U # previously loaded (%R)\n", info->name, info->path); } + return mod; } @@ -1398,78 +1945,226 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct _Py_ext_module_loader_info *info, PyObject *spec, PyObject *modules) { + /* Core modules go through _PyImport_FixupBuiltin(). */ + assert(!is_core_module(tstate->interp, info->name, info->path)); + PyObject *mod = NULL; PyModuleDef *def = NULL; + struct extensions_cache_value *cached = NULL; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + + /* We cannot know if the module is single-phase init or + * multi-phase init until after we call its init function. Even + * in isolated interpreters (that do not support single-phase init), + * the init function will run without restriction. For multi-phase + * init modules that isn't a problem because the init function only + * runs PyModuleDef_Init() on the module's def and then returns it. + * + * However, for single-phase init the module's init function will + * create the module, create other objects (and allocate other + * memory), populate it and its module state, and initialze static + * types. Some modules store other objects and data in global C + * variables and register callbacks with the runtime/stdlib or + * even external libraries (which is part of why we can't just + * dlclose() the module in the error case). That's a problem + * for isolated interpreters since all of the above happens + * and only then * will the import fail. Memory will leak, + * callbacks will still get used, and sometimes there + * will be crashes (memory access violations + * and use-after-free). + * + * To put it another way, if the module is single-phase init + * then the import will probably break interpreter isolation + * and should fail ASAP. However, the module's init function + * will still get run. That means it may still store state + * in the shared-object/DLL address space (which never gets + * closed/cleared), including objects (e.g. static types). + * This is a problem for isolated subinterpreters since each + * has its own object allocator. If the loaded shared-object + * still holds a reference to an object after the corresponding + * interpreter has finalized then either we must let it leak + * or else any later use of that object by another interpreter + * (or across multiple init-fini cycles) will crash the process. + * + * To avoid all of that, we make sure the module's init function + * is always run first with the main interpreter active. If it was + * already the main interpreter then we can continue loading the + * module like normal. Otherwise, right after the init function, + * we take care of some import state bookkeeping, switch back + * to the subinterpreter, check for single-phase init, + * and then continue loading like normal. */ + + bool switched = false; + /* We *could* leave in place a legacy interpreter here + * (one that shares obmalloc/GIL with main interp), + * but there isn't a big advantage, we anticipate + * such interpreters will be increasingly uncommon, + * and the code is a bit simpler if we always switch + * to the main interpreter. */ + PyThreadState *main_tstate = switch_to_main_interpreter(tstate); + if (main_tstate == NULL) { + return NULL; + } + else if (main_tstate != tstate) { + switched = true; + /* In the switched case, we could play it safe + * by getting the main interpreter's import lock here. + * It's unlikely to matter though. */ + } struct _Py_ext_module_loader_result res; - if (_PyImport_RunModInitFunc(p0, info, &res) < 0) { + int rc = _PyImport_RunModInitFunc(p0, info, &res); + if (rc < 0) { /* We discard res.def. */ assert(res.module == NULL); - assert(PyErr_Occurred()); - goto finally; } - assert(!PyErr_Occurred()); + else { + assert(!PyErr_Occurred()); + assert(res.err == NULL); - mod = res.module; - res.module = NULL; - def = res.def; - assert(def != NULL); + mod = res.module; + res.module = NULL; + def = res.def; + assert(def != NULL); - if (mod == NULL) { - //assert(!is_singlephase(def)); + /* Do anything else that should be done + * while still using the main interpreter. */ + if (res.kind == _Py_ext_module_kind_SINGLEPHASE) { + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + // XXX There's a refleak somewhere with the filename. + // Until we can track it down, we intern it. + PyObject *filename = Py_NewRef(info->filename); + PyUnicode_InternInPlace(&filename); + if (PyModule_AddObjectRef(mod, "__file__", filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + + /* Update global import state. */ + assert(def->m_base.m_index != 0); + struct singlephase_global_update singlephase = { + // XXX Modules that share a def should each get their own index, + // whereas currently they share (which means the per-interpreter + // cache is less reliable than it should be). + .m_index=def->m_base.m_index, + .origin=info->origin, +#ifdef Py_GIL_DISABLED + .md_gil=((PyModuleObject *)mod)->md_gil, +#endif + }; + // gh-88216: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + if (def->m_size == -1) { + /* We will reload from m_copy. */ + assert(def->m_base.m_init == NULL); + singlephase.m_dict = PyModule_GetDict(mod); + assert(singlephase.m_dict != NULL); + } + else { + /* We will reload via the init function. */ + assert(def->m_size >= 0); + assert(def->m_base.m_copy == NULL); + singlephase.m_init = p0; + } + cached = update_global_state_for_extension( + tstate, info->path, info->name, def, &singlephase); + if (cached == NULL) { + assert(PyErr_Occurred()); + goto main_finally; + } + } + } + +main_finally: + /* Switch back to the subinterpreter. */ + if (switched) { + assert(main_tstate != tstate); + + /* Handle any exceptions, which we cannot propagate directly + * to the subinterpreter. */ + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* We trust it will be caught again soon. */ + PyErr_Clear(); + } + else { + /* Printing the exception should be sufficient. */ + PyErr_PrintEx(0); + } + } + + /* Any module we got from the init function will have to be + * reloaded in the subinterpreter. */ + Py_CLEAR(mod); + + PyThreadState_Clear(main_tstate); + (void)PyThreadState_Swap(tstate); + PyThreadState_Delete(main_tstate); + } + + /*****************************************************************/ + /* At this point we are back to the interpreter we started with. */ + /*****************************************************************/ + + /* Finally we handle the error return from _PyImport_RunModInitFunc(). */ + if (rc < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); + goto error; + } + + if (res.kind == _Py_ext_module_kind_MULTIPHASE) { + assert_multiphase_def(def); assert(mod == NULL); + /* Note that we cheat a little by not repeating the calls + * to _PyImport_GetModInitFunc() and _PyImport_RunModInitFunc(). */ mod = PyModule_FromDefAndSpec(def, spec); if (mod == NULL) { - goto finally; + goto error; } } else { - assert(is_singlephase(def)); - assert(PyModule_Check(mod)); + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); + assert_singlephase_def(def); - const char *name_buf = PyBytes_AS_STRING(info->name_encoded); if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { - Py_CLEAR(mod); - goto finally; + goto error; } + assert(!PyErr_Occurred()); - /* Remember pointer to module init function. */ - def->m_base.m_init = p0; - - if (info->filename != NULL) { - /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(mod, "__file__", info->filename) < 0) { - PyErr_Clear(); /* Not important enough to report */ + if (switched) { + /* We switched to the main interpreter to run the init + * function, so now we will "reload" the module from the + * cached data using the original subinterpreter. */ + assert(mod == NULL); + mod = reload_singlephase_extension(tstate, cached, info); + if (mod == NULL) { + goto error; } + assert(!PyErr_Occurred()); + assert(PyModule_Check(mod)); } - - struct singlephase_global_update singlephase = {0}; - // gh-88216: Extensions and def->m_base.m_copy can be updated - // when the extension module doesn't support sub-interpreters. - if (def->m_size == -1 - && !is_core_module(tstate->interp, info->name, info->path)) - { - singlephase.m_dict = PyModule_GetDict(mod); - assert(singlephase.m_dict != NULL); - } - if (update_global_state_for_extension( - tstate, info->path, info->name, def, &singlephase) < 0) - { - Py_CLEAR(mod); - goto finally; - } - - PyObject *modules = get_modules_dict(tstate, true); - if (finish_singlephase_extension( - tstate, mod, def, info->name, modules) < 0) - { - Py_CLEAR(mod); - goto finally; + else { + assert(mod != NULL); + assert(PyModule_Check(mod)); + + /* Update per-interpreter import state. */ + PyObject *modules = get_modules_dict(tstate, true); + if (finish_singlephase_extension( + tstate, mod, cached, info->name, modules) < 0) + { + goto error; + } } } -finally: + _Py_ext_module_loader_result_clear(&res); return mod; + +error: + Py_XDECREF(mod); + _Py_ext_module_loader_result_clear(&res); + return NULL; } @@ -1477,13 +2172,14 @@ static int clear_singlephase_extension(PyInterpreterState *interp, PyObject *name, PyObject *path) { - PyModuleDef *def = _extensions_cache_get(path, name); - if (def == NULL) { + struct extensions_cache_value *cached = _extensions_cache_get(path, name); + if (cached == NULL) { if (PyErr_Occurred()) { return -1; } return 0; } + PyModuleDef *def = cached->def; /* Clear data set when the module was initially loaded. */ def->m_base.m_init = NULL; @@ -1491,8 +2187,9 @@ clear_singlephase_extension(PyInterpreterState *interp, // We leave m_index alone since there's no reason to reset it. /* Clear the PyState_*Module() cache entry. */ - if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { - if (_modules_by_index_clear_one(interp, def) < 0) { + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_check(interp, index) == NULL) { + if (_modules_by_index_clear_one(interp, index) < 0) { return -1; } } @@ -1532,21 +2229,36 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, * module state, but we also don't populate def->m_base.m_copy * for them. */ assert(is_core_module(tstate->interp, nameobj, nameobj)); - assert(is_singlephase(def)); + assert_singlephase_def(def); assert(def->m_size == -1); assert(def->m_base.m_copy == NULL); - - struct singlephase_global_update singlephase = { - /* We don't want def->m_base.m_copy populated. */ - .m_dict=NULL, - }; - if (update_global_state_for_extension( - tstate, nameobj, nameobj, def, &singlephase) < 0) - { - goto finally; + assert(def->m_base.m_index >= 0); + + /* We aren't using import_find_extension() for core modules, + * so we have to do the extra check to make sure the module + * isn't already in the global cache before calling + * update_global_state_for_extension(). */ + struct extensions_cache_value *cached + = _extensions_cache_get(nameobj, nameobj); + if (cached == NULL) { + struct singlephase_global_update singlephase = { + .m_index=def->m_base.m_index, + /* We don't want def->m_base.m_copy populated. */ + .m_dict=NULL, + .origin=_Py_ext_module_origin_CORE, +#ifdef Py_GIL_DISABLED + /* Unused when m_dict == NULL. */ + .md_gil=NULL, +#endif + }; + cached = update_global_state_for_extension( + tstate, nameobj, nameobj, def, &singlephase); + if (cached == NULL) { + goto finally; + } } - if (finish_singlephase_extension(tstate, mod, def, nameobj, modules) < 0) { + if (finish_singlephase_extension(tstate, mod, cached, nameobj, modules) < 0) { goto finally; } @@ -1583,16 +2295,30 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return NULL; } - PyObject *mod = import_find_extension(tstate, &info); + struct extensions_cache_value *cached = NULL; + PyObject *mod = import_find_extension(tstate, &info, &cached); if (mod != NULL) { assert(!_PyErr_Occurred(tstate)); - assert(is_singlephase(_PyModule_GetDef(mod))); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); goto finally; } else if (_PyErr_Occurred(tstate)) { goto finally; } + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } + struct _inittab *found = NULL; for (struct _inittab *p = INITTAB; p->name != NULL; p++) { if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { @@ -1613,9 +2339,23 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) goto finally; } +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif /* Now load it. */ mod = import_run_extension( tstate, p0, &info, spec, get_modules_dict(tstate, true)); +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; + } +#endif finally: _Py_ext_module_loader_info_clear(&info); @@ -2953,7 +3693,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef header import_level++; - t1 = _PyTime_PerfCounterUnchecked(); + // ignore error: don't block import if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); accumulated = 0; } @@ -2968,7 +3709,9 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) mod != NULL); if (import_time) { - PyTime_t cum = _PyTime_PerfCounterUnchecked() - t1; + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + PyTime_t cum = t2 - t1; import_level--; fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", @@ -3937,10 +4680,15 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) return NULL; } - mod = import_find_extension(tstate, &info); + struct extensions_cache_value *cached = NULL; + mod = import_find_extension(tstate, &info, &cached); if (mod != NULL) { assert(!_PyErr_Occurred(tstate)); - assert(is_singlephase(_PyModule_GetDef(mod))); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); goto finally; } else if (_PyErr_Occurred(tstate)) { @@ -3948,6 +4696,15 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } /* Otherwise it must be multi-phase init or the first time it's loaded. */ + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } + if (PySys_Audit("import", "OOOOO", info.name, info.filename, Py_None, Py_None, Py_None) < 0) { @@ -3973,10 +4730,22 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) goto finally; } +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif mod = import_run_extension( tstate, p0, &info, spec, get_modules_dict(tstate, true)); - if (mod == NULL) { +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; } +#endif // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? if (fp) { @@ -4095,6 +4864,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { {Py_mod_exec, imp_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/importdl.c b/Python/importdl.c index 3e4ea175c0cfbc..7c42d37283c495 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -28,6 +28,11 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, const char *pathname, FILE *fp); #endif + +/***********************************/ +/* module info to use when loading */ +/***********************************/ + static const char * const ascii_only_prefix = "PyInit"; static const char * const nonascii_prefix = "PyInitU"; @@ -106,9 +111,12 @@ _Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) int _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *filename, + _Py_ext_module_origin origin) { - struct _Py_ext_module_loader_info info = {0}; + struct _Py_ext_module_loader_info info = { + .origin=origin, + }; assert(name != NULL); if (!PyUnicode_Check(name)) { @@ -178,12 +186,25 @@ _Py_ext_module_loader_info_init_for_builtin( .name_encoded=name_encoded, /* We won't need filename. */ .path=name, + .origin=_Py_ext_module_origin_BUILTIN, .hook_prefix=ascii_only_prefix, .newcontext=NULL, }; return 0; } +int +_Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *info, + PyObject *name) +{ + if (_Py_ext_module_loader_info_init_for_builtin(info, name) < 0) { + return -1; + } + info->origin = _Py_ext_module_origin_CORE; + return 0; +} + int _Py_ext_module_loader_info_init_from_spec( struct _Py_ext_module_loader_info *p_info, @@ -198,13 +219,159 @@ _Py_ext_module_loader_info_init_from_spec( Py_DECREF(name); return -1; } - int err = _Py_ext_module_loader_info_init(p_info, name, filename); + /* We could also accommodate builtin modules here without much trouble. */ + _Py_ext_module_origin origin = _Py_ext_module_origin_DYNAMIC; + int err = _Py_ext_module_loader_info_init(p_info, name, filename, origin); Py_DECREF(name); Py_DECREF(filename); return err; } +/********************************/ +/* module init function results */ +/********************************/ + +void +_Py_ext_module_loader_result_clear(struct _Py_ext_module_loader_result *res) +{ + /* Instead, the caller should have called + * _Py_ext_module_loader_result_apply_error(). */ + assert(res->err == NULL); + *res = (struct _Py_ext_module_loader_result){0}; +} + +static void +_Py_ext_module_loader_result_set_error( + struct _Py_ext_module_loader_result *res, + enum _Py_ext_module_loader_result_error_kind kind) +{ +#ifndef NDEBUG + switch (kind) { + case _Py_ext_module_loader_result_EXCEPTION: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(PyErr_Occurred()); + break; + case _Py_ext_module_loader_result_ERR_MISSING: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: /* fall through */ + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: /* fall through */ + case _Py_ext_module_loader_result_ERR_NOT_MODULE: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(!PyErr_Occurred()); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +#endif + + assert(res->err == NULL && res->_err.exc == NULL); + res->err = &res->_err; + *res->err = (struct _Py_ext_module_loader_result_error){ + .kind=kind, + .exc=PyErr_GetRaisedException(), + }; + + /* For some kinds, we also set/check res->kind. */ + switch (kind) { + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + assert(res->kind == _Py_ext_module_kind_UNKNOWN); + res->kind = _Py_ext_module_kind_INVALID; + break; + /* None of the rest affect the result kind. */ + case _Py_ext_module_loader_result_EXCEPTION: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: /* fall through */ + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: /* fall through */ + case _Py_ext_module_loader_result_ERR_NOT_MODULE: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +} + +void +_Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name) +{ + assert(!PyErr_Occurred()); + assert(res->err != NULL && res->err == &res->_err); + struct _Py_ext_module_loader_result_error err = *res->err; + res->err = NULL; + + /* We're otherwise done with the result at this point. */ + _Py_ext_module_loader_result_clear(res); + +#ifndef NDEBUG + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(err.exc != NULL); + break; + case _Py_ext_module_loader_result_ERR_MISSING: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: /* fall through */ + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: /* fall through */ + case _Py_ext_module_loader_result_ERR_NOT_MODULE: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(err.exc == NULL); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +#endif + + const char *msg = NULL; + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: + break; + case _Py_ext_module_loader_result_ERR_MISSING: + msg = "initialization of %s failed without raising an exception"; + break; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + msg = "initialization of %s raised unreported exception"; + break; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + msg = "init function of %s returned uninitialized object"; + break; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: + msg = "initialization of %s did not return PyModuleDef"; + break; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: + msg = "initialization of %s did not return an extension module"; + break; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + msg = "initialization of %s did not return a valid extension module"; + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + PyErr_Format(PyExc_SystemError, + "loading %s failed due to init function", name); + return; + } + + if (err.exc != NULL) { + PyErr_SetRaisedException(err.exc); + err.exc = NULL; /* PyErr_SetRaisedException() stole our reference. */ + if (msg != NULL) { + _PyErr_FormatFromCause(PyExc_SystemError, msg, name); + } + } + else { + assert(msg != NULL); + PyErr_Format(PyExc_SystemError, msg, name); + } +} + + +/********************************************/ +/* getting/running the module init function */ +/********************************************/ + PyModInitFunction _PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, FILE *fp) @@ -245,8 +412,9 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, struct _Py_ext_module_loader_info *info, struct _Py_ext_module_loader_result *p_res) { - struct _Py_ext_module_loader_result res = {0}; - const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + struct _Py_ext_module_loader_result res = { + .kind=_Py_ext_module_kind_UNKNOWN, + }; /* Call the module init function. */ @@ -258,18 +426,25 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, /* Validate the result (and populate "res". */ if (m == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "initialization of %s failed without raising an exception", - name_buf); + /* The init func for multi-phase init modules is expected + * to return a PyModuleDef after calling PyModuleDef_Init(). + * That function never raises an exception nor returns NULL, + * so at this point it must be a single-phase init modules. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + if (PyErr_Occurred()) { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_EXCEPTION); + } + else { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING); } goto error; } else if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "initialization of %s raised unreported exception", - name_buf); + /* Likewise, we infer that this is a single-phase init module. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNREPORTED_EXC); /* We would probably be correct to decref m here, * but we weren't doing so before, * so we stick with doing nothing. */ @@ -281,9 +456,8 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, /* This can happen when a PyModuleDef is returned without calling * PyModuleDef_Init on it */ - PyErr_Format(PyExc_SystemError, - "init function of %s returned uninitialized object", - name_buf); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNINITIALIZED); /* Likewise, decref'ing here makes sense. However, the original * code has a note about "prevent segfault in DECREF", * so we play it safe and leave it alone. */ @@ -293,48 +467,50 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { /* multi-phase init */ + res.kind = _Py_ext_module_kind_MULTIPHASE; res.def = (PyModuleDef *)m; /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } else if (info->hook_prefix == nonascii_prefix) { - /* It should have been multi-phase init? */ + /* Non-ASCII is only supported for multi-phase init. */ + res.kind = _Py_ext_module_kind_MULTIPHASE; /* Don't allow legacy init for non-ASCII module names. */ - PyErr_Format( - PyExc_SystemError, - "initialization of %s did not return PyModuleDef", - name_buf); - Py_DECREF(m); - return -1; + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE); + goto error; } else { /* single-phase init (legacy) */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; res.module = m; if (!PyModule_Check(m)) { - PyErr_Format(PyExc_SystemError, - "initialization of %s did not return an extension " - "module", name_buf); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NOT_MODULE); goto error; } res.def = _PyModule_GetDef(m); if (res.def == NULL) { - PyErr_Format(PyExc_SystemError, - "initialization of %s did not return a valid extension " - "module", name_buf); + PyErr_Clear(); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING_DEF); goto error; } } assert(!PyErr_Occurred()); + assert(res.err == NULL); *p_res = res; return 0; error: - assert(PyErr_Occurred()); + assert(!PyErr_Occurred()); + assert(res.err != NULL); Py_CLEAR(res.module); res.def = NULL; *p_res = res; + p_res->err = &p_res->_err; return -1; } diff --git a/Python/initconfig.c b/Python/initconfig.c index d91a8199b544dc..a28c08c5318ddc 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -60,7 +60,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(hash_seed, ULONG), SPEC(faulthandler, BOOL), SPEC(tracemalloc, UINT), - SPEC(perf_profiling, BOOL), + SPEC(perf_profiling, UINT), SPEC(import_time, BOOL), SPEC(code_debug_ranges, BOOL), SPEC(show_ref_count, BOOL), @@ -1703,6 +1703,20 @@ config_init_perf_profiling(PyConfig *config) if (xoption) { config->perf_profiling = 1; } + env = config_get_env(config, "PYTHON_PERF_JIT_SUPPORT"); + if (env) { + if (_Py_str_to_int(env, &active) != 0) { + active = 0; + } + if (active) { + config->perf_profiling = 2; + } + } + xoption = config_get_xoption(config, L"perf_jit"); + if (xoption) { + config->perf_profiling = 2; + } + return _PyStatus_OK(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 328a3b1733d604..77d3489afcfc72 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -459,7 +459,6 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) } #define CHECK(test) do { \ - ASSERT_WORLD_STOPPED_OR_LOCKED(code); \ if (!(test)) { \ dump_instrumentation_data(code, i, stderr); \ } \ @@ -516,10 +515,6 @@ sanity_check_instrumentation(PyCodeObject *code) if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } - if (data->per_instruction_tools) { - uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; - CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]); - } } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); @@ -626,9 +621,10 @@ de_instrument(PyCodeObject *code, int i, int event) return; } CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - *opcode_ptr = deinstrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented); if (_PyOpcode_Caches[deinstrumented]) { - instr[1].counter = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); } } @@ -676,8 +672,6 @@ de_instrument_per_instruction(PyCodeObject *code, int i) } assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION); - /* Keep things clean for sanity check */ - code->_co_monitoring->per_instruction_opcodes[i] = 0; } @@ -703,8 +697,10 @@ instrument(PyCodeObject *code, int i) int deopt = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); - *opcode_ptr = instrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); if (_PyOpcode_Caches[deopt]) { + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); instr[1].counter = adaptive_counter_warmup(); } } @@ -989,6 +985,7 @@ set_global_version(PyThreadState *tstate, uint32_t version) static bool is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); return global_version(interp) == code->_co_instrumentation_version; } @@ -996,11 +993,24 @@ is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) static bool instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); _Py_LocalMonitors expected = local_union( interp->monitors, code->_co_monitoring->local_monitors); return monitors_equals(code->_co_monitoring->active_monitors, expected); } + +static int +debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code) +{ + int res; + LOCK_CODE(code); + res = is_version_up_to_date(code, interp) && + instrumentation_cross_checks(interp, code); + UNLOCK_CODE(); + return res; +} + #endif static inline uint8_t @@ -1015,8 +1025,7 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, event = PY_MONITORING_EVENT_CALL; } if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { - CHECK(is_version_up_to_date(code, interp)); - CHECK(instrumentation_cross_checks(interp, code)); + CHECK(debug_check_sanity(interp, code)); if (code->_co_monitoring->tools) { tools = code->_co_monitoring->tools[i]; } @@ -1214,15 +1223,12 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); + assert(tstate->tracing == 0); + assert(debug_check_sanity(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; - if (tstate->tracing) { - goto done; - } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = 0; @@ -1338,8 +1344,7 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); + assert(debug_check_sanity(tstate->interp, code)); int offset = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); @@ -1670,9 +1675,11 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + // Initialize all of the instructions so if local events change while another thread is executing + // we know what the original opcode was. for (int i = 0; i < code_len; i++) { - code->_co_monitoring->per_instruction_opcodes[i] = 0; + int opcode = _PyCode_CODE(code)[i].op.code; + code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } } if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { @@ -1681,7 +1688,6 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { code->_co_monitoring->per_instruction_tools[i] = 0; } @@ -1691,21 +1697,16 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } static int -instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) +force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); - if (is_version_up_to_date(code, interp)) { - assert( - interp->ceval.instrumentation_version == 0 || - instrumentation_cross_checks(interp, code) - ); - return 0; - } +#ifdef _Py_TIER2 if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); +#endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation * data for potential statically allocated code @@ -1766,7 +1767,6 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) // GH-103845: We need to remove both the line and instruction instrumentation before // adding new ones, otherwise we may remove the newly added instrumentation. - uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; @@ -1774,9 +1774,7 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { - if (removed_line_tools) { - remove_line_tools(code, i, removed_line_tools); - } + remove_line_tools(code, i, removed_line_tools); } i += _PyInstruction_GetLength(code, i); } @@ -1788,15 +1786,14 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); continue; } - if (removed_per_instruction_tools) { - remove_per_instruction_tools(code, i, removed_per_instruction_tools); - } + remove_per_instruction_tools(code, i, removed_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; @@ -1804,9 +1801,7 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { - if (new_line_tools) { - add_line_tools(code, i, new_line_tools); - } + add_line_tools(code, i, new_line_tools); } i += _PyInstruction_GetLength(code, i); } @@ -1818,12 +1813,11 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); continue; } - if (new_per_instruction_tools) { - add_per_instruction_tools(code, i, new_per_instruction_tools); - } + add_per_instruction_tools(code, i, new_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } + done: FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version, global_version(interp)); @@ -1834,6 +1828,22 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) return 0; } +static int +instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) +{ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + + if (is_version_up_to_date(code, interp)) { + assert( + interp->ceval.instrumentation_version == 0 || + instrumentation_cross_checks(interp, code) + ); + return 0; + } + + return force_instrument_lock_held(code, interp); +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { @@ -1943,7 +1953,9 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) goto done; } set_global_version(tstate, new_version); +#ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 1); +#endif res = instrument_all_executing_code_objects(interp); done: _PyEval_StartTheWorld(interp); @@ -1978,14 +1990,8 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent goto done; } set_local_events(local, tool_id, events); - if (is_version_up_to_date(code, interp)) { - /* Force instrumentation update */ - code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; - } - - _Py_Executors_InvalidateDependency(interp, code, 1); - res = instrument_lock_held(code, interp); + res = force_instrument_lock_held(code, interp); done: UNLOCK_CODE(); @@ -2417,3 +2423,304 @@ PyObject *_Py_CreateMonitoringObject(void) Py_DECREF(mod); return NULL; } + + +static int +capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject **args, Py_ssize_t nargs, int event) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; + + uint8_t tools = state->active; + assert(args[1] == NULL); + args[1] = codelike; + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, "offset must be non-negative"); + return -1; + } + PyObject *offset_obj = PyLong_FromLong(offset); + if (offset_obj == NULL) { + return -1; + } + assert(args[2] == NULL); + args[2] = offset_obj; + Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject **callargs = &args[1]; + int err = 0; + + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools ^= (1 << tool); + int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + err = -1; + break; + } + else { + /* DISABLE */ + if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + PyErr_Format(PyExc_ValueError, + "Cannot disable %s events. Callback removed.", + event_names[event]); + /* Clear tool to prevent infinite loop */ + Py_CLEAR(interp->monitoring_callables[tool][event]); + err = -1; + break; + } + else { + state->active &= ~(1 << tool); + } + } + } + return err; +} + +int +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (global_version(interp) == *version) { + return 0; + } + + _Py_GlobalMonitors *m = &interp->monitors; + for (Py_ssize_t i = 0; i < length; i++) { + int event = event_types[i]; + state_array[i].active = m->tools[event]; + } + *version = global_version(interp); + return 0; +} + +int +PyMonitoring_ExitScope(void) +{ + return 0; +} + +int +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_START); +} + +int +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_RESUME); +} + + + +int +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_RETURN); +} + +int +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_YIELD); +} + +int +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + assert(state->active); + PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 }; + return capi_call_instrumentation(state, codelike, offset, args, 4, + PY_MONITORING_EVENT_CALL); +} + +int +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + assert(state->active); + PyObject *lno = PyLong_FromLong(lineno); + if (lno == NULL) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, lno }; + int res= capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_LINE); + Py_DECREF(lno); + return res; +} + +int +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_JUMP); +} + +int +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH); +} + +int +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_C_RETURN); +} + +static inline int +exception_event_setup(PyObject **exc, int event) { + *exc = PyErr_GetRaisedException(); + if (*exc == NULL) { + PyErr_Format(PyExc_ValueError, + "Firing event %d with no exception set", + event); + return -1; + } + return 0; +} + + +static inline int +exception_event_teardown(int err, PyObject *exc) { + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_DECREF(exc); + } + return err; +} + +int +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_THROW; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_C_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RERAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_UNWIND; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_STOP_ITERATION; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} diff --git a/Python/intrinsics.c b/Python/intrinsics.c index d3146973b75178..a6b2c108b67175 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -123,18 +123,15 @@ static PyObject * import_star(PyThreadState* tstate, PyObject *from) { _PyInterpreterFrame *frame = tstate->current_frame; - if (_PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - PyObject *locals = frame->f_locals; + PyObject *locals = _PyFrame_GetLocals(frame); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); return NULL; } int err = import_all_from(tstate, locals, from); - _PyFrame_LocalsToFast(frame, 0); + Py_DECREF(locals); if (err < 0) { return NULL; } @@ -265,6 +262,7 @@ _PyIntrinsics_BinaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_BOUND, make_typevar_with_bound) INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_CONSTRAINTS, make_typevar_with_constraints) INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_FUNCTION_TYPE_PARAMS, _Py_set_function_type_params) + INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPEPARAM_DEFAULT, _Py_set_typeparam_default) }; #undef INTRINSIC_FUNC_ENTRY diff --git a/Python/jit.c b/Python/jit.c index 8782adb847cfd6..7c316a410dda6a 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -16,8 +16,6 @@ #include "pycore_sliceobject.h" #include "pycore_jit.h" -#include "jit_stencils.h" - // Memory management stuff: //////////////////////////////////////////////////// #ifndef MS_WINDOWS @@ -146,253 +144,275 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, #define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) #define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) -// Fill all of stencil's holes in the memory pointed to by base, using the -// values in patches. -static void -patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) +// LLD is a great reference for performing relocations... just keep in +// mind that Tools/jit/build.py does filtering and preprocessing for us! +// Here's a good place to start for each platform: +// - aarch64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h +// - aarch64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - aarch64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp +// - i686-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp +// - x86_64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp + +// Many of these patches are "relaxing", meaning that they can rewrite the +// code they're patching to be more efficient (like turning a 64-bit memory +// load into a 32-bit immediate load). These patches have an "x" in their name. +// Relative patches have an "r" in their name. + +// 32-bit absolute address. +void +patch_32(unsigned char *location, uint64_t value) { - for (size_t i = 0; i < stencil->holes_size; i++) { - const Hole *hole = &stencil->holes[i]; - unsigned char *location = base + hole->offset; - uint64_t value = patches[hole->value] + (uintptr_t)hole->symbol + hole->addend; - uint8_t *loc8 = (uint8_t *)location; - uint32_t *loc32 = (uint32_t *)location; - uint64_t *loc64 = (uint64_t *)location; - // LLD is a great reference for performing relocations... just keep in - // mind that Tools/jit/build.py does filtering and preprocessing for us! - // Here's a good place to start for each platform: - // - aarch64-apple-darwin: - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h - // - aarch64-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - aarch64-unknown-linux-gnu: - // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp - // - i686-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - x86_64-apple-darwin: - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp - // - x86_64-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - x86_64-unknown-linux-gnu: - // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp - switch (hole->kind) { - case HoleKind_IMAGE_REL_I386_DIR32: - // 32-bit absolute address. - // Check that we're not out of range of 32 unsigned bits: - assert(value < (1ULL << 32)); - *loc32 = (uint32_t)value; - continue; - case HoleKind_ARM64_RELOC_UNSIGNED: - case HoleKind_R_AARCH64_ABS64: - case HoleKind_X86_64_RELOC_UNSIGNED: - case HoleKind_R_X86_64_64: - // 64-bit absolute address. - *loc64 = value; - continue; - case HoleKind_IMAGE_REL_AMD64_REL32: - case HoleKind_IMAGE_REL_I386_REL32: - case HoleKind_R_X86_64_GOTPCRELX: - case HoleKind_R_X86_64_REX_GOTPCRELX: - case HoleKind_X86_64_RELOC_GOT: - case HoleKind_X86_64_RELOC_GOT_LOAD: { - // 32-bit relative address. - // Try to relax the GOT load into an immediate value: - uint64_t relaxed = *(uint64_t *)(value + 4) - 4; - if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && - (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) - { - if (loc8[-2] == 0x8B) { - // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] - loc8[-2] = 0x8D; - value = relaxed; - } - else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { - // call qword ptr [rip + AAA] -> nop; call XXX - loc8[-2] = 0x90; - loc8[-1] = 0xE8; - value = relaxed; - } - else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { - // jmp qword ptr [rip + AAA] -> nop; jmp XXX - loc8[-2] = 0x90; - loc8[-1] = 0xE9; - value = relaxed; - } - } - } - // Fall through... - case HoleKind_R_X86_64_GOTPCREL: - case HoleKind_R_X86_64_PC32: - case HoleKind_X86_64_RELOC_SIGNED: - case HoleKind_X86_64_RELOC_BRANCH: - // 32-bit relative address. - value -= (uintptr_t)location; - // Check that we're not out of range of 32 signed bits: - assert((int64_t)value >= -(1LL << 31)); - assert((int64_t)value < (1LL << 31)); - *loc32 = (uint32_t)value; - continue; - case HoleKind_IMAGE_REL_ARM64_BRANCH26: - case HoleKind_R_AARCH64_CALL26: - case HoleKind_R_AARCH64_JUMP26: - // 28-bit relative branch. - assert(IS_AARCH64_BRANCH(*loc32)); - value -= (uintptr_t)location; - // Check that we're not out of range of 28 signed bits: - assert((int64_t)value >= -(1 << 27)); - assert((int64_t)value < (1 << 27)); - // Since instructions are 4-byte aligned, only use 26 bits: - assert(get_bits(value, 0, 2) == 0); - set_bits(loc32, 0, value, 2, 26); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G0_NC: - // 16-bit low part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 0 of 3"): - assert(get_bits(*loc32, 21, 2) == 0); - set_bits(loc32, 5, value, 0, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G1_NC: - // 16-bit middle-low part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 1 of 3"): - assert(get_bits(*loc32, 21, 2) == 1); - set_bits(loc32, 5, value, 16, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G2_NC: - // 16-bit middle-high part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 2 of 3"): - assert(get_bits(*loc32, 21, 2) == 2); - set_bits(loc32, 5, value, 32, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G3: - // 16-bit high part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 3 of 3"): - assert(get_bits(*loc32, 21, 2) == 3); - set_bits(loc32, 5, value, 48, 16); - continue; - case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: - case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21: - case HoleKind_R_AARCH64_ADR_GOT_PAGE: - // 21-bit count of pages between this page and an absolute address's - // page... I know, I know, it's weird. Pairs nicely with - // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). - assert(IS_AARCH64_ADRP(*loc32)); - // Try to relax the pair of GOT loads into an immediate value: - const Hole *next_hole = &stencil->holes[i + 1]; - if (i + 1 < stencil->holes_size && - (next_hole->kind == HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12 || - next_hole->kind == HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L || - next_hole->kind == HoleKind_R_AARCH64_LD64_GOT_LO12_NC) && - next_hole->offset == hole->offset + 4 && - next_hole->symbol == hole->symbol && - next_hole->addend == hole->addend && - next_hole->value == hole->value) - { - unsigned char reg = get_bits(loc32[0], 0, 5); - assert(IS_AARCH64_LDR_OR_STR(loc32[1])); - // There should be only one register involved: - assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. - assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. - uint64_t relaxed = *(uint64_t *)value; - if (relaxed < (1UL << 16)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xD503201F; - i++; - continue; - } - if (relaxed < (1ULL << 32)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; - i++; - continue; - } - relaxed = value - (uintptr_t)location; - if ((relaxed & 0x3) == 0 && - (int64_t)relaxed >= -(1L << 19) && - (int64_t)relaxed < (1L << 19)) - { - // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop - loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; - loc32[1] = 0xD503201F; - i++; - continue; - } - } - // Fall through... - case HoleKind_ARM64_RELOC_PAGE21: - // Number of pages between this page and the value's page: - value = (value >> 12) - ((uintptr_t)location >> 12); - // Check that we're not out of range of 21 signed bits: - assert((int64_t)value >= -(1 << 20)); - assert((int64_t)value < (1 << 20)); - // value[0:2] goes in loc[29:31]: - set_bits(loc32, 29, value, 0, 2); - // value[2:21] goes in loc[5:26]: - set_bits(loc32, 5, value, 2, 19); - continue; - case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: - case HoleKind_ARM64_RELOC_PAGEOFF12: - case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A: - case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L: - case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: - // 12-bit low part of an absolute address. Pairs nicely with - // ARM64_RELOC_GOT_LOAD_PAGE21 (above). - assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); - // There might be an implicit shift encoded in the instruction: - uint8_t shift = 0; - if (IS_AARCH64_LDR_OR_STR(*loc32)) { - shift = (uint8_t)get_bits(*loc32, 30, 2); - // If both of these are set, the shift is supposed to be 4. - // That's pretty weird, and it's never actually been observed... - assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); - } - value = get_bits(value, 0, 12); - assert(get_bits(value, 0, shift) == 0); - set_bits(loc32, 10, value, shift, 12); - continue; - } - Py_UNREACHABLE(); + uint32_t *loc32 = (uint32_t *)location; + // Check that we're not out of range of 32 unsigned bits: + assert(value < (1ULL << 32)); + *loc32 = (uint32_t)value; +} + +// 32-bit relative address. +void +patch_32r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value -= (uintptr_t)location; + // Check that we're not out of range of 32 signed bits: + assert((int64_t)value >= -(1LL << 31)); + assert((int64_t)value < (1LL << 31)); + *loc32 = (uint32_t)value; +} + +// 64-bit absolute address. +void +patch_64(unsigned char *location, uint64_t value) +{ + uint64_t *loc64 = (uint64_t *)location; + *loc64 = value; +} + +// 12-bit low part of an absolute address. Pairs nicely with patch_aarch64_21r +// (below). +void +patch_aarch64_12(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); + // There might be an implicit shift encoded in the instruction: + uint8_t shift = 0; + if (IS_AARCH64_LDR_OR_STR(*loc32)) { + shift = (uint8_t)get_bits(*loc32, 30, 2); + // If both of these are set, the shift is supposed to be 4. + // That's pretty weird, and it's never actually been observed... + assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); } + value = get_bits(value, 0, 12); + assert(get_bits(value, 0, shift) == 0); + set_bits(loc32, 10, value, shift, 12); } -static void -copy_and_patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) +// Relaxable 12-bit low part of an absolute address. Pairs nicely with +// patch_aarch64_21rx (below). +void +patch_aarch64_12x(unsigned char *location, uint64_t value) { - memcpy(base, stencil->body, stencil->body_size); - patch(base, stencil, patches); + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_21rx. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_12(location, value); } -static void -emit(const StencilGroup *group, uintptr_t patches[]) +// 16-bit low part of an absolute address. +void +patch_aarch64_16a(unsigned char *location, uint64_t value) { - copy_and_patch((unsigned char *)patches[HoleValue_DATA], &group->data, patches); - copy_and_patch((unsigned char *)patches[HoleValue_CODE], &group->code, patches); + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 0 of 3"): + assert(get_bits(*loc32, 21, 2) == 0); + set_bits(loc32, 5, value, 0, 16); } +// 16-bit middle-low part of an absolute address. +void +patch_aarch64_16b(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 1 of 3"): + assert(get_bits(*loc32, 21, 2) == 1); + set_bits(loc32, 5, value, 16, 16); +} + +// 16-bit middle-high part of an absolute address. +void +patch_aarch64_16c(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 2 of 3"): + assert(get_bits(*loc32, 21, 2) == 2); + set_bits(loc32, 5, value, 32, 16); +} + +// 16-bit high part of an absolute address. +void +patch_aarch64_16d(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 3 of 3"): + assert(get_bits(*loc32, 21, 2) == 3); + set_bits(loc32, 5, value, 48, 16); +} + +// 21-bit count of pages between this page and an absolute address's page... I +// know, I know, it's weird. Pairs nicely with patch_aarch64_12 (above). +void +patch_aarch64_21r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value = (value >> 12) - ((uintptr_t)location >> 12); + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // value[0:2] goes in loc[29:31]: + set_bits(loc32, 29, value, 0, 2); + // value[2:21] goes in loc[5:26]: + set_bits(loc32, 5, value, 2, 19); +} + +// Relaxable 21-bit count of pages between this page and an absolute address's +// page. Pairs nicely with patch_aarch64_12x (above). +void +patch_aarch64_21rx(unsigned char *location, uint64_t value) +{ + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_12x. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_21r(location, value); +} + +// 28-bit relative branch. +void +patch_aarch64_26r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_BRANCH(*loc32)); + value -= (uintptr_t)location; + // Check that we're not out of range of 28 signed bits: + assert((int64_t)value >= -(1 << 27)); + assert((int64_t)value < (1 << 27)); + // Since instructions are 4-byte aligned, only use 26 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 0, value, 2, 26); +} + +// A pair of patch_aarch64_21rx and patch_aarch64_12x. +void +patch_aarch64_33rx(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + // Try to relax the pair of GOT loads into an immediate value: + assert(IS_AARCH64_ADRP(*loc32)); + unsigned char reg = get_bits(loc32[0], 0, 5); + assert(IS_AARCH64_LDR_OR_STR(loc32[1])); + // There should be only one register involved: + assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. + assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. + uint64_t relaxed = *(uint64_t *)value; + if (relaxed < (1UL << 16)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + if (relaxed < (1ULL << 32)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; + return; + } + relaxed = value - (uintptr_t)location; + if ((relaxed & 0x3) == 0 && + (int64_t)relaxed >= -(1L << 19) && + (int64_t)relaxed < (1L << 19)) + { + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + // Couldn't do it. Just patch the two instructions normally: + patch_aarch64_21rx(location, value); + patch_aarch64_12x(location + 4, value); +} + +// Relaxable 32-bit relative address. +void +patch_x86_64_32rx(unsigned char *location, uint64_t value) +{ + uint8_t *loc8 = (uint8_t *)location; + // Try to relax the GOT load into an immediate value: + uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && + (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) + { + if (loc8[-2] == 0x8B) { + // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] + loc8[-2] = 0x8D; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { + // call qword ptr [rip + AAA] -> nop; call XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE8; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { + // jmp qword ptr [rip + AAA] -> nop; jmp XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE9; + value = relaxed; + } + } + patch_32r(location, value); +} + +#include "jit_stencils.h" + // Compiles executor in-place. Don't forget to call _PyJIT_Free later! int -_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) +_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], size_t length) { + const StencilGroup *group; // Loop once to find the total compiled size: - size_t instruction_starts[UOP_MAX_TRACE_LENGTH]; + uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; size_t code_size = 0; size_t data_size = 0; + group = &trampoline; + code_size += group->code_size; + data_size += group->data_size; for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; - const StencilGroup *group = &stencil_groups[instruction->opcode]; + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; instruction_starts[i] = code_size; - code_size += group->code.body_size; - data_size += group->data.body_size; + code_size += group->code_size; + data_size += group->data_size; } - code_size += stencil_groups[_FATAL_ERROR].code.body_size; - data_size += stencil_groups[_FATAL_ERROR].data.body_size; + group = &stencil_groups[_FATAL_ERROR]; + code_size += group->code_size; + data_size += group->data_size; // Round up to the nearest page: size_t page_size = get_page_size(); assert((page_size & (page_size - 1)) == 0); @@ -402,67 +422,35 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size if (memory == NULL) { return -1; } + // Update the offsets of each instruction: + for (size_t i = 0; i < length; i++) { + instruction_starts[i] += (uintptr_t)memory; + } // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; + // Compile the trampoline, which handles converting between the native + // calling convention and the calling convention used by jitted code + // (which may be different for efficiency reasons). On platforms where + // we don't change calling conventions, the trampoline is empty and + // nothing is emitted here: + group = &trampoline; + group->emit(code, data, executor, NULL, instruction_starts); + code += group->code_size; + data += group->data_size; assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; - const StencilGroup *group = &stencil_groups[instruction->opcode]; - // Think of patches as a dictionary mapping HoleValue to uintptr_t: - uintptr_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uintptr_t)code; - patches[HoleValue_CONTINUE] = (uintptr_t)code + group->code.body_size; - patches[HoleValue_DATA] = (uintptr_t)data; - patches[HoleValue_EXECUTOR] = (uintptr_t)executor; - patches[HoleValue_OPARG] = instruction->oparg; - #if SIZEOF_VOID_P == 8 - patches[HoleValue_OPERAND] = instruction->operand; - #else - assert(SIZEOF_VOID_P == 4); - patches[HoleValue_OPERAND_HI] = instruction->operand >> 32; - patches[HoleValue_OPERAND_LO] = instruction->operand & UINT32_MAX; - #endif - switch (instruction->format) { - case UOP_FORMAT_TARGET: - patches[HoleValue_TARGET] = instruction->target; - break; - case UOP_FORMAT_EXIT: - assert(instruction->exit_index < executor->exit_count); - patches[HoleValue_EXIT_INDEX] = instruction->exit_index; - if (instruction->error_target < length) { - patches[HoleValue_ERROR_TARGET] = (uintptr_t)memory + instruction_starts[instruction->error_target]; - } - break; - case UOP_FORMAT_JUMP: - assert(instruction->jump_target < length); - patches[HoleValue_JUMP_TARGET] = (uintptr_t)memory + instruction_starts[instruction->jump_target]; - if (instruction->error_target < length) { - patches[HoleValue_ERROR_TARGET] = (uintptr_t)memory + instruction_starts[instruction->error_target]; - } - break; - default: - assert(0); - Py_FatalError("Illegal instruction format"); - } - patches[HoleValue_TOP] = (uintptr_t)memory + instruction_starts[1]; - patches[HoleValue_ZERO] = 0; - emit(group, patches); - code += group->code.body_size; - data += group->data.body_size; + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; + group->emit(code, data, executor, instruction, instruction_starts); + code += group->code_size; + data += group->data_size; } // Protect against accidental buffer overrun into data: - const StencilGroup *group = &stencil_groups[_FATAL_ERROR]; - uintptr_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uintptr_t)code; - patches[HoleValue_CONTINUE] = (uintptr_t)code; - patches[HoleValue_DATA] = (uintptr_t)data; - patches[HoleValue_EXECUTOR] = (uintptr_t)executor; - patches[HoleValue_TOP] = (uintptr_t)code; - patches[HoleValue_ZERO] = 0; - emit(group, patches); - code += group->code.body_size; - data += group->data.body_size; + group = &stencil_groups[_FATAL_ERROR]; + group->emit(code, data, executor, NULL, instruction_starts); + code += group->code_size; + data += group->data_size; assert(code == memory + code_size); assert(data == memory + code_size + data_size); if (mark_executable(memory, total_size)) { @@ -470,6 +458,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size return -1; } executor->jit_code = memory; + executor->jit_side_entry = memory + trampoline.code_size; executor->jit_size = total_size; return 0; } @@ -481,6 +470,7 @@ _PyJIT_Free(_PyExecutorObject *executor) size_t size = executor->jit_size; if (memory) { executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; if (jit_free(memory, size)) { PyErr_WriteUnraisable(NULL); diff --git a/Python/lock.c b/Python/lock.c index 91c66df8fd9093..239e56ad929ea3 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -5,13 +5,13 @@ #include "pycore_lock.h" #include "pycore_parking_lot.h" #include "pycore_semaphore.h" -#include "pycore_time.h" // _PyTime_MonotonicUnchecked() +#include "pycore_time.h" // _PyTime_Add() #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN -# include // SwitchToThread() +# include // SwitchToThread() #elif defined(HAVE_SCHED_H) -# include // sched_yield() +# include // sched_yield() #endif // If a thread waits on a lock for longer than TIME_TO_BE_FAIR_NS (1 ms), then @@ -66,7 +66,9 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyTime_Add(now, timeout); @@ -143,7 +145,9 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) { uint8_t v = 0; if (entry) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); int should_be_fair = now > entry->time_to_be_fair; entry->handed_off = should_be_fair; @@ -277,12 +281,12 @@ _PyEvent_Notify(PyEvent *evt) void PyEvent_Wait(PyEvent *evt) { - while (!PyEvent_WaitTimed(evt, -1)) + while (!PyEvent_WaitTimed(evt, -1, /*detach=*/1)) ; } int -PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach) { for (;;) { uint8_t v = _Py_atomic_load_uint8(&evt->v); @@ -298,7 +302,7 @@ PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) uint8_t expected = _Py_HAS_PARKED; (void) _PyParkingLot_Park(&evt->v, &expected, sizeof(evt->v), - timeout_ns, NULL, 1); + timeout_ns, NULL, detach); return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED; } diff --git a/Python/marshal.c b/Python/marshal.c index 4bd8bb1d3a9308..ca22d6d679a230 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1952,6 +1952,7 @@ marshal_module_exec(PyObject *mod) static PyModuleDef_Slot marshalmodule_slots[] = { {Py_mod_exec, marshal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 4061ba33cea53e..fa4f1f8cbb475a 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -163,6 +163,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_GENERAL, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -174,8 +175,9 @@ static void *opcode_targets[256] = { &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NON_PY_GENERAL, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, + &&TARGET_CALL_PY_GENERAL, &&TARGET_CALL_STR_1, &&TARGET_CALL_TUPLE_1, &&TARGET_CALL_TYPE_1, @@ -233,8 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, diff --git a/Python/optimizer.c b/Python/optimizer.c index fcd7d18f2c2e22..8be2c0ffbd78e9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,3 +1,5 @@ +#ifdef _Py_TIER2 + #include "Python.h" #include "opcode.h" #include "pycore_interp.h" @@ -21,6 +23,19 @@ #define MAX_EXECUTORS_SIZE 256 +#ifdef Py_DEBUG +static int +base_opcode(PyCodeObject *code, int offset) +{ + int opcode = _Py_GetBaseOpcode(code, offset); + if (opcode == ENTER_EXECUTOR) { + int oparg = _PyCode_CODE(code)[offset].op.arg; + _PyExecutorObject *ex = code->co_executors->executors[oparg]; + return ex->vm_data.opcode; + } + return opcode; +} +#endif static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) @@ -73,7 +88,7 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO Py_INCREF(executor); if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); - _Py_ExecutorClear(code->co_executors->executors[index]); + _Py_ExecutorDetach(code->co_executors->executors[index]); } else { assert(code->co_executors->size == index); @@ -268,10 +283,14 @@ static PyMethodDef executor_methods[] = { ///////////////////// Experimental UOp Optimizer ///////////////////// +static int executor_clear(_PyExecutorObject *executor); +static void unlink_executor(_PyExecutorObject *executor); + static void uop_dealloc(_PyExecutorObject *self) { _PyObject_GC_UNTRACK(self); - _Py_ExecutorClear(self); + assert(self->vm_data.code == NULL); + unlink_executor(self); #ifdef _Py_JIT _PyJIT_Free(self); #endif @@ -377,13 +396,6 @@ PySequenceMethods uop_as_sequence = { .sq_item = (ssizeargfunc)uop_item, }; -static int -executor_clear(PyObject *o) -{ - _Py_ExecutorClear((_PyExecutorObject *)o); - return 0; -} - static int executor_traverse(PyObject *o, visitproc visit, void *arg) { @@ -394,6 +406,29 @@ executor_traverse(PyObject *o, visitproc visit, void *arg) return 0; } +static PyObject * +get_jit_code(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#ifndef _Py_JIT + PyErr_SetString(PyExc_RuntimeError, "JIT support not enabled."); + return NULL; +#else + _PyExecutorObject *executor = (_PyExecutorObject *)self; + if (executor->jit_code == NULL || executor->jit_size == 0) { + Py_RETURN_NONE; + } + return PyBytes_FromStringAndSize(executor->jit_code, executor->jit_size); +#endif +} + +static PyMethodDef uop_executor_methods[] = { + { "is_valid", is_valid, METH_NOARGS, NULL }, + { "get_jit_code", get_jit_code, METH_NOARGS, NULL}, + { "get_opcode", get_opcode, METH_NOARGS, NULL }, + { "get_oparg", get_oparg, METH_NOARGS, NULL }, + { NULL, NULL }, +}; + static int executor_is_gc(PyObject *o) { @@ -408,9 +443,9 @@ PyTypeObject _PyUOpExecutor_Type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, - .tp_methods = executor_methods, + .tp_methods = uop_executor_methods, .tp_traverse = executor_traverse, - .tp_clear = executor_clear, + .tp_clear = (inquiry)executor_clear, .tp_is_gc = executor_is_gc, }; @@ -423,6 +458,14 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_FOR_ITER] = _FOR_ITER_TIER_TWO, }; +static const uint8_t +is_for_iter_test[MAX_UOP_ID + 1] = { + [_GUARD_NOT_EXHAUSTED_RANGE] = 1, + [_GUARD_NOT_EXHAUSTED_LIST] = 1, + [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, + [_FOR_ITER_TIER_TWO] = 1, +}; + static const uint16_t BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP, @@ -572,7 +615,6 @@ translate_bytecode_to_trace( uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; - uint32_t extended = 0; DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); @@ -586,7 +628,6 @@ translate_bytecode_to_trace( if (opcode == EXTENDED_ARG) { instr++; - extended = 1; opcode = instr->op.code; oparg = (oparg << 8) | instr->op.arg; if (opcode == EXTENDED_ARG) { @@ -605,6 +646,9 @@ translate_bytecode_to_trace( if (opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_NO_INTERRUPT) { instr += 1 + _PyOpcode_Caches[opcode] - (int32_t)oparg; initial_instr = instr; + if (opcode == JUMP_BACKWARD) { + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + } continue; } else { @@ -704,7 +748,7 @@ translate_bytecode_to_trace( int nuops = expansion->nuops; RESERVE(nuops + 1); /* One extra for exit */ int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _POP_FRAME || last_op == _RETURN_GENERATOR) { + if (last_op == _POP_FRAME || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { // Check for trace stack underflow now: // We can't bail e.g. in the middle of // LOAD_CONST + _POP_FRAME. @@ -747,12 +791,15 @@ translate_bytecode_to_trace( case OPARG_REPLACED: uop = _PyUOp_Replacements[uop]; assert(uop != 0); - if (uop == _FOR_ITER_TIER_TWO) { - target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 2 + extended; - assert(_PyCode_CODE(code)[target-2].op.code == END_FOR || - _PyCode_CODE(code)[target-2].op.code == INSTRUMENTED_END_FOR); - assert(_PyCode_CODE(code)[target-1].op.code == POP_TOP); +#ifdef Py_DEBUG + { + uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); + uint32_t jump_target = next_inst + oparg; + assert(base_opcode(code, jump_target) == END_FOR || + base_opcode(code, jump_target) == INSTRUMENTED_END_FOR); + assert(base_opcode(code, jump_target+1) == POP_TOP); } +#endif break; default: fprintf(stderr, @@ -763,7 +810,7 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } - if (uop == _POP_FRAME || uop == _RETURN_GENERATOR) { + if (uop == _POP_FRAME || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { TRACE_STACK_POP(); /* Set the operand to the function or code object returned to, * to assist optimization passes. (See _PUSH_FRAME below.) @@ -929,7 +976,7 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _SIDE_EXIT || opcode == _DYNAMIC_EXIT) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } @@ -940,6 +987,7 @@ static void make_exit(_PyUOpInstruction *inst, int opcode, int target) { inst->opcode = opcode; inst->oparg = 0; + inst->operand = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; } @@ -954,6 +1002,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int32_t current_error = -1; int32_t current_error_target = -1; int32_t current_popped = -1; + int32_t current_exit_op = -1; /* Leaving in NOPs slows down the interpreter and messes up the stats */ _PyUOpInstruction *copy_to = &buffer[0]; for (int i = 0; i < length; i++) { @@ -972,21 +1021,20 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int opcode = inst->opcode; int32_t target = (int32_t)uop_get_target(inst); if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { - if (target != current_jump_target) { - uint16_t exit_op; - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { - if (opcode == _TIER2_RESUME_CHECK) { - exit_op = _EVAL_BREAKER_EXIT; - } - else { - exit_op = _SIDE_EXIT; - } - } - else { - exit_op = _DEOPT; - } - make_exit(&buffer[next_spare], exit_op, target); - current_jump_target = target; + uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? + _EXIT_TRACE : _DEOPT; + int32_t jump_target = target; + if (is_for_iter_test[opcode]) { + /* Target the POP_TOP immediately after the END_FOR, + * leaving only the iterator on the stack. */ + int extended_arg = inst->oparg > 255; + int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; + jump_target = next_inst + inst->oparg + 1; + } + if (jump_target != current_jump_target || current_exit_op != exit_op) { + make_exit(&buffer[next_spare], exit_op, jump_target); + current_exit_op = exit_op; + current_jump_target = jump_target; current_jump = next_spare; next_spare++; } @@ -1065,7 +1113,7 @@ sanity_check(_PyExecutorObject *executor) CHECK(target_unused(opcode)); break; case UOP_FORMAT_EXIT: - CHECK(opcode == _SIDE_EXIT); + CHECK(opcode == _EXIT_TRACE); CHECK(inst->exit_index < executor->exit_count); break; case UOP_FORMAT_JUMP: @@ -1091,10 +1139,9 @@ sanity_check(_PyExecutorObject *executor) uint16_t opcode = inst->opcode; CHECK( opcode == _DEOPT || - opcode == _SIDE_EXIT || - opcode == _EVAL_BREAKER_EXIT || + opcode == _EXIT_TRACE || opcode == _ERROR_POP_N); - if (opcode == _SIDE_EXIT) { + if (opcode == _EXIT_TRACE) { CHECK(inst->format == UOP_FORMAT_EXIT); } } @@ -1132,7 +1179,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil dest--; *dest = buffer[i]; assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _SIDE_EXIT) { + if (opcode == _EXIT_TRACE) { executor->exits[next_exit].target = buffer[i].target; dest->exit_index = next_exit; dest->format = UOP_FORMAT_EXIT; @@ -1166,6 +1213,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil #endif #ifdef _Py_JIT executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; if (_PyJIT_Compile(executor, executor->trace, length)) { Py_DECREF(executor); @@ -1188,6 +1236,7 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) inst->opcode = _COLD_EXIT; inst->oparg = oparg; executor->vm_data.valid = true; + executor->vm_data.linked = false; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { assert(executor->vm_data.bloom.bits[i] == 0); } @@ -1196,6 +1245,7 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) #endif #ifdef _Py_JIT executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; if (_PyJIT_Compile(executor, executor->trace, 1)) { return -1; @@ -1326,7 +1376,7 @@ PyTypeObject _PyCounterExecutor_Type = { .tp_dealloc = (destructor)counter_dealloc, .tp_methods = executor_methods, .tp_traverse = executor_traverse, - .tp_clear = executor_clear, + .tp_clear = (inquiry)executor_clear, }; static int @@ -1349,14 +1399,13 @@ counter_optimize( return 0; } _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[5] = { - { .opcode = _START_EXECUTOR, .jump_target = 4, .format=UOP_FORMAT_JUMP }, + _PyUOpInstruction buffer[4] = { + { .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP }, { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, - { .opcode = _EXIT_TRACE, .jump_target = 4, .format=UOP_FORMAT_JUMP }, - { .opcode = _SIDE_EXIT, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } + { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, 5, &EMPTY_FILTER); + _PyExecutorObject *executor = make_executor_from_uops(buffer, 4, &EMPTY_FILTER); if (executor == NULL) { return -1; } @@ -1501,15 +1550,13 @@ link_executor(_PyExecutorObject *executor) links->next = NULL; } else { - _PyExecutorObject *next = head->vm_data.links.next; - links->previous = head; - links->next = next; - if (next != NULL) { - next->vm_data.links.previous = executor; - } - head->vm_data.links.next = executor; + assert(head->vm_data.links.previous == NULL); + links->previous = NULL; + links->next = head; + head->vm_data.links.previous = executor; + interp->executor_list_head = executor; } - executor->vm_data.valid = true; + executor->vm_data.linked = true; /* executor_list_head must be first in list */ assert(interp->executor_list_head->vm_data.links.previous == NULL); } @@ -1517,7 +1564,11 @@ link_executor(_PyExecutorObject *executor) static void unlink_executor(_PyExecutorObject *executor) { + if (!executor->vm_data.linked) { + return; + } _PyExecutorLinkListNode *links = &executor->vm_data.links; + assert(executor->vm_data.valid); _PyExecutorObject *next = links->next; _PyExecutorObject *prev = links->previous; if (next != NULL) { @@ -1532,7 +1583,7 @@ unlink_executor(_PyExecutorObject *executor) assert(interp->executor_list_head == executor); interp->executor_list_head = next; } - executor->vm_data.valid = false; + executor->vm_data.linked = false; } /* This must be called by optimizers before using the executor */ @@ -1546,23 +1597,15 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s link_executor(executor); } -/* This must be called by executors during dealloc */ +/* Detaches the executor from the code object (if any) that + * holds a reference to it */ void -_Py_ExecutorClear(_PyExecutorObject *executor) +_Py_ExecutorDetach(_PyExecutorObject *executor) { - if (!executor->vm_data.valid) { - return; - } - unlink_executor(executor); PyCodeObject *code = executor->vm_data.code; if (code == NULL) { return; } - for (uint32_t i = 0; i < executor->exit_count; i++) { - Py_DECREF(executor->exits[i].executor); - executor->exits[i].executor = &COLD_EXITS[i]; - executor->exits[i].temperature = initial_unreachable_backoff_counter(); - } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; assert(instruction->op.code == ENTER_EXECUTOR); int index = instruction->op.arg; @@ -1570,7 +1613,36 @@ _Py_ExecutorClear(_PyExecutorObject *executor) instruction->op.code = executor->vm_data.opcode; instruction->op.arg = executor->vm_data.oparg; executor->vm_data.code = NULL; - Py_CLEAR(code->co_executors->executors[index]); + code->co_executors->executors[index] = NULL; + Py_DECREF(executor); +} + +static int +executor_clear(_PyExecutorObject *executor) +{ + if (!executor->vm_data.valid) { + return 0; + } + assert(executor->vm_data.valid == 1); + unlink_executor(executor); + executor->vm_data.valid = 0; + /* It is possible for an executor to form a reference + * cycle with itself, so decref'ing a side exit could + * free the executor unless we hold a strong reference to it + */ + Py_INCREF(executor); + for (uint32_t i = 0; i < executor->exit_count; i++) { + const _PyExecutorObject *cold = &COLD_EXITS[i]; + const _PyExecutorObject *side = executor->exits[i].executor; + executor->exits[i].temperature = initial_unreachable_backoff_counter(); + if (side != cold) { + executor->exits[i].executor = cold; + Py_DECREF(side); + } + } + _Py_ExecutorDetach(executor); + Py_DECREF(executor); + return 0; } void @@ -1591,17 +1663,42 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_BloomFilter_Add(&obj_filter, obj); /* Walk the list of executors */ /* TO DO -- Use a tree to avoid traversing as many objects */ + bool no_memory = false; + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + PyErr_Clear(); + no_memory = true; + } + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { assert(exec->vm_data.valid); _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - _Py_ExecutorClear(exec); + unlink_executor(exec); + if (no_memory) { + exec->vm_data.valid = 0; + } else { + if (PyList_Append(invalidate, (PyObject *)exec) < 0) { + PyErr_Clear(); + no_memory = true; + exec->vm_data.valid = 0; + } + } if (is_invalidation) { OPT_STAT_INC(executors_invalidated); } } exec = next; } + if (invalidate != NULL) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + } + return; } /* Invalidate all executors */ @@ -1610,15 +1707,18 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) { while (interp->executor_list_head) { _PyExecutorObject *executor = interp->executor_list_head; + assert(executor->vm_data.valid == 1 && executor->vm_data.linked == 1); if (executor->vm_data.code) { // Clear the entire code object so its co_executors array be freed: _PyCode_Clear_Executors(executor->vm_data.code); } else { - _Py_ExecutorClear(executor); + executor_clear(executor); } if (is_invalidation) { OPT_STAT_INC(executors_invalidated); } } } + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 9315d7228b5732..842b2e489239af 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,3 +1,5 @@ +#ifdef _Py_TIER2 + /* * This file contains the support code for CPython's uops optimizer. * It also performs some simple optimizations. @@ -603,3 +605,5 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_successes); return length; } + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 8bc56342774790..928bc03382b8fb 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -629,6 +629,15 @@ dummy_func(void) { frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); } + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + first_valid_check_stack = NULL; + goto done; + } + op(_POP_FRAME, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; @@ -673,6 +682,15 @@ dummy_func(void) { } } + op(_YIELD_VALUE, (unused -- res)) { + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + } + + op(_FOR_ITER_GEN_FRAME, ( -- )) { + /* We are about to hit the end of the trace */ + goto done; + } + op(_CHECK_STACK_SPACE, ( --)) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; @@ -709,7 +727,7 @@ dummy_func(void) { if (first_valid_check_stack == NULL) { first_valid_check_stack = corresponding_check_stack; } - else { + else if (corresponding_check_stack) { // delete all but the first valid _CHECK_STACK_SPACE corresponding_check_stack->opcode = _NOP; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4102d00171fbaf..2a4efd73d794df 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -684,6 +684,13 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + case _YIELD_VALUE: { + _Py_UopsSymbol *res; + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + stack_pointer[-1] = res; + break; + } + case _POP_EXCEPT: { stack_pointer += -1; break; @@ -1440,11 +1447,8 @@ } case _FOR_ITER_GEN_FRAME: { - _PyInterpreterFrame *gen_frame; - gen_frame = sym_new_not_null(ctx); - if (gen_frame == NULL) goto out_of_space; - stack_pointer[0] = (_Py_UopsSymbol *)gen_frame; - stack_pointer += 1; + /* We are about to hit the end of the trace */ + goto done; break; } @@ -1555,6 +1559,58 @@ break; } + case _PY_FRAME_GENERAL: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + first_valid_check_stack = NULL; + goto done; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _CHECK_FUNCTION_VERSION: { + break; + } + + case _CHECK_METHOD_VERSION: { + break; + } + + case _EXPAND_METHOD: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + method = sym_new_not_null(ctx); + if (method == NULL) goto out_of_space; + self = sym_new_not_null(ctx); + if (self == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + break; + } + + case _CALL_NON_PY_GENERAL: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { _Py_UopsSymbol *null; _Py_UopsSymbol *callable; @@ -1688,7 +1744,7 @@ if (first_valid_check_stack == NULL) { first_valid_check_stack = corresponding_check_stack; } - else { + else if (corresponding_check_stack) { // delete all but the first valid _CHECK_STACK_SPACE corresponding_check_stack->opcode = _NOP; } @@ -1696,8 +1752,6 @@ break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ - case _CALL_TYPE_1: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); @@ -2136,10 +2190,6 @@ break; } - case _SIDE_EXIT: { - break; - } - case _ERROR_POP_N: { stack_pointer += -oparg; break; @@ -2149,7 +2199,3 @@ break; } - case _EVAL_BREAKER_EXIT: { - break; - } - diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 204599b08766c3..4aeb04fe0405d2 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -1,3 +1,4 @@ +#ifdef _Py_TIER2 #include "Python.h" @@ -163,19 +164,26 @@ _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) return true; } - bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym) { + if (_Py_uop_sym_is_not_null(sym)) { + sym_set_bottom(sym); + return false; + } sym_set_flag(sym, IS_NULL); - return !_Py_uop_sym_is_bottom(sym); + return true; } bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) { + if (_Py_uop_sym_is_null(sym)) { + sym_set_bottom(sym); + return false; + } sym_set_flag(sym, NOT_NULL); - return !_Py_uop_sym_is_bottom(sym); + return true; } @@ -506,3 +514,5 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) Py_XDECREF(val_43); return NULL; } + +#endif /* _Py_TIER2 */ diff --git a/Python/parking_lot.c b/Python/parking_lot.c index b368b500ccdfdb..e2def9e249cd56 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -6,7 +6,7 @@ #include "pycore_pyerrors.h" // _Py_FatalErrorFormat #include "pycore_pystate.h" // _PyThreadState_GET #include "pycore_semaphore.h" // _PySemaphore -#include "pycore_time.h" //_PyTime_MonotonicUnchecked() +#include "pycore_time.h" // _PyTime_Add() #include @@ -120,13 +120,18 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) struct timespec ts; #if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) - PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); - + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = sem_clockwait(&sema->platform_sem, CLOCK_MONOTONIC, &ts); #else - PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); @@ -163,7 +168,9 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) _PyTime_AsTimespec_clamp(timeout, &ts); err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); #else - PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); + PyTime_t now; + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c new file mode 100644 index 00000000000000..2a29318b1054a5 --- /dev/null +++ b/Python/perf_jit_trampoline.c @@ -0,0 +1,616 @@ +#include "Python.h" +#include "pycore_ceval.h" // _PyPerf_Callbacks +#include "pycore_frame.h" +#include "pycore_interp.h" + + +#ifdef PY_HAVE_PERF_TRAMPOLINE + +#include +#include +#include +#include // mmap() +#include +#include // sysconf() +#include // gettimeofday() +#include + +// ---------------------------------- +// Perf jitdump API +// ---------------------------------- + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; + void* mapped_buffer; + size_t mapped_size; + int code_id; +} PerfMapJitState; + +static PerfMapJitState perf_jit_map_state; + +/* +Usually the binary and libraries are mapped in separate region like below: + + address -> + --+---------------------+--//--+---------------------+-- + | .text | .data | ... | | .text | .data | ... | + --+---------------------+--//--+---------------------+-- + myprog libc.so + +So it'd be easy and straight-forward to find a mapped binary or library from an +address. + +But for JIT code, the code arena only cares about the code section. But the +resulting DSOs (which is generated by perf inject -j) contain ELF headers and +unwind info too. Then it'd generate following address space with synthesized +MMAP events. Let's say it has a sample between address B and C. + + sample + | + address -> A B v C + --------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + --------------------------------------------------------------------------------------------------- + +If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see +the unwind info. If it maps both .text section and unwind sections, the sample +could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing +which one is right. So to make perf happy we have non-overlapping ranges for each +DSO: + + address -> + ------------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + ------------------------------------------------------------------------------------------------------- + +As the trampolines are constant, we add a constant padding but in general the padding needs to have the +size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + */ + +#define PERF_JIT_CODE_PADDING 0x100 +#define trampoline_api _PyRuntime.ceval.perf.trampoline_api + +typedef uint64_t uword; +typedef const char* CodeComments; + +#define Pd "d" +#define MB (1024 * 1024) + +#define EM_386 3 +#define EM_X86_64 62 +#define EM_ARM 40 +#define EM_AARCH64 183 +#define EM_RISCV 243 + +#define TARGET_ARCH_IA32 0 +#define TARGET_ARCH_X64 0 +#define TARGET_ARCH_ARM 0 +#define TARGET_ARCH_ARM64 0 +#define TARGET_ARCH_RISCV32 0 +#define TARGET_ARCH_RISCV64 0 + +#define FLAG_generate_perf_jitdump 0 +#define FLAG_write_protect_code 0 +#define FLAG_write_protect_vm_isolate 0 +#define FLAG_code_comments 0 + +#define UNREACHABLE() + +static uword GetElfMachineArchitecture(void) { +#if TARGET_ARCH_IA32 + return EM_386; +#elif TARGET_ARCH_X64 + return EM_X86_64; +#elif TARGET_ARCH_ARM + return EM_ARM; +#elif TARGET_ARCH_ARM64 + return EM_AARCH64; +#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 + return EM_RISCV; +#else + UNREACHABLE(); + return 0; +#endif +} + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t elf_mach_target; + uint32_t reserved; + uint32_t process_id; + uint64_t time_stamp; + uint64_t flags; +} Header; + + enum PerfEvent { + PerfLoad = 0, + PerfMove = 1, + PerfDebugInfo = 2, + PerfClose = 3, + PerfUnwindingInfo = 4 +}; + +struct BaseEvent { + uint32_t event; + uint32_t size; + uint64_t time_stamp; + }; + +typedef struct { + struct BaseEvent base; + uint32_t process_id; + uint32_t thread_id; + uint64_t vma; + uint64_t code_address; + uint64_t code_size; + uint64_t code_id; +} CodeLoadEvent; + +typedef struct { + struct BaseEvent base; + uint64_t unwind_data_size; + uint64_t eh_frame_hdr_size; + uint64_t mapped_size; +} CodeUnwindingInfoEvent; + +static const intptr_t nanoseconds_per_second = 1000000000; + +// Dwarf encoding constants + +static const uint8_t DwarfUData4 = 0x03; +static const uint8_t DwarfSData4 = 0x0b; +static const uint8_t DwarfPcRel = 0x10; +static const uint8_t DwarfDataRel = 0x30; +// static uint8_t DwarfOmit = 0xff; +typedef struct { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + int32_t eh_frame_ptr; + int32_t eh_fde_count; + int32_t from; + int32_t to; +} EhFrameHeader; + +static int64_t get_current_monotonic_ticks(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + UNREACHABLE(); + return 0; + } + // Convert to nanoseconds. + int64_t result = ts.tv_sec; + result *= nanoseconds_per_second; + result += ts.tv_nsec; + return result; +} + +static int64_t get_current_time_microseconds(void) { + // gettimeofday has microsecond resolution. + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + UNREACHABLE(); + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; +} + + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + + +static void perf_map_jit_write_fully(const void* buffer, size_t size) { + FILE* out_file = perf_jit_map_state.perf_map; + const char* ptr = (const char*)(buffer); + while (size > 0) { + const size_t written = fwrite(ptr, 1, size, out_file); + if (written == 0) { + UNREACHABLE(); + break; + } + size -= written; + ptr += written; + } +} + +static void perf_map_jit_write_header(int pid, FILE* out_file) { + Header header; + header.magic = 0x4A695444; + header.version = 1; + header.size = sizeof(Header); + header.elf_mach_target = GetElfMachineArchitecture(); + header.process_id = pid; + header.time_stamp = get_current_time_microseconds(); + header.flags = 0; + perf_map_jit_write_fully(&header, sizeof(header)); +} + +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; + } + + const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) + if (page_size == -1) { + close(fd); + return NULL; + } + + // The perf jit interface forces us to map the first page of the file + // to signal that we are using the interface. + perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); + if (perf_jit_map_state.mapped_buffer == NULL) { + close(fd); + return NULL; + } + perf_jit_map_state.mapped_size = page_size; + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; + } + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; + } + perf_jit_map_state.code_id = 0; + + trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + return &perf_jit_map_state; +} + +/* DWARF definitions. */ + +#define DWRF_CIE_VERSION 1 + +enum { + DWRF_CFA_nop = 0x0, + DWRF_CFA_offset_extended = 0x5, + DWRF_CFA_def_cfa = 0xc, + DWRF_CFA_def_cfa_offset = 0xe, + DWRF_CFA_offset_extended_sf = 0x11, + DWRF_CFA_advance_loc = 0x40, + DWRF_CFA_offset = 0x80 +}; + +enum + { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* FDE data encoding. */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* FDE flags. */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + + DWRF_EH_PE_indirect = 0x80 + }; + +enum { DWRF_TAG_compile_unit = 0x11 }; + +enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; + +enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; + +enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; + +enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; + +enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; + +enum { +#ifdef __x86_64__ + /* Yes, the order is strange, but correct. */ + DWRF_REG_AX, + DWRF_REG_DX, + DWRF_REG_CX, + DWRF_REG_BX, + DWRF_REG_SI, + DWRF_REG_DI, + DWRF_REG_BP, + DWRF_REG_SP, + DWRF_REG_8, + DWRF_REG_9, + DWRF_REG_10, + DWRF_REG_11, + DWRF_REG_12, + DWRF_REG_13, + DWRF_REG_14, + DWRF_REG_15, + DWRF_REG_RA, +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_REG_SP = 31, + DWRF_REG_RA = 30, +#else +# error "Unsupported target architecture" +#endif +}; + +typedef struct ELFObjectContext +{ + uint8_t* p; /* Pointer to next address in obj.space. */ + uint8_t* startp; /* Pointer to start address in obj.space. */ + uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ + uint32_t code_size; /* Size of machine code. */ +} ELFObjectContext; + +/* Append a null-terminated string. */ +static uint32_t +elfctx_append_string(ELFObjectContext* ctx, const char* str) +{ + uint8_t* p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + do { + *p++ = (uint8_t)*str; + } while (*str++); + ctx->p = p; + return ofs; +} + +/* Append a SLEB128 value. */ +static void +elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) +{ + uint8_t* p = ctx->p; + for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { + *p++ = (uint8_t)((v & 0x7f) | 0x80); + } + *p++ = (uint8_t)(v & 0x7f); + ctx->p = p; +} + +/* Append a ULEB128 to buffer. */ +static void +elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) +{ + uint8_t* p = ctx->p; + for (; v >= 0x80; v >>= 7) { + *p++ = (char)((v & 0x7f) | 0x80); + } + *p++ = (char)v; + ctx->p = p; +} + +/* Shortcuts to generate DWARF structures. */ +#define DWRF_U8(x) (*p++ = (x)) +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ + } +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +/* Initialize .eh_frame section. */ +static void +elf_init_ehframe(ELFObjectContext* ctx) +{ + uint8_t* p = ctx->p; + uint8_t* framep = p; + + /* Emit DWARF EH CIE. */ + DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ + DWRF_U8(DWRF_CIE_VERSION); + DWRF_STR("zR"); /* Augmentation. */ + DWRF_UV(1); /* Code alignment factor. */ + DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DWRF_U8(DWRF_REG_RA); /* Return address register. */ + DWRF_UV(1); + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ + DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->eh_frame_p = p; + + /* Emit DWARF EH FDE. */ + DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ + DWRF_U32(-0x30); /* Machine code offset relative to .text. */ + DWRF_U32(ctx->code_size); /* Machine code length. */ + DWRF_U8(0); /* Augmentation data. */ + /* Registers saved in CFRAME. */ +#ifdef __x86_64__ + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_advance_loc | 6); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); + /* Extra registers saved for JIT-compiled code. */ +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_U8(DWRF_CFA_advance_loc | 1); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); + DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); + DWRF_U8(DWRF_CFA_advance_loc | 3); + DWRF_U8(DWRF_CFA_offset | -(64 - 29)); + DWRF_U8(DWRF_CFA_offset | -(64 - 30)); + DWRF_U8(DWRF_CFA_def_cfa_offset); + DWRF_UV(0); +#else +# error "Unsupported target architecture" +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t));) + + ctx->p = p; +} + +static void perf_map_jit_write_entry(void *state, const void *code_addr, + unsigned int code_size, PyCodeObject *co) +{ + + if (perf_jit_map_state.perf_map == NULL) { + void* ret = perf_map_jit_init(); + if(ret == NULL){ + return; + } + } + + const char *entry = ""; + if (co->co_qualname != NULL) { + entry = PyUnicode_AsUTF8(co->co_qualname); + } + const char *filename = ""; + if (co->co_filename != NULL) { + filename = PyUnicode_AsUTF8(co->co_filename); + } + + + size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; + char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); + if (perf_map_entry == NULL) { + return; + } + snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); + + const size_t name_length = strlen(perf_map_entry); + uword base = (uword)code_addr; + uword size = code_size; + + // Write the code unwinding info event. + + // Create unwinding information (eh frame) + ELFObjectContext ctx; + char buffer[1024]; + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + elf_init_ehframe(&ctx); + int eh_frame_size = ctx.p - ctx.startp; + + // Populate the unwind info event for perf + CodeUnwindingInfoEvent ev2; + ev2.base.event = PerfUnwindingInfo; + ev2.base.time_stamp = get_current_monotonic_ticks(); + ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + // Ensure we have enough space between DSOs when perf maps them + assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); + int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; + int padding_size = round_up(content_size, 8) - content_size; + ev2.base.size = content_size + padding_size; + perf_map_jit_write_fully(&ev2, sizeof(ev2)); + + + // Populate the eh Frame header + EhFrameHeader f; + f.version = 1; + f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; + f.fde_count_enc = DwarfUData4; + f.table_enc = DwarfSData4 | DwarfDataRel; + f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); + f.eh_fde_count = 1; + f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; + f.to = -(eh_frame_size - cie_size); + + perf_map_jit_write_fully(ctx.startp, eh_frame_size); + perf_map_jit_write_fully(&f, sizeof(f)); + + char padding_bytes[] = "\0\0\0\0\0\0\0\0"; + perf_map_jit_write_fully(&padding_bytes, padding_size); + + // Write the code load event. + CodeLoadEvent ev; + ev.base.event = PerfLoad; + ev.base.size = sizeof(ev) + (name_length+1) + size; + ev.base.time_stamp = get_current_monotonic_ticks(); + ev.process_id = getpid(); + ev.thread_id = syscall(SYS_gettid); + ev.vma = base; + ev.code_address = base; + ev.code_size = size; + perf_jit_map_state.code_id += 1; + ev.code_id = perf_jit_map_state.code_id; + + perf_map_jit_write_fully(&ev, sizeof(ev)); + perf_map_jit_write_fully(perf_map_entry, name_length+1); + perf_map_jit_write_fully((void*)(base), size); + return; +} + +static int perf_map_jit_fini(void* state) { + if (perf_jit_map_state.perf_map != NULL) { + // close the file + PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); + fclose(perf_jit_map_state.perf_map); + PyThread_release_lock(perf_jit_map_state.map_lock); + + // clean up the lock and state + PyThread_free_lock(perf_jit_map_state.map_lock); + perf_jit_map_state.perf_map = NULL; + } + if (perf_jit_map_state.mapped_buffer != NULL) { + munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + } + trampoline_api.state = NULL; + return 0; +} + +_PyPerf_Callbacks _Py_perfmap_jit_callbacks = { + &perf_map_jit_init, + &perf_map_jit_write_entry, + &perf_map_jit_fini, +}; + +#endif diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 750ba18d3510ed..f144f7d436fe68 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -143,6 +143,8 @@ any DWARF information available for them). #include // mmap() #include #include // sysconf() +#include // gettimeofday() + #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__) #define PY_HAVE_INVALIDATE_ICACHE @@ -187,12 +189,19 @@ struct code_arena_st { typedef struct code_arena_st code_arena_t; typedef struct trampoline_api_st trampoline_api_t; +enum perf_trampoline_type { + PERF_TRAMPOLINE_UNSET = 0, + PERF_TRAMPOLINE_TYPE_MAP = 1, + PERF_TRAMPOLINE_TYPE_JITDUMP = 2, +}; + #define perf_status _PyRuntime.ceval.perf.status #define extra_code_index _PyRuntime.ceval.perf.extra_code_index #define perf_code_arena _PyRuntime.ceval.perf.code_arena #define trampoline_api _PyRuntime.ceval.perf.trampoline_api #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork +#define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type static void perf_map_write_entry(void *state, const void *code_addr, @@ -220,6 +229,8 @@ static void* perf_map_init_state(void) { PyUnstable_PerfMapState_Init(); + trampoline_api.code_padding = 0; + perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; return NULL; } @@ -236,6 +247,30 @@ _PyPerf_Callbacks _Py_perfmap_callbacks = { &perf_map_free_state, }; + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + +// TRAMPOLINE MANAGEMENT API + static int new_code_arena(void) { @@ -256,6 +291,7 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; + size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -264,9 +300,9 @@ new_code_arena(void) // measurable performance improvement by rounding trampolines up to 32-bit // or 64-bit alignment. - size_t n_copies = mem_size / code_size; + size_t n_copies = mem_size / chunk_size; for (size_t i = 0; i < n_copies; i++) { - memcpy(memory + i * code_size, start, code_size * sizeof(char)); + memcpy(memory + i * chunk_size, start, code_size * sizeof(char)); } // Some systems may prevent us from creating executable code on the fly. int res = mprotect(memory, mem_size, PROT_READ | PROT_EXEC); @@ -320,16 +356,18 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - code_arena->size_left -= code_arena->code_size; - code_arena->current_addr += code_arena->code_size; + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + code_arena->size_left -= total_code_size; + code_arena->current_addr += total_code_size; return trampoline; } static inline py_trampoline compile_trampoline(void) { + size_t total_code_size = round_up(perf_code_arena->code_size + trampoline_api.code_padding, 16); if ((perf_code_arena == NULL) || - (perf_code_arena->size_left <= perf_code_arena->code_size)) { + (perf_code_arena->size_left <= total_code_size)) { if (new_code_arena() < 0) { return NULL; } @@ -480,6 +518,7 @@ _PyPerfTrampoline_Fini(void) } if (perf_status == PERF_STATUS_OK) { trampoline_api.free_state(trampoline_api.state); + perf_trampoline_type = PERF_TRAMPOLINE_UNSET; } extra_code_index = -1; perf_status = PERF_STATUS_NO_INIT; @@ -508,6 +547,9 @@ _PyPerfTrampoline_AfterFork_Child(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE if (persist_after_fork) { + if (perf_trampoline_type != PERF_TRAMPOLINE_TYPE_MAP) { + return PyStatus_Error("Failed to copy perf map file as perf trampoline type is not type map."); + } _PyPerfTrampoline_Fini(); char filename[256]; pid_t parent_pid = getppid(); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0f3ca4a6687514..67bbbd01ca0c48 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -71,6 +71,9 @@ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); +#ifdef __ANDROID__ +static PyStatus init_android_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -621,9 +624,11 @@ static int builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) { PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef _Py_TIER2 if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { _Py_Executors_InvalidateAll(interp, 1); } +#endif RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; } @@ -849,6 +854,11 @@ pycore_interp_init(PyThreadState *tstate) return status; } + status = _PyCode_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _PyDtoa_Init(interp); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1205,7 +1215,14 @@ init_interp_main(PyThreadState *tstate) #ifdef PY_HAVE_PERF_TRAMPOLINE if (config->perf_profiling) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_callbacks) < 0 || + _PyPerf_Callbacks *cur_cb; + if (config->perf_profiling == 1) { + cur_cb = &_Py_perfmap_callbacks; + } + else { + cur_cb = &_Py_perfmap_jit_callbacks; + } + if (_PyPerfTrampoline_SetCallbacks(cur_cb) < 0 || _PyPerfTrampoline_Init(config->perf_profiling) < 0) { return _PyStatus_ERR("can't initialize the perf trampoline"); } @@ -1223,6 +1240,13 @@ init_interp_main(PyThreadState *tstate) return status; } +#ifdef __ANDROID__ + status = init_android_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + #ifdef Py_DEBUG run_presite(tstate); #endif @@ -1262,30 +1286,30 @@ init_interp_main(PyThreadState *tstate) } // Turn on experimental tier 2 (uops-based) optimizer + // This is also needed when the JIT is enabled +#ifdef _Py_TIER2 if (is_main_interp) { -#ifndef _Py_JIT - // No JIT, maybe use the tier two interpreter: - char *envvar = Py_GETENV("PYTHON_UOPS"); - int enabled = envvar != NULL && *envvar > '0'; - if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { - enabled = 1; + int enabled = 1; +#if _Py_TIER2 & 2 + enabled = 0; +#endif + char *env = Py_GETENV("PYTHON_JIT"); + if (env && *env != '\0') { + // PYTHON_JIT=0|1 overrides the default + enabled = *env != '0'; } if (enabled) { -#else - // Always enable tier two for JIT builds (ignoring the environment - // variable and command-line option above): - if (true) { -#endif PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) { - return _PyStatus_ERR("can't initialize optimizer"); + return _PyStatus_ERR("can't install optimizer"); } Py_DECREF(opt); } } +#endif if (!is_main_interp) { // The main interpreter is handled in Py_Main(), for now. @@ -1645,10 +1669,12 @@ finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; +#ifdef _Py_TIER2 // Invalidate all executors and turn off tier 2 optimizer _Py_Executors_InvalidateAll(interp, 0); _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); Py_XDECREF(old); +#endif // Stop watching __builtin__ modifications PyDict_Unwatch(0, interp->builtins); @@ -1806,6 +1832,8 @@ finalize_interp_types(PyInterpreterState *interp) _PyTypes_Fini(interp); + _PyCode_Fini(interp); + // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses // a dict internally. _PyUnicode_ClearInterned(interp); @@ -2334,7 +2362,7 @@ static PyStatus add_main_module(PyInterpreterState *interp) { PyObject *m, *d, *ann_dict; - m = PyImport_AddModule("__main__"); + m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); @@ -2400,54 +2428,6 @@ init_import_site(void) return _PyStatus_OK(); } -/* Check if a file descriptor is valid or not. - Return 0 if the file descriptor is invalid, return non-zero otherwise. */ -static int -is_valid_fd(int fd) -{ -/* dup() is faster than fstat(): fstat() can require input/output operations, - whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python - startup. Problem: dup() doesn't check if the file descriptor is valid on - some platforms. - - fcntl(fd, F_GETFD) is even faster, because it only checks the process table. - It is preferred over dup() when available, since it cannot fail with the - "too many open files" error (EMFILE). - - bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other - side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with - EBADF. FreeBSD has similar issue (bpo-32849). - - Only use dup() on Linux where dup() is enough to detect invalid FD - (bpo-32849). -*/ - if (fd < 0) { - return 0; - } -#if defined(F_GETFD) && ( \ - defined(__linux__) || \ - defined(__APPLE__) || \ - defined(__wasm__)) - return fcntl(fd, F_GETFD) >= 0; -#elif defined(__linux__) - int fd2 = dup(fd); - if (fd2 >= 0) { - close(fd2); - } - return (fd2 >= 0); -#elif defined(MS_WINDOWS) - HANDLE hfile; - _Py_BEGIN_SUPPRESS_IPH - hfile = (HANDLE)_get_osfhandle(fd); - _Py_END_SUPPRESS_IPH - return (hfile != INVALID_HANDLE_VALUE - && GetFileType(hfile) != FILE_TYPE_UNKNOWN); -#else - struct stat st; - return (fstat(fd, &st) == 0); -#endif -} - /* returns Py_None if the fd is not valid */ static PyObject* create_stdio(const PyConfig *config, PyObject* io, @@ -2461,8 +2441,9 @@ create_stdio(const PyConfig *config, PyObject* io, int buffering, isatty; const int buffered_stdio = config->buffered_stdio; - if (!is_valid_fd(fd)) + if (!_Py_IsValidFD(fd)) { Py_RETURN_NONE; + } /* stdin is always opened in buffered mode, first because it shouldn't make a difference in common use cases, second because TextIOWrapper @@ -2578,9 +2559,9 @@ create_stdio(const PyConfig *config, PyObject* io, Py_XDECREF(text); Py_XDECREF(raw); - if (PyErr_ExceptionMatches(PyExc_OSError) && !is_valid_fd(fd)) { + if (PyErr_ExceptionMatches(PyExc_OSError) && !_Py_IsValidFD(fd)) { /* Issue #24891: the file descriptor was closed after the first - is_valid_fd() check was called. Ignore the OSError and set the + _Py_IsValidFD() check was called. Ignore the OSError and set the stream to None. */ PyErr_Clear(); Py_RETURN_NONE; @@ -2719,6 +2700,73 @@ init_sys_streams(PyThreadState *tstate) } +#ifdef __ANDROID__ +#include + +static PyObject * +android_log_write_impl(PyObject *self, PyObject *args) +{ + int prio = 0; + const char *tag = NULL; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { + return NULL; + } + + // Despite its name, this function is part of the public API + // (https://developer.android.com/ndk/reference/group/logging). + __android_log_write(prio, tag, text); + Py_RETURN_NONE; +} + + +static PyMethodDef android_log_write_method = { + "android_log_write", android_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_android_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_android_support = NULL; + PyObject *android_log_write = NULL; + PyObject *result = NULL; + + _android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) { + goto error; + } + + android_log_write = PyCFunction_New(&android_log_write_method, NULL); + if (android_log_write == NULL) { + goto error; + } + + // These log priorities match those used by Java's System.out and System.err. + result = PyObject_CallMethod( + _android_support, "init_streams", "Oii", + android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Android streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(android_log_write); + Py_XDECREF(_android_support); + return status; +} + +#endif // __ANDROID__ + + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PyThreadState *tstate) diff --git a/Python/pystate.c b/Python/pystate.c index 78b39c9a577404..b1e085bb806915 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -404,6 +404,7 @@ _Py_COMP_DIAG_POP &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ + &(runtime)->_main_interpreter.code_state.mutex, \ } static void @@ -653,8 +654,10 @@ init_interpreter(PyInterpreterState *interp, } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; +#ifdef _Py_TIER2 (void)_Py_SetOptimizer(interp, NULL); interp->executor_list_head = NULL; +#endif if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); @@ -806,9 +809,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) tstate->_status.cleared = 0; } +#ifdef _Py_TIER2 _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); assert(old != NULL); Py_DECREF(old); +#endif /* It is possible that any of the objects below have a finalizer that runs Python code or otherwise relies on a thread state @@ -839,9 +844,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } PyConfig_Clear(&interp->config); - Py_CLEAR(interp->codec_search_path); - Py_CLEAR(interp->codec_search_cache); - Py_CLEAR(interp->codec_error_registry); + _PyCodec_Fini(interp); assert(interp->imports.modules == NULL); assert(interp->imports.modules_by_index == NULL); @@ -1485,6 +1488,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_limit = NULL; tstate->what_event = -1; tstate->previous_executor = NULL; + tstate->dict_global_version = 0; tstate->delete_later = NULL; @@ -2053,19 +2057,36 @@ _PyThreadState_Attach(PyThreadState *tstate) Py_FatalError("non-NULL old thread state"); } - _PyEval_AcquireLock(tstate); - // XXX assert(tstate_is_alive(tstate)); - current_fast_set(&_PyRuntime, tstate); - tstate_activate(tstate); + while (1) { + int acquired_gil = _PyEval_AcquireLock(tstate); - if (!tstate_try_attach(tstate)) { - tstate_wait_attach(tstate); - } + // XXX assert(tstate_is_alive(tstate)); + current_fast_set(&_PyRuntime, tstate); + tstate_activate(tstate); + + if (!tstate_try_attach(tstate)) { + tstate_wait_attach(tstate); + } #ifdef Py_GIL_DISABLED - _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); + if (_PyEval_IsGILEnabled(tstate) != acquired_gil) { + // The GIL was enabled between our call to _PyEval_AcquireLock() + // and when we attached (the GIL can't go from enabled to disabled + // here because only a thread holding the GIL can disable + // it). Detach and try again. + assert(!acquired_gil); + tstate_set_detached(tstate, _Py_THREAD_DETACHED); + tstate_deactivate(tstate); + current_fast_clear(&_PyRuntime); + continue; + } + _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); +#else + (void)acquired_gil; #endif + break; + } // Resume previous critical section. This acquires the lock(s) from the // top-most critical section. @@ -2238,7 +2259,8 @@ stop_the_world(struct _stoptheworld_state *stw) } PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) - if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) { + int detach = 0; + if (PyEvent_WaitTimed(&stw->stop_event, wait_ns, detach)) { assert(stw->thread_countdown == 0); break; } @@ -2820,9 +2842,11 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, if (eval_frame == interp->eval_frame) { return; } +#ifdef _Py_TIER2 if (eval_frame != NULL) { _Py_Executors_InvalidateAll(interp, 1); } +#endif RARE_EVENT_INC(set_eval_frame_func); interp->eval_frame = eval_frame; } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 2970248da13705..ce7f194e929c9c 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -83,8 +83,6 @@ _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, return res; } - -/* Parse input from a file and execute it */ int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) @@ -1275,17 +1273,20 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py _PyRuntime.signals.unhandled_keyboard_interrupt = 0; /* Set globals['__builtins__'] if it doesn't exist */ - if (globals != NULL) { - int has_builtins = PyDict_ContainsString(globals, "__builtins__"); - if (has_builtins < 0) { + if (!globals || !PyDict_Check(globals)) { + PyErr_SetString(PyExc_SystemError, "globals must be a real dict"); + return NULL; + } + int has_builtins = PyDict_ContainsString(globals, "__builtins__"); + if (has_builtins < 0) { + return NULL; + } + if (!has_builtins) { + if (PyDict_SetItemString(globals, "__builtins__", + tstate->interp->builtins) < 0) + { return NULL; } - if (!has_builtins) { - if (PyDict_SetItemString(globals, "__builtins__", - tstate->interp->builtins) < 0) { - return NULL; - } - } } v = PyEval_EvalCode((PyObject*)co, globals, locals); diff --git a/Python/pytime.c b/Python/pytime.c index d5b38047b6db31..560aea33f201a0 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -898,6 +898,10 @@ static int py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); + if (raise_exc) { + // raise_exc requires to hold the GIL + assert(PyGILState_Check()); + } #ifdef MS_WINDOWS FILETIME system_time; @@ -1004,29 +1008,28 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -PyTime_t -_PyTime_TimeUnchecked(void) +int +PyTime_Time(PyTime_t *result) { - PyTime_t t; - if (py_get_system_clock(&t, NULL, 0) < 0) { - // If clock_gettime(CLOCK_REALTIME) or gettimeofday() fails: - // silently ignore the failure and return 0. - t = 0; + if (py_get_system_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; } - return t; + return 0; } int -PyTime_Time(PyTime_t *result) +PyTime_TimeRaw(PyTime_t *result) { - if (py_get_system_clock(result, NULL, 1) < 0) { + if (py_get_system_clock(result, NULL, 0) < 0) { *result = 0; return -1; } return 0; } + int _PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) { @@ -1140,6 +1143,10 @@ static int py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); + if (raise_exc) { + // raise_exc requires to hold the GIL + assert(PyGILState_Check()); + } #if defined(MS_WINDOWS) if (py_get_win_perf_counter(tp, info, raise_exc) < 0) { @@ -1225,22 +1232,21 @@ py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -PyTime_t -_PyTime_MonotonicUnchecked(void) +int +PyTime_Monotonic(PyTime_t *result) { - PyTime_t t; - if (py_get_monotonic_clock(&t, NULL, 0) < 0) { - // Ignore silently the error and return 0. - t = 0; + if (py_get_monotonic_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; } - return t; + return 0; } int -PyTime_Monotonic(PyTime_t *result) +PyTime_MonotonicRaw(PyTime_t *result) { - if (py_get_monotonic_clock(result, NULL, 1) < 0) { + if (py_get_monotonic_clock(result, NULL, 0) < 0) { *result = 0; return -1; } @@ -1262,17 +1268,17 @@ _PyTime_PerfCounterWithInfo(PyTime_t *t, _Py_clock_info_t *info) } -PyTime_t -_PyTime_PerfCounterUnchecked(void) +int +PyTime_PerfCounter(PyTime_t *result) { - return _PyTime_MonotonicUnchecked(); + return PyTime_Monotonic(result); } int -PyTime_PerfCounter(PyTime_t *result) +PyTime_PerfCounterRaw(PyTime_t *result) { - return PyTime_Monotonic(result); + return PyTime_MonotonicRaw(result); } @@ -1346,7 +1352,9 @@ _PyTime_gmtime(time_t t, struct tm *tm) PyTime_t _PyDeadline_Init(PyTime_t timeout) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return _PyTime_Add(now, timeout); } @@ -1354,6 +1362,8 @@ _PyDeadline_Init(PyTime_t timeout) PyTime_t _PyDeadline_Get(PyTime_t deadline) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return deadline - now; } diff --git a/Python/specialize.c b/Python/specialize.c index ee51781372166a..9ac428c3593f56 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -215,6 +215,7 @@ print_gc_stats(FILE *out, GCStats *stats) } } +#ifdef _Py_TIER2 static void print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) { @@ -249,7 +250,6 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) stats->optimizer_failure_reason_no_memory); fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); - for (int i = 0; i <= MAX_UOP_ID; i++) { if (stats->opcode[i].execution_count) { fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); @@ -258,7 +258,6 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); } } - for (int i = 0; i < 256; i++) { if (stats->unsupported_opcode[i]) { fprintf( @@ -289,6 +288,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) } } } +#endif static void print_rare_event_stats(FILE *out, RareEventStats *stats) @@ -309,7 +309,9 @@ print_stats(FILE *out, PyStats *stats) print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); +#ifdef _Py_TIER2 print_optimization_stats(out, &stats->optimization_stats); +#endif print_rare_event_stats(out, &stats->rare_event_stats); } @@ -1787,8 +1789,7 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return -1; } if (Py_TYPE(tp) != &PyType_Type) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_METACLASS); - return -1; + goto generic; } if (tp->tp_new == PyBaseObject_Type.tp_new) { PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); @@ -1805,58 +1806,11 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); return 0; } - return -1; - } - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); - return -1; -} - -#ifdef Py_STATS -static int -builtin_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_CFUNC_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; - case METH_NOARGS: - return SPEC_FAIL_CALL_CFUNC_NOARGS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; - } -} - -static int -meth_descr_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_NOARGS: - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; } +generic: + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } -#endif // Py_STATS static int specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, @@ -1899,8 +1853,8 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, return 0; } } - SPECIALIZATION_FAIL(CALL, meth_descr_call_fail_kind(descr->d_method->ml_flags)); - return -1; + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } static int @@ -1915,36 +1869,25 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } - if (kind != SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(CALL, kind); + int argcount = -1; + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; } - int argcount = code->co_argcount; - int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults); - int min_args = argcount-defcount; - // GH-105840: min_args is negative when somebody sets too many __defaults__! - if (min_args < 0 || nargs > argcount || nargs < min_args) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; + if (kind == SIMPLE_FUNCTION) { + argcount = code->co_argcount; } - assert(nargs <= argcount && nargs >= min_args); - assert(min_args >= 0 && defcount >= 0); - assert(defcount == 0 || func->func_defaults != NULL); int version = _PyFunction_GetVersionForCurrentState(func); if (version == 0) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->func_version, version); - if (argcount == nargs) { + if (argcount == nargs + bound_method) { instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } - else if (bound_method) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); - return -1; - } else { - instr->op.code = CALL_PY_WITH_DEFAULTS; + instr->op.code = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; } return 0; } @@ -1953,6 +1896,7 @@ static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { if (PyCFunction_GET_FUNCTION(callable) == NULL) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER); return 1; } switch (PyCFunction_GET_FLAGS(callable) & @@ -1989,38 +1933,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return 0; } default: - SPECIALIZATION_FAIL(CALL, - builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); - return 1; - } -} - -#ifdef Py_STATS -static int -call_fail_kind(PyObject *callable) -{ - assert(!PyCFunction_CheckExact(callable)); - assert(!PyFunction_Check(callable)); - assert(!PyType_Check(callable)); - assert(!Py_IS_TYPE(callable, &PyMethodDescr_Type)); - assert(!PyMethod_Check(callable)); - if (PyInstanceMethod_Check(callable)) { - return SPEC_FAIL_CALL_INSTANCE_METHOD; - } - // builtin method - else if (PyCMethod_Check(callable)) { - return SPEC_FAIL_CALL_CMETHOD; - } - else if (Py_TYPE(callable) == &PyWrapperDescr_Type) { - return SPEC_FAIL_CALL_OPERATOR_WRAPPER; - } - else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) { - return SPEC_FAIL_CALL_METHOD_WRAPPER; + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } - return SPEC_FAIL_OTHER; } -#endif // Py_STATS - void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) @@ -2045,7 +1961,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { - fail = specialize_py_call((PyFunctionObject *)func, instr, nargs+1, true); + fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true); } else { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); @@ -2053,8 +1969,8 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } } else { - SPECIALIZATION_FAIL(CALL, call_fail_kind(callable)); - fail = -1; + instr->op.code = CALL_NON_PY_GENERAL; + fail = 0; } if (fail) { STAT_INC(CALL, failure); diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 08a66f447e2258..9686d10563aa4d 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -5,6 +5,7 @@ static const char* _Py_stdlib_module_names[] = { "__future__", "_abc", "_aix_support", +"_android_support", "_ast", "_asyncio", "_bisect", @@ -19,6 +20,7 @@ static const char* _Py_stdlib_module_names[] = { "_codecs_tw", "_collections", "_collections_abc", +"_colorize", "_compat_pickle", "_compression", "_contextvars", @@ -63,6 +65,7 @@ static const char* _Py_stdlib_module_names[] = { "_pydecimal", "_pyio", "_pylong", +"_pyrepl", "_queue", "_random", "_scproxy", diff --git a/Python/symtable.c b/Python/symtable.c index eecd159b2c3f17..2ec21a2d376da2 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -2275,6 +2275,24 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_QUIT(st, 1); } +static int +symtable_visit_type_param_bound_or_default(struct symtable *st, expr_ty e, identifier name, void *key) +{ + if (e) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + if (!symtable_enter_block(st, name, TypeVarBoundBlock, key, LOCATION(e))) + return 0; + st->st_cur->ste_can_see_class_scope = is_in_class; + if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(e))) { + VISIT_QUIT(st, 0); + } + VISIT(st, expr, e); + if (!symtable_exit_block(st)) + return 0; + } + return 1; +} + static int symtable_visit_type_param(struct symtable *st, type_param_ty tp) { @@ -2287,28 +2305,39 @@ symtable_visit_type_param(struct symtable *st, type_param_ty tp) case TypeVar_kind: if (!symtable_add_def(st, tp->v.TypeVar.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) VISIT_QUIT(st, 0); - if (tp->v.TypeVar.bound) { - int is_in_class = st->st_cur->ste_can_see_class_scope; - if (!symtable_enter_block(st, tp->v.TypeVar.name, - TypeVarBoundBlock, (void *)tp, - LOCATION(tp))) - VISIT_QUIT(st, 0); - st->st_cur->ste_can_see_class_scope = is_in_class; - if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(tp->v.TypeVar.bound))) { - VISIT_QUIT(st, 0); - } - VISIT(st, expr, tp->v.TypeVar.bound); - if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + + // We must use a different key for the bound and default. The obvious choice would be to + // use the .bound and .default_value pointers, but that fails when the expression immediately + // inside the bound or default is a comprehension: we would reuse the same key for + // the comprehension scope. Therefore, use the address + 1 as the second key. + // The only requirement for the key is that it is unique and it matches the logic in + // compile.c where the scope is retrieved. + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.bound, tp->v.TypeVar.name, + (void *)tp)) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.default_value, tp->v.TypeVar.name, + (void *)((uintptr_t)tp + 1))) { + VISIT_QUIT(st, 0); } break; case TypeVarTuple_kind: - if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { VISIT_QUIT(st, 0); + } + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVarTuple.default_value, tp->v.TypeVarTuple.name, + (void *)tp)) { + VISIT_QUIT(st, 0); + } break; case ParamSpec_kind: - if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { VISIT_QUIT(st, 0); + } + if (!symtable_visit_type_param_bound_or_default(st, tp->v.ParamSpec.default_value, tp->v.ParamSpec.name, + (void *)tp)) { + VISIT_QUIT(st, 0); + } break; } VISIT_QUIT(st, 1); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 7af363678e8e86..4da13e4552e786 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1022,13 +1022,6 @@ static PyObject * call_trampoline(PyThreadState *tstate, PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - /* Discard any previous modifications the frame's fast locals */ - if (frame->f_fast_as_locals) { - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - } - /* call the Python-level function */ if (arg == NULL) { arg = Py_None; @@ -1036,7 +1029,6 @@ call_trampoline(PyThreadState *tstate, PyObject* callback, PyObject *args[3] = {(PyObject *)frame, whatstrings[what], arg}; PyObject *result = _PyObject_VectorcallTstate(tstate, callback, args, 3, NULL); - PyFrame_LocalsToFast(frame, 1); return result; } @@ -1407,12 +1399,6 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(finalizer)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { - return NULL; - } - } - else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { - return NULL; } if (firstiter && firstiter != Py_None) { @@ -1422,15 +1408,33 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(firstiter)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + } + + PyObject *cur_finalizer = _PyEval_GetAsyncGenFinalizer(); + + if (finalizer && finalizer != Py_None) { + if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { return NULL; } } - else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { return NULL; } + if (firstiter && firstiter != Py_None) { + if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + goto error; + } + } + else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + goto error; + } + Py_RETURN_NONE; + +error: + _PyEval_SetAsyncGenFinalizer(cur_finalizer); + return NULL; } PyDoc_STRVAR(set_asyncgen_hooks_doc, @@ -2165,8 +2169,10 @@ static PyObject * sys__clear_internal_caches_impl(PyObject *module) /*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ { +#ifdef _Py_TIER2 PyInterpreterState *interp = _PyInterpreterState_GET(); _Py_Executors_InvalidateAll(interp, 0); +#endif PyType_ClearCache(); Py_RETURN_NONE; } @@ -2288,6 +2294,16 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; + } + } + } } else { PyErr_Format(PyExc_ValueError, "invalid backend: %s", backend); @@ -2391,6 +2407,39 @@ sys__get_cpu_count_config_impl(PyObject *module) return config->cpu_count; } +/*[clinic input] +sys._baserepl + +Private function for getting the base REPL +[clinic start generated code]*/ + +static PyObject * +sys__baserepl_impl(PyObject *module) +/*[clinic end generated code: output=f19a36375ebe0a45 input=ade0ebb9fab56f3c]*/ +{ + PyCompilerFlags cf = _PyCompilerFlags_INIT; + PyRun_AnyFileExFlags(stdin, "", 0, &cf); + Py_RETURN_NONE; +} + +/*[clinic input] +sys._is_gil_enabled -> bool + +Return True if the GIL is currently enabled and False otherwise. +[clinic start generated code]*/ + +static int +sys__is_gil_enabled_impl(PyObject *module) +/*[clinic end generated code: output=57732cf53f5b9120 input=7e9c47f15a00e809]*/ +{ +#ifdef Py_GIL_DISABLED + return _PyEval_IsGILEnabled(_PyThreadState_GET()); +#else + return 1; +#endif +} + + static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { @@ -2494,10 +2543,6 @@ PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { return 0; } -#ifdef __cplusplus -} -#endif - static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ @@ -2560,6 +2605,7 @@ static PyMethodDef sys_methods[] = { SYS_UNRAISABLEHOOK_METHODDEF SYS_GET_INT_MAX_STR_DIGITS_METHODDEF SYS_SET_INT_MAX_STR_DIGITS_METHODDEF + SYS__BASEREPL_METHODDEF #ifdef Py_STATS SYS__STATS_ON_METHODDEF SYS__STATS_OFF_METHODDEF @@ -2567,6 +2613,7 @@ static PyMethodDef sys_methods[] = { SYS__STATS_DUMP_METHODDEF #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF + SYS__IS_GIL_ENABLED_METHODDEF {NULL, NULL} // sentinel }; @@ -3738,6 +3785,9 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) if (sysmod == NULL) { return _PyStatus_ERR("failed to create a module object"); } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(sysmod, Py_MOD_GIL_NOT_USED); +#endif PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 9dca833ff203ca..425658131c2fce 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -77,17 +77,18 @@ EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) } } else if (milliseconds != 0) { /* wait at least until the deadline */ - PyTime_t nanoseconds = (PyTime_t)milliseconds * (1000 * 1000); - PyTime_t deadline = _PyTime_Add(_PyTime_PerfCounterUnchecked(), nanoseconds); + PyTime_t timeout = (PyTime_t)milliseconds * (1000 * 1000); + PyTime_t deadline = _PyDeadline_Init(timeout); while (mutex->locked) { - PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, - _PyTime_ROUND_TIMEOUT); + PyTime_t microseconds = _PyTime_AsMicroseconds(timeout, + _PyTime_ROUND_TIMEOUT); if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { result = WAIT_FAILED; break; } - nanoseconds = deadline - _PyTime_PerfCounterUnchecked(); - if (nanoseconds <= 0) { + + timeout = _PyDeadline_Get(deadline); + if (timeout <= 0) { break; } } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 65d366e91c322a..f588b4620da0d3 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -158,12 +158,14 @@ _PyThread_cond_after(long long us, struct timespec *abs) PyTime_t t; #ifdef CONDATTR_MONOTONIC if (condattr_monotonic) { - t = _PyTime_MonotonicUnchecked(); + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&t); } else #endif { - t = _PyTime_TimeUnchecked(); + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&t); } t = _PyTime_Add(t, timeout); _PyTime_AsTimespec_clamp(t, abs); @@ -506,7 +508,10 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, struct timespec abs_timeout; // Local scope for deadline { - PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &abs_timeout); } #else @@ -522,8 +527,11 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, &abs_timeout)); #else - PyTime_t abs_time = _PyTime_Add(_PyTime_TimeUnchecked(), - timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t abs_time = _PyTime_Add(now, timeout); + struct timespec ts; _PyTime_AsTimespec_clamp(abs_time, &ts); status = fix_status(sem_timedwait(thelock, &ts)); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 19b64c619feb6a..e3ec72062f6931 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -906,6 +906,10 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } + if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { + return -1; + } + if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ return 0; @@ -1352,8 +1356,12 @@ _PyTraceMalloc_Fini(void) Do nothing if tracemalloc is not tracing memory allocations or if the object memory block is not already traced. */ int -_PyTraceMalloc_NewReference(PyObject *op) +_PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ignore)) { + if (event != PyRefTracer_CREATE) { + return 0; + } + assert(PyGILState_Check()); if (!tracemalloc_config.tracing) { diff --git a/README.rst b/README.rst index cab9519bd7a76c..44b020e0c7ed5f 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.13.0 alpha 6 +This is Python version 3.14.0 alpha 0 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 5c1851f09338a0..258b58c03c6800 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -4,13 +4,13 @@ import hashlib import json import glob -import pathlib +from pathlib import Path, PurePosixPath, PureWindowsPath import subprocess import sys import urllib.request import typing -CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent +CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent # Before adding a new entry to this list, double check that # the license expression is a valid SPDX license expression: @@ -119,9 +119,16 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: # 1 means matches, 0 means no matches. assert git_check_ignore_proc.returncode in (0, 1) + # Paths may or may not be quoted, Windows quotes paths. + git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z") + # Return the list of paths sorted git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines() - return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) + git_check_not_ignored = [] + for line in git_check_ignore_lines: + if match := git_check_ignore_re.fullmatch(line): + git_check_not_ignored.append(match.group(2) or match.group(3)) + return sorted(git_check_not_ignored) def get_externals() -> list[str]: @@ -238,12 +245,20 @@ def create_source_sbom() -> None: ) for path in paths: + + # Normalize the filename from any combination of slashes. + path = str(PurePosixPath(PureWindowsPath(path))) + # Skip directories and excluded files if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude: continue # SPDX requires SHA1 to be used for files, but we provide SHA256 too. data = (CPYTHON_ROOT_DIR / path).read_bytes() + # We normalize line-endings for consistent checksums. + # This is a rudimentary check for binary files. + if b"\x00" not in data: + data = data.replace(b"\r\n", b"\n") checksum_sha1 = hashlib.sha1(data).hexdigest() checksum_sha256 = hashlib.sha256(data).hexdigest() diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 79a8f850ec1451..1b8cccf80872c8 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -43,6 +43,7 @@ Objects/enumobject.c - PyReversed_Type - Objects/fileobject.c - PyStdPrinter_Type - Objects/floatobject.c - PyFloat_Type - Objects/frameobject.c - PyFrame_Type - +Objects/frameobject.c - PyFrameLocalsProxy_Type - Objects/funcobject.c - PyClassMethod_Type - Objects/funcobject.c - PyFunction_Type - Objects/funcobject.c - PyStaticMethod_Type - @@ -92,6 +93,7 @@ Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - Objects/typevarobject.c - _PyTypeAlias_Type - +Objects/typevarobject.c - _PyNoDefault_Type - Objects/unicodeobject.c - PyUnicodeIter_Type - Objects/unicodeobject.c - PyUnicode_Type - Objects/weakrefobject.c - _PyWeakref_CallableProxyType - @@ -310,6 +312,7 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - +Objects/typevarobject.c - _Py_NoDefaultStruct - Python/instrumentation.c - _PyInstrumentation_DISABLE - Python/instrumentation.c - _PyInstrumentation_MISSING - @@ -395,6 +398,7 @@ Modules/xxmodule.c - Str_Type - Modules/xxmodule.c - Xxo_Type - Modules/xxsubtype.c - spamdict_type - Modules/xxsubtype.c - spamlist_type - +Modules/_testcapi/monitoring.c - PyCodeLike_Type - ##----------------------- ## non-static types - initialized once diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 87b695de23e25e..a1c1553d8ddc10 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -344,7 +344,7 @@ Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - Python/codecs.c - ucnhash_capi - -Python/codecs.c _PyCodecRegistry_Init methods - +Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LABEL - Python/compile.c - NO_LOCATION - Python/dynload_shlib.c - _PyImport_DynLoadFiletab - @@ -365,6 +365,8 @@ Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/lock.c - TIME_TO_BE_FAIR_NS - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - +Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - +Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 18cefa08328804..fdb635486b9531 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -411,6 +411,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "PyCell_New", "PyFloat_AS_DOUBLE", "_PyFrame_PushUnchecked", + "Py_FatalError", ) ESCAPING_FUNCTIONS = ( diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 74165acd831131..5fdc812a00f059 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -255,7 +255,7 @@ def proxyval(self, visited): Derived classes will override this. - For example, a PyIntObject* with ob_ival 42 in the inferior process + For example, a PyLongObjectPtr* with long_value 42 in the inferior process should result in an int(42) in this process. visited: a set of all gdb.Value pyobject pointers already visited @@ -867,7 +867,7 @@ class PyLongObjectPtr(PyObjectPtr): def proxyval(self, visited): ''' - Python's Include/longobjrep.h has this declaration: + Python's Include/longinterpr.h has this declaration: typedef struct _PyLongValue { uintptr_t lv_tag; /* Number of digits, sign and flags */ @@ -876,14 +876,18 @@ def proxyval(self, visited): struct _longobject { PyObject_HEAD - _PyLongValue long_value; + _PyLongValue long_value; }; with this description: The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + The sign of the value is stored in the lower 2 bits of lv_tag. + - 0: Positive + - 1: Zero + - 2: Negative + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. where SHIFT can be either: #define PyLong_SHIFT 30 diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 04a6c0780bf972..0f5aa9ce656bc9 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -7,33 +7,37 @@ This version of CPython can be built with an experimental just-in-time compiler. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 18 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-18`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 16 on Ubuntu/Debian: +Install LLVM 18 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 16 +sudo ./llvm.sh 18 ``` ### macOS -Install LLVM 16 with [Homebrew](https://brew.sh): +Install LLVM 18 with [Homebrew](https://brew.sh): ```sh -brew install llvm@16 +brew install llvm@18 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. ### Windows -Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +### Dev Containers + +If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 603bbef59ba2e6..45bd69ff861b56 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -1,4 +1,5 @@ """Utilities for invoking LLVM tools.""" + import asyncio import functools import os @@ -7,7 +8,7 @@ import subprocess import typing -_LLVM_VERSION = 16 +_LLVM_VERSION = 18 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") _P = typing.ParamSpec("_P") diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 045fd502a03c12..228fc389584dd7 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -1,7 +1,9 @@ """Schema for the JSON produced by llvm-readobj --elf-output-style=JSON.""" + import typing HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_BRANCH26", "ARM64_RELOC_GOT_LOAD_PAGE21", "ARM64_RELOC_GOT_LOAD_PAGEOFF12", "ARM64_RELOC_PAGE21", @@ -16,8 +18,10 @@ "IMAGE_REL_I386_REL32", "R_AARCH64_ABS64", "R_AARCH64_ADR_GOT_PAGE", + "R_AARCH64_ADR_PREL_PG_HI21", "R_AARCH64_CALL26", "R_AARCH64_JUMP26", + "R_AARCH64_ADD_ABS_LO12_NC", "R_AARCH64_LD64_GOT_LO12_NC", "R_AARCH64_MOVW_UABS_G0_NC", "R_AARCH64_MOVW_UABS_G1_NC", @@ -68,12 +72,12 @@ class _COFFSymbol(typing.TypedDict): class _ELFSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int class _MachOSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int @@ -99,7 +103,7 @@ class ELFSection(typing.TypedDict): Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] SectionData: dict[typing.Literal["Bytes"], list[int]] Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] - Type: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Name"], str] class MachOSection(typing.TypedDict): diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 243bb3dd134f70..6e046df3026ae9 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -1,7 +1,9 @@ """Core data structures for compiled code templates.""" + import dataclasses import enum import sys +import typing import _schema @@ -29,7 +31,7 @@ class HoleValue(enum.Enum): OPARG = enum.auto() # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): OPERAND = enum.auto() - # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI and _JIT_OPERAND_LO): + # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): OPERAND_HI = enum.auto() OPERAND_LO = enum.auto() # The current uop's target (exposed as _JIT_TARGET): @@ -46,6 +48,73 @@ class HoleValue(enum.Enum): ZERO = enum.auto() +# Map relocation types to our JIT's patch functions. "r" suffixes indicate that +# the patch function is relative. "x" suffixes indicate that they are "relaxing" +# (see comments in jit.c for more info): +_PATCH_FUNCS = { + # aarch64-apple-darwin: + "ARM64_RELOC_BRANCH26": "patch_aarch64_26r", + "ARM64_RELOC_GOT_LOAD_PAGE21": "patch_aarch64_21rx", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12": "patch_aarch64_12x", + "ARM64_RELOC_PAGE21": "patch_aarch64_21r", + "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", + "ARM64_RELOC_UNSIGNED": "patch_64", + # x86_64-pc-windows-msvc: + "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", + # aarch64-pc-windows-msvc: + "IMAGE_REL_ARM64_BRANCH26": "patch_aarch64_26r", + "IMAGE_REL_ARM64_PAGEBASE_REL21": "patch_aarch64_21rx", + "IMAGE_REL_ARM64_PAGEOFFSET_12A": "patch_aarch64_12", + "IMAGE_REL_ARM64_PAGEOFFSET_12L": "patch_aarch64_12x", + # i686-pc-windows-msvc: + "IMAGE_REL_I386_DIR32": "patch_32", + "IMAGE_REL_I386_REL32": "patch_x86_64_32rx", + # aarch64-unknown-linux-gnu: + "R_AARCH64_ABS64": "patch_64", + "R_AARCH64_ADD_ABS_LO12_NC": "patch_aarch64_12", + "R_AARCH64_ADR_GOT_PAGE": "patch_aarch64_21rx", + "R_AARCH64_ADR_PREL_PG_HI21": "patch_aarch64_21r", + "R_AARCH64_CALL26": "patch_aarch64_26r", + "R_AARCH64_JUMP26": "patch_aarch64_26r", + "R_AARCH64_LD64_GOT_LO12_NC": "patch_aarch64_12x", + "R_AARCH64_MOVW_UABS_G0_NC": "patch_aarch64_16a", + "R_AARCH64_MOVW_UABS_G1_NC": "patch_aarch64_16b", + "R_AARCH64_MOVW_UABS_G2_NC": "patch_aarch64_16c", + "R_AARCH64_MOVW_UABS_G3": "patch_aarch64_16d", + # x86_64-unknown-linux-gnu: + "R_X86_64_64": "patch_64", + "R_X86_64_GOTPCREL": "patch_32r", + "R_X86_64_GOTPCRELX": "patch_x86_64_32rx", + "R_X86_64_PC32": "patch_32r", + "R_X86_64_REX_GOTPCRELX": "patch_x86_64_32rx", + # x86_64-apple-darwin: + "X86_64_RELOC_BRANCH": "patch_32r", + "X86_64_RELOC_GOT": "patch_x86_64_32rx", + "X86_64_RELOC_GOT_LOAD": "patch_x86_64_32rx", + "X86_64_RELOC_SIGNED": "patch_32r", + "X86_64_RELOC_UNSIGNED": "patch_64", +} +# Translate HoleValues to C expressions: +_HOLE_EXPRS = { + HoleValue.CODE: "(uintptr_t)code", + HoleValue.CONTINUE: "(uintptr_t)code + sizeof(code_body)", + HoleValue.DATA: "(uintptr_t)data", + HoleValue.EXECUTOR: "(uintptr_t)executor", + # These should all have been turned into DATA values by process_relocations: + # HoleValue.GOT: "", + HoleValue.OPARG: "instruction->oparg", + HoleValue.OPERAND: "instruction->operand", + HoleValue.OPERAND_HI: "(instruction->operand >> 32)", + HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", + HoleValue.TARGET: "instruction->target", + HoleValue.JUMP_TARGET: "instruction_starts[instruction->jump_target]", + HoleValue.ERROR_TARGET: "instruction_starts[instruction->error_target]", + HoleValue.EXIT_INDEX: "instruction->exit_index", + HoleValue.TOP: "instruction_starts[1]", + HoleValue.ZERO: "", +} + + @dataclasses.dataclass class Hole: """ @@ -62,19 +131,43 @@ class Hole: symbol: str | None # ...plus this addend: addend: int + func: str = dataclasses.field(init=False) # Convenience method: replace = dataclasses.replace - def as_c(self) -> str: - """Dump this hole as an initialization of a C Hole struct.""" - parts = [ - f"{self.offset:#x}", - f"HoleKind_{self.kind}", - f"HoleValue_{self.value.name}", - f"&{self.symbol}" if self.symbol else "NULL", - f"{_signed(self.addend):#x}", - ] - return f"{{{', '.join(parts)}}}" + def __post_init__(self) -> None: + self.func = _PATCH_FUNCS[self.kind] + + def fold(self, other: typing.Self) -> typing.Self | None: + """Combine two holes into a single hole, if possible.""" + if ( + self.offset + 4 == other.offset + and self.value == other.value + and self.symbol == other.symbol + and self.addend == other.addend + and self.func == "patch_aarch64_21rx" + and other.func == "patch_aarch64_12x" + ): + # These can *only* be properly relaxed when they appear together and + # patch the same value: + folded = self.replace() + folded.func = "patch_aarch64_33rx" + return folded + return None + + def as_c(self, where: str) -> str: + """Dump this hole as a call to a patch_* function.""" + location = f"{where} + {self.offset:#x}" + value = _HOLE_EXPRS[self.value] + if self.symbol: + if value: + value += " + " + value += f"(uintptr_t)&{self.symbol}" + if _signed(self.addend): + if value: + value += " + " + value += f"{_signed(self.addend):#x}" + return f"{self.func}({location}, {value});" @dataclasses.dataclass @@ -203,7 +296,8 @@ def process_relocations(self, *, alignment: int = 1) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( - hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + hole.kind + in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} and hole.value is HoleValue.ZERO ): self.code.pad(alignment) @@ -263,6 +357,10 @@ def _emit_global_offset_table(self) -> None: ) self.data.body.extend([0] * 8) + def as_c(self, opname: str) -> str: + """Dump this hole as a StencilGroup initializer.""" + return f"{{emit_{opname}, {len(self.code.body)}, {len(self.data.body)}}}" + def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: """ diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 66db358679239e..023ef498a21d7c 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -1,4 +1,5 @@ """Target-specific code generation, parsing, and processing.""" + import asyncio import dataclasses import hashlib @@ -38,9 +39,10 @@ class _Target(typing.Generic[_S, _R]): _: dataclasses.KW_ONLY alignment: int = 1 args: typing.Sequence[str] = () + ghccc: bool = False prefix: str = "" + stable: bool = False debug: bool = False - force: bool = False verbose: bool = False def _compute_digest(self, out: pathlib.Path) -> str: @@ -85,7 +87,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) for wrapped_section in sections: self._handle_section(wrapped_section["Section"], group) - assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) + # The trampoline's entry point is just named "_ENTRY", since on some + # platforms we later assume that any function starting with "_JIT_" uses + # the GHC calling convention: + entry_symbol = "_JIT_ENTRY" if "_JIT_ENTRY" in group.symbols else "_ENTRY" + assert group.symbols[entry_symbol] == (_stencils.HoleValue.CODE, 0) if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) @@ -103,6 +109,9 @@ def _handle_relocation( async def _compile( self, opname: str, c: pathlib.Path, tempdir: pathlib.Path ) -> _stencils.StencilGroup: + # "Compile" the trampoline to an empty stencil group if it's not needed: + if opname == "trampoline" and not self.ghccc: + return _stencils.StencilGroup() o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", @@ -130,13 +139,45 @@ async def _compile( "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", - "-o", - f"{o}", "-std=c11", - f"{c}", *self.args, ] - await _llvm.run("clang", args, echo=self.verbose) + if self.ghccc: + # This is a bit of an ugly workaround, but it makes the code much + # smaller and faster, so it's worth it. We want to use the GHC + # calling convention, but Clang doesn't support it. So, we *first* + # compile the code to LLVM IR, perform some text replacements on the + # IR to change the calling convention(!), and then compile *that*. + # Once we have access to Clang 19, we can get rid of this and use + # __attribute__((preserve_none)) directly in the C code instead: + ll = tempdir / f"{opname}.ll" + args_ll = args + [ + # -fomit-frame-pointer is necessary because the GHC calling + # convention uses RBP to pass arguments: + "-S", + "-emit-llvm", + "-fomit-frame-pointer", + "-o", + f"{ll}", + f"{c}", + ] + await _llvm.run("clang", args_ll, echo=self.verbose) + ir = ll.read_text() + # This handles declarations, definitions, and calls to named symbols + # starting with "_JIT_": + ir = re.sub( + r"(((noalias|nonnull|noundef) )*ptr @_JIT_\w+\()", r"ghccc \1", ir + ) + # This handles calls to anonymous callees, since anything with + # "musttail" needs to use the same calling convention: + ir = ir.replace("musttail call", "musttail call ghccc") + # Sometimes *both* replacements happen at the same site, so fix it: + ir = ir.replace("ghccc ghccc", "ghccc") + ll.write_text(ir) + args_o = args + ["-Wno-unused-command-line-argument", "-o", f"{o}", f"{ll}"] + else: + args_o = args + ["-o", f"{o}", f"{c}"] + await _llvm.run("clang", args_o, echo=self.verbose) return await self._parse(o) async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: @@ -146,17 +187,26 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: + coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) + tasks.append(group.create_task(coro, name="trampoline")) for opname in opnames: coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) tasks.append(group.create_task(coro, name=opname)) return {task.get_name(): task.result() for task in tasks} - def build(self, out: pathlib.Path, *, comment: str = "") -> None: + def build( + self, out: pathlib.Path, *, comment: str = "", force: bool = False + ) -> None: """Build jit_stencils.h in the given directory.""" + if not self.stable: + warning = f"JIT support for {self.triple} is still experimental!" + request = "Please report any issues you encounter.".center(len(warning)) + outline = "=" * len(warning) + print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest(out)}\n" jit_stencils = out / "jit_stencils.h" if ( - not self.force + not force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) ): @@ -221,7 +271,7 @@ def _handle_relocation( case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + "Type": {"Name": "IMAGE_REL_I386_DIR32" as kind}, }: offset += base value, symbol = self._unwrap_dllimport(s) @@ -230,7 +280,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind }, }: offset += base @@ -242,7 +292,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_ARM64_BRANCH26" + "Name": "IMAGE_REL_ARM64_BRANCH26" | "IMAGE_REL_ARM64_PAGEBASE_REL21" | "IMAGE_REL_ARM64_PAGEOFFSET_12A" | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind @@ -262,7 +312,7 @@ class _ELF( def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: - section_type = section["Type"]["Value"] + section_type = section["Type"]["Name"] flags = {flag["Name"] for flag in section["Flags"]["Flags"]} if section_type == "SHT_RELA": assert "SHF_INFO_LINK" in flags, flags @@ -290,7 +340,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = len(stencil.body) + symbol["Value"] - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset stencil.body.extend(section["SectionData"]["Bytes"]) @@ -312,9 +362,9 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "R_AARCH64_ADR_GOT_PAGE" + "Name": "R_AARCH64_ADR_GOT_PAGE" | "R_AARCH64_LD64_GOT_LO12_NC" | "R_X86_64_GOTPCREL" | "R_X86_64_GOTPCRELX" @@ -327,8 +377,8 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -371,7 +421,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = symbol["Value"] - start_address - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset assert "Relocations" in section @@ -387,9 +437,9 @@ def _handle_relocation( match relocation: case { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + "Name": "ARM64_RELOC_GOT_LOAD_PAGE21" | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind }, }: @@ -399,8 +449,8 @@ def _handle_relocation( addend = 0 case { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -410,14 +460,12 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_SIGNED" as kind}, + "Section": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_SIGNED" as kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, - "Type": { - "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind - }, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -427,12 +475,12 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": kind}, + "Section": {"Name": s}, + "Type": {"Name": kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -445,23 +493,27 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" + # ghccc currently crashes Clang when combined with musttail on aarch64. :( + target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): - args = ["-mcmodel=large"] - return _MachO(host, alignment=8, args=args, prefix="_") - if re.fullmatch(r"aarch64-pc-windows-msvc", host): + target = _MachO(host, alignment=8, prefix="_") + elif re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - return _COFF(host, alignment=8, args=args) - if re.fullmatch(r"aarch64-.*-linux-gnu", host): - args = ["-mcmodel=large"] - return _ELF(host, alignment=8, args=args) - if re.fullmatch(r"i686-pc-windows-msvc", host): + target = _COFF(host, alignment=8, args=args) + elif re.fullmatch(r"aarch64-.*-linux-gnu", host): + args = ["-fpic"] + target = _ELF(host, alignment=8, args=args) + elif re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] - return _COFF(host, args=args, prefix="_") - if re.fullmatch(r"x86_64-apple-darwin.*", host): - return _MachO(host, prefix="_") - if re.fullmatch(r"x86_64-pc-windows-msvc", host): + target = _COFF(host, args=args, ghccc=True, prefix="_") + elif re.fullmatch(r"x86_64-apple-darwin.*", host): + target = _MachO(host, ghccc=True, prefix="_") + elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - return _COFF(host, args=args) - if re.fullmatch(r"x86_64-.*-linux-gnu", host): - return _ELF(host) - raise ValueError(host) + target = _COFF(host, args=args, ghccc=True) + elif re.fullmatch(r"x86_64-.*-linux-gnu", host): + args = ["-fpic"] + target = _ELF(host, args=args, ghccc=True) + else: + raise ValueError(host) + return target diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index cbc1ed2fa6543a..9d11094f85c7ff 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -1,95 +1,65 @@ """Utilities for writing StencilGroups out to a C header file.""" + +import itertools import typing -import _schema import _stencils -def _dump_header() -> typing.Iterator[str]: - yield "typedef enum {" - for kind in typing.get_args(_schema.HoleKind): - yield f" HoleKind_{kind}," - yield "} HoleKind;" - yield "" - yield "typedef enum {" - for value in _stencils.HoleValue: - yield f" HoleValue_{value.name}," - yield "} HoleValue;" - yield "" - yield "typedef struct {" - yield " const size_t offset;" - yield " const HoleKind kind;" - yield " const HoleValue value;" - yield " const void *symbol;" - yield " const uint64_t addend;" - yield "} Hole;" - yield "" - yield "typedef struct {" - yield " const size_t body_size;" - yield " const unsigned char * const body;" - yield " const size_t holes_size;" - yield " const Hole * const holes;" - yield "} Stencil;" - yield "" +def _dump_footer(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: yield "typedef struct {" - yield " const Stencil code;" - yield " const Stencil data;" + yield " void (*emit)(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[]);" + yield " size_t code_size;" + yield " size_t data_size;" yield "} StencilGroup;" yield "" - - -def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: - yield "#define INIT_STENCIL(STENCIL) { \\" - yield " .body_size = Py_ARRAY_LENGTH(STENCIL##_body) - 1, \\" - yield " .body = STENCIL##_body, \\" - yield " .holes_size = Py_ARRAY_LENGTH(STENCIL##_holes) - 1, \\" - yield " .holes = STENCIL##_holes, \\" - yield "}" - yield "" - yield "#define INIT_STENCIL_GROUP(OP) { \\" - yield " .code = INIT_STENCIL(OP##_code), \\" - yield " .data = INIT_STENCIL(OP##_data), \\" - yield "}" + yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" yield "" - yield "static const StencilGroup stencil_groups[512] = {" - for opname in opnames: - yield f" [{opname}] = INIT_STENCIL_GROUP({opname})," + yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" + for opname, group in sorted(groups.items()): + if opname == "trampoline": + continue + yield f" [{opname}] = {group.as_c(opname)}," yield "};" - yield "" - yield "#define GET_PATCHES() { \\" - for value in _stencils.HoleValue: - yield f" [HoleValue_{value.name}] = (uintptr_t)0xBADBADBADBADBADB, \\" - yield "}" def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: - yield f"// {opname}" + yield "void" + yield f"emit_{opname}(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[])" + yield "{" for part, stencil in [("code", group.code), ("data", group.data)]: for line in stencil.disassembly: - yield f"// {line}" + yield f" // {line}" if stencil.body: - size = len(stencil.body) + 1 - yield f"static const unsigned char {opname}_{part}_body[{size}] = {{" + yield f" const unsigned char {part}_body[{len(stencil.body)}] = {{" for i in range(0, len(stencil.body), 8): row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8]) - yield f" {row}" - yield "};" - else: - yield f"static const unsigned char {opname}_{part}_body[1];" - if stencil.holes: - size = len(stencil.holes) + 1 - yield f"static const Hole {opname}_{part}_holes[{size}] = {{" - for hole in stencil.holes: - yield f" {hole.as_c()}," - yield "};" - else: - yield f"static const Hole {opname}_{part}_holes[1];" + yield f" {row}" + yield " };" + # Data is written first (so relaxations in the code work properly): + for part, stencil in [("data", group.data), ("code", group.code)]: + if stencil.body: + yield f" memcpy({part}, {part}_body, sizeof({part}_body));" + skip = False + stencil.holes.sort(key=lambda hole: hole.offset) + for hole, pair in itertools.zip_longest(stencil.holes, stencil.holes[1:]): + if skip: + skip = False + continue + if pair and (folded := hole.fold(pair)): + skip = True + hole = folded + yield f" {hole.as_c(part)}" + yield "}" yield "" def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: """Yield a JIT compiler line-by-line as a C header file.""" - yield from _dump_header() - for opname, group in groups.items(): + for opname, group in sorted(groups.items()): yield from _dump_stencil(opname, group) yield from _dump_footer(groups) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4d4ace14ebf26c..4a23c6f0afa74a 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -1,4 +1,5 @@ """Build an experimental just-in-time compiler for CPython.""" + import argparse import pathlib import shlex @@ -23,6 +24,5 @@ ) args = parser.parse_args() args.target.debug = args.debug - args.target.force = args.force args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment) + args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force) diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt new file mode 100644 index 00000000000000..84e8c0ee8afedb --- /dev/null +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -0,0 +1,79 @@ +test_multiprocessing_fork +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess +test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling +test.test_cmd_line.CmdLineTest.test_no_std_streams +test.test_cmd_line.CmdLineTest.test_no_stdin +test.test_concurrent_futures.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_ressources_gced_in_workers +test.test_external_inspection.TestGetStackTrace.test_remote_stack_trace +test.test_external_inspection.TestGetStackTrace.test_self_trace +test.test_faulthandler.FaultHandlerTests.test_enable_fd +test.test_faulthandler.FaultHandlerTests.test_enable_file +test.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists +test.test_os.TimerfdTests.test_timerfd_initval +test.test_os.TimerfdTests.test_timerfd_interval +test.test_os.TimerfdTests.test_timerfd_TFD_TIMER_ABSTIME +test.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.PosixPathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PosixPathTest.test_is_mount_root +test.test_posix.TestPosixSpawn.test_close_file +test.test_posix.TestPosixSpawnP.test_close_file +test.test_posixpath.PosixPathTest.test_ismount +test.test_signal.StressTest.test_stress_modifying_handlers +test.test_socket.BasicCANTest.testFilter +test.test_socket.BasicCANTest.testLoopback +test.test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm +test.test_socket.LinuxKernelCryptoAPI.test_aes_cbc +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCase.test_empty_env +test.test_subprocess.ProcessTestCase.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCase.test_one_environment_variable +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCaseNoPoll.test_empty_env +test.test_subprocess.ProcessTestCaseNoPoll.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCaseNoPoll.test_one_environment_variable +test.test_venv.BasicTest.test_zippath_from_non_installed_posix \ No newline at end of file diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 3e81fd15bb8093..a81e866e9da4b3 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -48,7 +48,7 @@ do { \ OPT_STAT_INC(traces_executed); \ __attribute__((musttail)) \ - return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); \ + return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ } while (0) #undef GOTO_TIER_ONE @@ -65,7 +65,7 @@ do { \ #define PATCH_VALUE(TYPE, NAME, ALIAS) \ PyAPI_DATA(void) ALIAS; \ - TYPE NAME = (TYPE)(uint64_t)&ALIAS; + TYPE NAME = (TYPE)(uintptr_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ do { \ @@ -106,7 +106,6 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * // The actual instruction definitions (only one will be used): if (uopcode == _JUMP_TO_TOP) { - CHECK_EVAL_BREAKER(); PATCH_JUMP(_JIT_TOP); } switch (uopcode) { diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c new file mode 100644 index 00000000000000..01b3d63a6790ba --- /dev/null +++ b/Tools/jit/trampoline.c @@ -0,0 +1,25 @@ +#include "Python.h" + +#include "pycore_ceval.h" +#include "pycore_frame.h" +#include "pycore_jit.h" + +// This is where the calling convention changes, on platforms that require it. +// The actual change is patched in while the JIT compiler is being built, in +// Tools/jit/_targets.py. On other platforms, this function compiles to nothing. +_Py_CODEUNIT * +_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +{ + // This is subtle. The actual trace will return to us once it exits, so we + // need to make sure that we stay alive until then. If our trace side-exits + // into another trace, and this trace is then invalidated, the code for + // *this function* will be freed and we'll crash upon return: + PyAPI_DATA(void) _JIT_EXECUTOR; + PyObject *executor = (PyObject *)(uintptr_t)&_JIT_EXECUTOR; + Py_INCREF(executor); + // Note that this is *not* a tail call: + PyAPI_DATA(void) _JIT_CONTINUE; + _Py_CODEUNIT *target = ((jit_func)&_JIT_CONTINUE)(frame, stack_pointer, tstate); + Py_SETREF(tstate->previous_executor, executor); + return target; +} diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index e0e179e3aede6d..7cddda9b06555d 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -464,11 +464,11 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; LPCWSTR locKey = L"#(loc.Include_launcherHelp)"; - LONGLONG detectedLauncher; + LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher)) && detectedLauncher) { + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherRemove)"; - } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &detectedLauncher)) && detectedLauncher) { + } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherUpgrade)"; } @@ -2671,7 +2671,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { /*Elevate when installing for all users*/ L"InstallAllUsers or " /*Elevate when installing the launcher for all users and it was not detected*/ - L"(Include_launcher and InstallLauncherAllUsers and not DetectedLauncher)" + L"(Include_launcher and InstallLauncherAllUsers and not BlockedLauncher)" L")", L"" }; diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs index a6922758f31f14..080598a0a486ef 100644 --- a/Tools/msi/bundle/packagegroups/launcher.wxs +++ b/Tools/msi/bundle/packagegroups/launcher.wxs @@ -11,7 +11,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> @@ -25,7 +25,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index 1f0586bb2f7d6d..da4c9331625dd9 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -18,7 +18,7 @@ def strongly_connected_components( exactly once; vertices not part of a SCC are returned as singleton sets. - From http://code.activestate.com/recipes/578507/. + From https://code.activestate.com/recipes/578507-strongly-connected-components-of-a-directed-graph/. """ identified: Set[str] = set() stack: List[str] = [] @@ -81,7 +81,7 @@ def topsort( {B, C} {A} - From http://code.activestate.com/recipes/577413/. + From https://code.activestate.com/recipes/577413-topological-sort/history/1/. """ # TODO: Use a faster algorithm? for k, v in data.items(): diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 61e75cf396ccb4..1767727373918f 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.9.0 +mypy==1.10.0 # needed for peg_generator: -types-psutil==5.9.5.20240316 -types-setuptools==69.2.0.20240317 +types-psutil==5.9.5.20240423 +types-setuptools==69.5.0.20240423 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 8cfa0df523dd16..9d5a18c881bf36 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.100.0 +hypothesis==6.100.2 diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 4f6648a7573184..74dbf4bb1cb688 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -15,7 +15,6 @@ race:_add_to_weak_set race:_in_weak_set race:_mi_heap_delayed_free_partial race:_PyEval_EvalFrameDefault -race:_PyFunction_SetVersion race:_PyImport_AcquireLock race:_PyImport_ReleaseLock race:_PyInterpreterState_SetNotRunningMain diff --git a/configure b/configure index 571ab8c882aa68..de426e6b686e68 100755 --- a/configure +++ b/configure @@ -1818,7 +1818,7 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) - --enable-experimental-jit + --enable-experimental-jit[=no|yes|yes-off|interpreter] build the experimental just-in-time compiler (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) @@ -1892,8 +1892,9 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-libmpdec build _decimal module using an installed libmpdec - library, see Doc/library/decimal.rst (default is no) + --with-system-libmpdec build _decimal module using an installed mpdecimal + library, see Doc/library/decimal.rst (default is + yes) --with-decimal-contextvar build _decimal module using a coroutine-local rather than a thread-local context (default is yes) @@ -7018,8 +7019,13 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( aarch64-apple-ios*/clang) : PY_SUPPORT_TIER=3 ;; #( + aarch64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( + x86_64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( *) : - PY_SUPPORT_TIER=0 + + PY_SUPPORT_TIER=0 ;; esac @@ -7103,6 +7109,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5 printf %s "checking for the Android arm ABI... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5 @@ -8226,11 +8235,25 @@ else $as_nop enable_experimental_jit=no fi -if test "x$enable_experimental_jit" = xno +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; +esac +if ${tier2_flags:+false} : then : else $as_nop - as_fn_append CFLAGS_NODIST " -D_Py_JIT" + as_fn_append CFLAGS_NODIST " $tier2_flags" +fi +if ${jit_flags:+false} : +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " $jit_flags" REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue @@ -8240,8 +8263,8 @@ fi fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 -printf "%s\n" "$enable_experimental_jit" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 +printf "%s\n" "$tier2_flags $jit_flags" >&6; } # Enable optimization flags @@ -14589,7 +14612,7 @@ if test ${with_system_libmpdec+y} then : withval=$with_system_libmpdec; else $as_nop - with_system_libmpdec="no" + with_system_libmpdec="yes" fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 @@ -14599,19 +14622,19 @@ if test "x$with_system_libmpdec" = xyes then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec" >&5 -printf %s "checking for libmpdec... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec >= 2.5.0" >&5 +printf %s "checking for libmpdec >= 2.5.0... " >&6; } if test -n "$LIBMPDEC_CFLAGS"; then pkg_cv_LIBMPDEC_CFLAGS="$LIBMPDEC_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libmpdec") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec" 2>/dev/null` + pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec >= 2.5.0" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -14623,12 +14646,12 @@ if test -n "$LIBMPDEC_LIBS"; then pkg_cv_LIBMPDEC_LIBS="$LIBMPDEC_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libmpdec") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec" 2>/dev/null` + pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec >= 2.5.0" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -14649,9 +14672,9 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec" 2>&1` + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` else - LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec" 2>&1` + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` fi # Put the nasty error message in config.log where it belongs echo "$LIBMPDEC_PKG_ERRORS" >&5 @@ -14678,6 +14701,61 @@ else $as_nop LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" fi +if test "x$with_system_libmpdec" = xyes +then : + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + +int +main (void) +{ +const char *x = mpd_version(); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + have_mpdec=yes +else $as_nop + have_mpdec=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + +else $as_nop + have_mpdec=yes + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} +fi + +if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; unable to build _decimal" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; unable to build _decimal" >&2;} +fi + # Disable forced inlining in debug builds, see GH-94847 if test "x$with_pydebug" = xyes then : @@ -26759,7 +26837,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26802,7 +26883,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26845,7 +26929,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26888,7 +26975,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26931,7 +27021,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26974,7 +27067,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27017,7 +27113,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27060,7 +27159,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27103,7 +27205,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27146,7 +27251,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27189,7 +27297,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -30371,7 +30482,7 @@ then : if true then : - if true + if test "$have_mpdec" = "yes" then : py_cv_module__decimal=yes else $as_nop diff --git a/configure.ac b/configure.ac index a2d6b1357efbc9..0f1b977591c260 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define([PYTHON_VERSION], [3.13]) +m4_define([PYTHON_VERSION], [3.14]) AC_PREREQ([2.71]) @@ -1148,6 +1148,9 @@ AS_CASE([$host/$ac_cv_cc_name], [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [PY_SUPPORT_TIER=0] ) @@ -1192,6 +1195,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + AC_MSG_CHECKING([for the Android arm ABI]) AC_MSG_RESULT([$_arm_arch]) if test "$_arm_arch" = 7; then @@ -1765,14 +1771,27 @@ fi # Check for --enable-experimental-jit: AC_MSG_CHECKING([for --enable-experimental-jit]) AC_ARG_ENABLE([experimental-jit], - [AS_HELP_STRING([--enable-experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@], [build the experimental just-in-time compiler (default is no)])], [], [enable_experimental_jit=no]) -AS_VAR_IF([enable_experimental_jit], - [no], +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) AC_MSG_ERROR( + [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; +esac +AS_VAR_IF([tier2_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])]) +AS_VAR_IF([jit_flags], + [], [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) @@ -1782,7 +1801,7 @@ AS_VAR_IF([enable_experimental_jit], [])]) AC_SUBST([REGEN_JIT_COMMAND]) AC_SUBST([JIT_STENCILS_H]) -AC_MSG_RESULT([$enable_experimental_jit]) +AC_MSG_RESULT([$tier2_flags $jit_flags]) # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) @@ -3955,16 +3974,16 @@ AC_ARG_WITH( [system_libmpdec], [AS_HELP_STRING( [--with-system-libmpdec], - [build _decimal module using an installed libmpdec library, see Doc/library/decimal.rst (default is no)] + [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] )], [], - [with_system_libmpdec="no"]) + [with_system_libmpdec="yes"]) AC_MSG_RESULT([$with_system_libmpdec]) AS_VAR_IF( [with_system_libmpdec], [yes], [PKG_CHECK_MODULES( - [LIBMPDEC], [libmpdec], [], + [LIBMPDEC], [libmpdec >= 2.5.0], [], [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} LIBMPDEC_INTERNAL=])], @@ -3972,6 +3991,29 @@ AS_VAR_IF( LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)"]) +AS_VAR_IF([with_system_libmpdec], [yes], + [WITH_SAVE_ENV([ + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + ], [const char *x = mpd_version();])], + [have_mpdec=yes], + [have_mpdec=no]) + ])], + [AS_VAR_SET([have_mpdec], [yes]) + AC_MSG_WARN([m4_normalize([ + the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; + consider using a system installed mpdecimal library.])])]) + +AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], + [AC_MSG_WARN([no system libmpdecimal found; unable to build _decimal])]) + # Disable forced inlining in debug builds, see GH-94847 AS_VAR_IF( [with_pydebug], [yes], @@ -6694,7 +6736,10 @@ AC_DEFUN([PY_CHECK_CURSES_FUNC], [py_var], [AC_COMPILE_IFELSE( [AC_LANG_PROGRAM( - [@%:@include ], [ + [ + #define NCURSES_OPAQUE 0 + #include + ], [ #ifndef $1 void *x=$1 #endif @@ -7646,7 +7691,9 @@ PY_STDLIB_MOD([_curses_panel], [], [test "$have_panel" != "no"], [$PANEL_CFLAGS $CURSES_CFLAGS], [$PANEL_LIBS $CURSES_LIBS] ) -PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) +PY_STDLIB_MOD([_decimal], + [], [test "$have_mpdec" = "yes"], + [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS])