From e351e9a9ee2e1ae13a21b1be1edf5ea0e761744f Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 13 Dec 2024 05:49:02 +0800 Subject: [PATCH] gh-127845: Minor improvements to iOS test runner script (GH-127846) Uses symlinks to install iOS framework into testbed clone, adds a verbose mode to the iOS runner to hide most Xcode output, adds another mechanism to disable terminal colors, and ensures that stdout is flushed after every write. (cherry picked from commit ba2d2fda93a03a91ac6cdff319fd23ef51848d51) Co-authored-by: Russell Keith-Magee --- Makefile.pre.in | 2 +- iOS/testbed/__main__.py | 66 ++++++++++++++----- iOS/testbed/iOSTestbedTests/iOSTestbedTests.m | 5 +- 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index ac695e635cf2dd..46a37ded970573 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2084,7 +2084,7 @@ testios: $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" # Run the testbed project - $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run -- test -uall --single-process --rerun -W + $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py index 22570ee0f3ed04..068272835a5b95 100644 --- a/iOS/testbed/__main__.py +++ b/iOS/testbed/__main__.py @@ -141,10 +141,12 @@ async def log_stream_task(initial_devices): else: suppress_dupes = False sys.stdout.write(line) + sys.stdout.flush() -async def xcode_test(location, simulator): +async def xcode_test(location, simulator, verbose): # Run the test suite on the named simulator + print("Starting xcodebuild...") args = [ "xcodebuild", "test", @@ -159,6 +161,9 @@ async def xcode_test(location, simulator): "-derivedDataPath", str(location / "DerivedData"), ] + if not verbose: + args += ["-quiet"] + async with async_process( *args, stdout=subprocess.PIPE, @@ -166,6 +171,7 @@ async def xcode_test(location, simulator): ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): sys.stdout.write(line) + sys.stdout.flush() status = await asyncio.wait_for(process.wait(), timeout=1) exit(status) @@ -182,7 +188,9 @@ def clone_testbed( sys.exit(10) if framework is None: - if not (source / "Python.xcframework/ios-arm64_x86_64-simulator/bin").is_dir(): + if not ( + source / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + ).is_dir(): print( f"The testbed being cloned ({source}) does not contain " f"a simulator framework. Re-run with --framework" @@ -202,33 +210,48 @@ def clone_testbed( ) sys.exit(13) - print("Cloning testbed project...") - shutil.copytree(source, target) + print("Cloning testbed project:") + print(f" Cloning {source}...", end="", flush=True) + shutil.copytree(source, target, symlinks=True) + print(" done") if framework is not None: if framework.suffix == ".xcframework": - print("Installing XCFramework...") - xc_framework_path = target / "Python.xcframework" - shutil.rmtree(xc_framework_path) - shutil.copytree(framework, xc_framework_path) + print(" Installing XCFramework...", end="", flush=True) + xc_framework_path = (target / "Python.xcframework").resolve() + if xc_framework_path.is_dir(): + shutil.rmtree(xc_framework_path) + else: + xc_framework_path.unlink() + xc_framework_path.symlink_to( + framework.relative_to(xc_framework_path.parent, walk_up=True) + ) + print(" done") else: - print("Installing simulator Framework...") + print(" Installing simulator framework...", end="", flush=True) sim_framework_path = ( target / "Python.xcframework" / "ios-arm64_x86_64-simulator" + ).resolve() + if sim_framework_path.is_dir(): + shutil.rmtree(sim_framework_path) + else: + sim_framework_path.unlink() + sim_framework_path.symlink_to( + framework.relative_to(sim_framework_path.parent, walk_up=True) ) - shutil.rmtree(sim_framework_path) - shutil.copytree(framework, sim_framework_path) + print(" done") else: - print("Using pre-existing iOS framework.") + print(" Using pre-existing iOS framework.") for app_src in apps: - print(f"Installing app {app_src.name!r}...") + print(f" Installing app {app_src.name!r}...", end="", flush=True) app_target = target / f"iOSTestbed/app/{app_src.name}" if app_target.is_dir(): shutil.rmtree(app_target) shutil.copytree(app_src, app_target) + print(" done") - print(f"Testbed project created in {target}") + print(f"Successfully cloned testbed: {target.resolve()}") def update_plist(testbed_path, args): @@ -243,10 +266,11 @@ def update_plist(testbed_path, args): plistlib.dump(info, f) -async def run_testbed(simulator: str, args: list[str]): +async def run_testbed(simulator: str, args: list[str], verbose: bool=False): location = Path(__file__).parent - print("Updating plist...") + print("Updating plist...", end="", flush=True) update_plist(location, args) + print(" done.") # Get the list of devices that are booted at the start of the test run. # The simulator started by the test suite will be detected as the new @@ -256,7 +280,7 @@ async def run_testbed(simulator: str, args: list[str]): try: async with asyncio.TaskGroup() as tg: tg.create_task(log_stream_task(initial_devices)) - tg.create_task(xcode_test(location, simulator)) + tg.create_task(xcode_test(location, simulator=simulator, verbose=verbose)) except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* subprocess.CalledProcessError as e: @@ -315,6 +339,11 @@ def main(): default="iPhone SE (3rd Generation)", help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')", ) + run.add_argument( + "-v", "--verbose", + action="store_true", + help="Enable verbose output", + ) try: pos = sys.argv.index("--") @@ -330,7 +359,7 @@ def main(): clone_testbed( source=Path(__file__).parent, target=Path(context.location), - framework=Path(context.framework) if context.framework else None, + framework=Path(context.framework).resolve() if context.framework else None, apps=[Path(app) for app in context.apps], ) elif context.subcommand == "run": @@ -348,6 +377,7 @@ def main(): asyncio.run( run_testbed( simulator=context.simulator, + verbose=context.verbose, args=test_args, ) ) diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index ac78456a61e65e..6db38253396c8d 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -24,8 +24,11 @@ - (void)testPython { NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; - // Disable all color, as the Xcode log can't display color + // Set some other common environment indicators to disable color, as the + // Xcode log can't display color. Stdout will report that it is *not* a + // TTY. setenv("NO_COLOR", "1", true); + setenv("PY_COLORS", "0", true); // Arguments to pass into the test suite runner. // argv[0] must identify the process; any subsequent arg