Add integration tests for install_swiftpm_dependencies

.buildkite/pipeline.yml

@@ -1,6 +1,45 @@
+# For convenience with the many install_swiftpm_dependencies steps, we declare agents and env for macOS in the root, then specialize the fewer remaining steps
+agents:
+  queue: mac
+env:
+  IMAGE_ID: xcode-15.4
+
 steps:
   - label: "🕵️ Lint"
     command: make lint
+    agents:
+      queue: default
 
   - label: "🔬 Test"
     command: make test
+    agents:
+      queue: default
+
+  - group: ":swift: install_swiftpm_dependencies Tests"
+    steps:
+      - label: ":swift: Standalone Swift Package - Autodetect"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_package_automatic.sh
+
+      - label: ":swift: Standalone Swift Package - Explicit"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_package_explicit.sh
+
+      - label: ":xcode: Xcode Project - Autodetect"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_project_automatic.sh
+
+      - label: ":xcode: Xcode Project - Explicit"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_project_explicit.sh
+
+      - label: ":xcode: Xcode Project - No Package.resolved"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_project_no_package.sh
+
+      - label: ":xcode: Xcode Workspace - Autodetect"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_workspace_automatic.sh
+
+      - label: ":xcode: Xcode Workspace - Explicit"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_workspace_explicit.sh
+
+      - label: ":xcode: Xcode Workspace and Project - Autodetect"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_workspace_and_project_automatic.sh
+
+      - label: ":xcode: Xcode Workspace and Project - Explicit"
+        command: tests/install_swiftpm_dependencies/test_scripts/test_workspace_and_project_explicit.sh

bin/install_swiftpm_dependencies

@@ -45,13 +45,21 @@ SPM_CACHE_LOCATION="${HOME}/Library/Caches/org.swift.swiftpm"
 # Try to guess what to do if no option provided explicitly
 if [[ -z "${XCWORKSPACE_PATH}" && -z "${XCODEPROJ_PATH}" && "${USE_SPM}" != "true" ]]; then
   echo "No \`--workspace\`, \`--project\` or \`--use-spm\` flag provided. Trying to guess the correct one to use..."
+
+  shopt -s nullglob
   FOUND_ROOT_WORKSPACE_PATHS=(*.xcworkspace)
-  if  [[ ${#FOUND_ROOT_WORKSPACE_PATHS[@]} -eq 1 && -d "${FOUND_ROOT_WORKSPACE_PATHS[0]}" && ! -f "Package.swift" ]]; then
+  FOUND_ROOT_PROJECT_PATHS=(*.xcodeproj)
+
+  if [[ ${#FOUND_ROOT_WORKSPACE_PATHS[@]} -eq 1 && -d "${FOUND_ROOT_WORKSPACE_PATHS[0]}" && ! -f "Package.swift" ]]; then
     XCWORKSPACE_PATH="${FOUND_ROOT_WORKSPACE_PATHS[0]}"
     echo " --> Found a single \`.xcworkspace\` file, and no \`Package.swift\` in the root of the repo."
     echo "     Defaulting to \`--workspace \"${XCWORKSPACE_PATH}\"\`."
-  elif [[ ${#FOUND_ROOT_WORKSPACE_PATHS[@]} -eq 0 && -f "Package.swift" ]]; then
-    echo " --> Found a \`Package.swift\`, and no \`.xcworkspace\` in the root of the repo."
+  elif [[ ${#FOUND_ROOT_WORKSPACE_PATHS[@]} -eq 0 && ${#FOUND_ROOT_PROJECT_PATHS[@] } -eq 1 && ! -f "Package.swift" ]]; then
+    XCODEPROJ_PATH="${FOUND_ROOT_PROJECT_PATHS[0]}"
+    echo " --> Found an \`.xcodeproj\`, and no \`Package.swift\` nor \`.xcworkspace\`in the root of the repo."
+    echo "     Defaulting to \`--project \"${XCODEPROJ_PATH}\`"
+  elif [[ ${#FOUND_ROOT_WORKSPACE_PATHS[@]} -eq 0 && ${#FOUND_ROOT_PROJECT_PATHS[@] } -eq 0 && -f "Package.swift" ]]; then
+    echo " --> Found a \`Package.swift\`, and no \`.xcworkspace\` or \`.xcodeproj\` in the root of the repo."
     echo "     Defaulting to \`--use-spm\`"
     USE_SPM=true
   else
@@ -111,15 +119,15 @@ elif [[ "${USE_SPM}" == "true" ]]; then
   swift package resolve
 fi
 
-
-# `checkouts` can be removed because the system can quickly generate them 
+# `checkouts` can be removed because the system can quickly generate them
 # instead of needing to download them in the cache each time.
-# 
-# `artifacts` should be removed because it causes issues when downloading 
-# certain packages to have the artifacts already present after extracting 
+#
+# `artifacts` should be removed because it causes issues when downloading
+# certain packages to have the artifacts already present after extracting
 # cache
 echo "~~~ Cleaning up cache files before saving cache"
 rm -rf "${SPM_CACHE_LOCATION}/checkouts" "${SPM_CACHE_LOCATION}/artifacts"
+echo "Done. Removed checkouts and artifacts subfolders from $SPM_CACHE_LOCATION"
 
 # If this is the first time we've seen this particular cache key, save it for the future
 echo "~~~ Saving SPM Cache"

tests/install_swiftpm_dependencies/Makefile

@@ -0,0 +1,4 @@
+regenerate_fixtures:
+	xcodegen generate --spec ./project/project.yml
+	xcodebuild -resolvePackageDependencies -project ./project/Demo.xcodeproj
+	cp ./project/Demo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved ./package_resolved_fixtures/valid.resolved

tests/install_swiftpm_dependencies/README.md

@@ -0,0 +1,17 @@
+# Integration tests for `install_swiftpm_dependencies`
+
+This folder contains a variety of Xcode and Swift Package projects to test the behavior of the `install_swiftpm_dependencies` in various configurations.
+
+Note that currently with "testing" we mean: making sure the CI build doesn't fail. There isn't yet an automated setup to verify the correct command behavior.
+
+Also note that the tests are intentionally at the integration / end-to-end level. We could structure `install_swiftpm_dependencies` so that it spits out the `xcodebuild` or `swift` it plans to run and test that, but we would lose important feedback on how `xcodebuild` and `swift` themselves behave.
+Feedback on Apple's tooling behavior is crucial to get, given their known quirks.
+
+## Development
+
+The integration tests use [XcodeGen](https://github.com/yonaskolb/XcodeGen) to code-generate the project files and keep the repository footprint small.
+
+However, by code-generating the xcodeproj file we lose tracking of the `Package.resolved` Xcode generates at build time, which is particularly problematic in CI.
+To compensate for this, we have dedicated automation that tracks a reference `Package.resolved` file for CI to use.
+
+If resolved file format were to change in the future, you can update the reference file via `make regenerate_resolved_fixtures`.

tests/install_swiftpm_dependencies/package/.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+/.build
+/Packages
+xcuserdata/
+DerivedData/
+.swiftpm/configuration/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc

tests/install_swiftpm_dependencies/package/Package.resolved

@@ -0,0 +1,15 @@
+{
+  "originHash" : "c908e0d89a41be29a11cf4ce76b34002b6c4863ff5d7eafd623b36c267284b12",
+  "pins" : [
+    {
+      "identity" : "screenobject",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/Automattic/ScreenObject",
+      "state" : {
+        "revision" : "328db56c62aab91440ec5e07cc9f7eef6e26a26e",
+        "version" : "0.2.3"
+      }
+    }
+  ],
+  "version" : 3
+}

tests/install_swiftpm_dependencies/package/Package.swift

@@ -0,0 +1,22 @@
+// swift-tools-version: 5.10
+
+import PackageDescription
+
+let package = Package(
+    name: "Demo",
+    products: [
+        .library(
+            name: "Demo",
+            targets: ["Demo"]),
+    ],
+    dependencies: [
+        .package(url: "https://github.com/Automattic/ScreenObject", from: "0.2.3")
+    ],
+    targets: [
+        .target(
+            name: "Demo"),
+        .testTarget(
+            name: "DemoTests",
+            dependencies: ["Demo"]),
+    ]
+)

tests/install_swiftpm_dependencies/package/Sources/Demo/Demo.swift

@@ -0,0 +1,5 @@
+struct Demo {
+    static func sayHello() -> String {
+        "Hello, world!"
+    }
+}

tests/install_swiftpm_dependencies/package/Tests/DemoTests/DemoTests.swift

@@ -0,0 +1,8 @@
+import XCTest
+@testable import Demo
+
+final class DemoTests: XCTestCase {
+    func testHello() throws {
+        XCTAssertEqual(Demo.sayHello(), "Hello, world!")
+    }
+}

tests/install_swiftpm_dependencies/package_resolved_fixtures/valid.resolved

@@ -0,0 +1,15 @@
+{
+  "originHash" : "5ca84ad21a87cfff30163bd7dd55abded567e0600b5a0adbc80cfe0c687ecc69",
+  "pins" : [
+    {
+      "identity" : "screenobject",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/Automattic/ScreenObject",
+      "state" : {
+        "revision" : "328db56c62aab91440ec5e07cc9f7eef6e26a26e",
+        "version" : "0.2.3"
+      }
+    }
+  ],
+  "version" : 3
+}

tests/install_swiftpm_dependencies/project/.gitignore

@@ -0,0 +1,5 @@
+# For convenience with the code generation, ignore everything in this folder other that what's required to code-gen and test behavior
+*
+!project.yml
+!Makefile
+!Demo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

tests/install_swiftpm_dependencies/project/Makefile

@@ -0,0 +1,4 @@
+.DEFAULT_GOAL := generate
+
+generate:
+	xcodegen generate

tests/install_swiftpm_dependencies/project/project.yml

@@ -0,0 +1,28 @@
+name: Demo
+options:
+  bundleIdPrefix: com.automattic.tests
+packages:
+  ScreenObject:
+    url: https://github.com/Automattic/ScreenObject
+    from: 0.2.3
+targets:
+  Demo:
+    type: framework
+    platform: iOS
+    sources: [../package/Sources]
+    settings:
+      GENERATE_INFOPLIST_FILE: YES
+    scheme:
+      testTargets: [DemoTests]
+  DemoTests:
+    target: Demo
+    type: bundle.unit-test
+    platform: iOS
+    sources: [../package/Tests/]
+    settings:
+      GENERATE_INFOPLIST_FILE: YES
+      # No need for code signing in this demo, plus, it's the test target
+      CODE_SIGNING_ALLOWED: NO
+    dependencies:
+      - target: Demo
+      - package: ScreenObject

tests/install_swiftpm_dependencies/test_scripts/run_tests_with_xcodebuild.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+echo "--- :xcode: Run tests to verify packages have been fetched and are available"
+xcodebuild test \
+  -scheme Demo \
+  -configuration Debug \
+  -destination 'platform=iOS Simulator' \
+  | xcbeautify

tests/install_swiftpm_dependencies/test_scripts/set_up_environment.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
+  echo "$0 must be sourced"
+  exit 1
+fi
+
+set -eu
+
+echo "--- :computer: Prepare environment"
+TESTS_LOCATION="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+NEW_PATH=$PATH:"$TESTS_LOCATION/../../../bin"
+
+export PATH=$NEW_PATH
+export TESTS_LOCATION

tests/install_swiftpm_dependencies/test_scripts/test_package_automatic.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/set_up_environment.sh"
+
+echo "--- :computer: Jump to test folder"
+pushd "$TESTS_LOCATION/../package"
+
+echo "--- :wrench: Run install_swiftpm_dependencies"
+install_swiftpm_dependencies
+
+echo "--- :xcode: Run tests to verify packages have been fetched and are available"
+swift test

tests/install_swiftpm_dependencies/test_scripts/test_package_explicit.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/set_up_environment.sh"
+
+echo "--- :computer: Jump to test folder"
+pushd "$TESTS_LOCATION/../package"
+
+echo "--- :wrench: Run install_swiftpm_dependencies"
+install_swiftpm_dependencies --use-spm
+
+echo "--- :xcode: Run tests to verify packages have been fetched and are available"
+swift test

tests/install_swiftpm_dependencies/test_scripts/test_project_automatic.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/set_up_environment.sh"
+
+echo "--- :computer: Jump to test folder"
+pushd "$TESTS_LOCATION/../project"
+
+echo "--- :computer: Generate project"
+brew install xcodegen
+make
+
+echo "--- :computer: Copy Package.resolved fixture"
+PROJECT="Demo.xcodeproj"
+XCODE_SPM_PATH="$PROJECT/project.xcworkspace/xcshareddata/swiftpm"
+mkdir -p "$XCODE_SPM_PATH"
+cp "$TESTS_LOCATION/../package_resolved_fixtures/valid.resolved" "$XCODE_SPM_PATH/Package.resolved"
+
+echo "--- :wrench: Run install_swiftpm_dependencies"
+install_swiftpm_dependencies
+
+"$TESTS_LOCATION/run_tests_with_xcodebuild.sh"

tests/install_swiftpm_dependencies/test_scripts/test_project_explicit.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/set_up_environment.sh"
+
+echo "--- :computer: Jump to test folder"
+pushd "$TESTS_LOCATION/../project"
+
+echo "--- :computer: Generate project"
+brew install xcodegen
+make
+
+echo "--- :computer: Copy Package.resolved fixture"
+PROJECT="Demo.xcodeproj"
+XCODE_SPM_PATH="$PROJECT/project.xcworkspace/xcshareddata/swiftpm"
+mkdir -p "$XCODE_SPM_PATH"
+cp "$TESTS_LOCATION/../package_resolved_fixtures/valid.resolved" "$XCODE_SPM_PATH/Package.resolved"
+
+echo "--- :wrench: Run install_swiftpm_dependencies"
+install_swiftpm_dependencies --project $PROJECT
+
+"$TESTS_LOCATION/run_tests_with_xcodebuild.sh"

tests/install_swiftpm_dependencies/test_scripts/test_project_no_package.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/set_up_environment.sh"
+
+echo "--- :computer: Jump to test folder"
+pushd "$TESTS_LOCATION/../project"
+
+echo "--- :computer: Generate project"
+brew install xcodegen
+make
+
+# Notice we do not set up the fixture Package.resolved.
+# As such, we expect the call to the plugin to fail.
+
+echo "--- :wrench: Run install_swiftpm_dependencies"
+PROJECT=Demo.xcodeproj
+LOGS_PATH=logs
+set +e
+install_swiftpm_dependencies --project $PROJECT 2>&1 | tee "$LOGS_PATH"
+CMD_EXIT_STATUS=$?
+set -e
+
+EXPECTED="Unable to find \`Package.resolved\` file ($PROJECT/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)"
+
+if [[ $CMD_EXIT_STATUS -eq 0 ]]; then
+  echo "+++ :x: install_swiftpm_dependencies unexpectedly succeeded without a Package.resolved in the project folder!"
+  echo "Expected: $EXPECTED"
+  echo "Got: $(cat $LOGS_PATH)"
+  exit 1
+else
+  if grep -qF "$EXPECTED" "$LOGS_PATH"; then
+    echo "^^^ +++"
+    echo "+++ :white_check_mark: install_swiftpm_dependencies failed as expected because there is no Package.resolved in the project folder."
+  else
+    echo "+++ :x: install_swiftpm_dependencies failed, but the message it printed is not what we expected."
+    echo "Expected: $EXPECTED"
+    echo "Got: $(cat $LOGS_PATH)"
+    exit 1
+  fi
+fi

tests/install_swiftpm_dependencies/test_scripts/test_workspace_and_project_automatic.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/set_up_environment.sh"
+
+echo "--- :computer: Jump to test folder"
+pushd "$TESTS_LOCATION/../workspace_and_project"
+
+echo "--- :computer: Generate project"
+brew install xcodegen
+make
+
+echo "--- :wrench: Run install_swiftpm_dependencies"
+install_swiftpm_dependencies
+
+"$TESTS_LOCATION/run_tests_with_xcodebuild.sh"