LLVM Build Caching (#225)

.dockerignore

@@ -0,0 +1,4 @@
+.dockerignore
+.git
+.gitignore
+/target

.github/actions/install-llvm/action.yml

@@ -0,0 +1,61 @@
+name: install-llvm
+description: "Builds and installs LLVM from source using install-llvm-from-source tasks"
+
+inputs:
+  version:
+    description: "The version of LLVM to install."
+    required: true
+  os:
+    description: "The OS being built upon."
+    required: true
+  directory:
+    description: "The directory to install LLVM binaries to."
+    required: true
+  target:
+    description: "The build script target."
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: LLVM install cache
+      id: cache-llvm
+      uses: actions/cache@v3
+      with:
+        path: ${{ inputs.directory }}
+        key: llvm-${{ inputs.version }}-${{ inputs.arch }}-${{ inputs.os }}-${{ inputs.target }}
+        restore-keys: llvm-${{ inputs.version }}-${{ inputs.arch }}-${{ inputs.os }}-${{ inputs.target }}
+
+    - name: Linux - Install build dependencies, ninja
+      run: sudo apt-get install -y ninja-build
+      shell: pwsh
+      if: ${{ (inputs.os == 'ubuntu-20.04') && (steps.cache-llvm.outputs.cache-hit != 'true') }}
+    - name: Windows - Install build dependencies, ninja
+      run: | 
+        choco install --accept-license -y ninja
+        choco uninstall -y llvm
+      shell: pwsh
+      if: ${{ (inputs.os == 'windows-2019') && (steps.cache-llvm.outputs.cache-hit != 'true') }}
+    - name: MacOS - Install build dependencies, ninja
+      run: brew install ninja
+      shell: pwsh
+      if: ${{ (inputs.os == 'macos-11') && (steps.cache-llvm.outputs.cache-hit != 'true') }}
+
+    - name: Configure long paths
+      run: git config --global core.longpaths true
+      shell: pwsh
+      if: ${{ steps.cache-llvm.outputs.cache-hit != 'true' }}
+
+    - name: Configure LLVM Environment
+      run: |
+        Write-Output "QIRLIB_CACHE_DIR=${{ inputs.directory }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+        Write-Output "LLVM_SYS_${{ inputs.version }}0_PREFIX=${{ inputs.directory }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+        Write-Output "PYQIR_LLVM_FEATURE_VERSION=llvm${{ inputs.version }}-0" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+      shell: pwsh
+
+    - name: Install LLVM
+      run: |
+        $target_prefix = if("${{ inputs.target }}" -eq "manylinux") {"manylinux-"} else {""}
+        ./build.ps1 -t "$($target_prefix)install-llvm-from-source"
+      shell: pwsh
+      if: ${{ steps.cache-llvm.outputs.cache-hit != 'true' }}

.github/actions/rust-toolchain/action.yml

@@ -0,0 +1,16 @@
+name: rust-toolchain
+inputs:
+  toolchain:
+    required: false
+  components:
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - shell: pwsh
+      run: |
+        rustup override set ${{ inputs.toolchain || 'stable' }}
+        ${{ inputs.components && format('rustup component add {0}', inputs.components) }}
+        cargo --version
+        rustc --version

.github/workflows/ci.yml

@@ -13,8 +13,6 @@ on:
 env:
   CARGO_TERM_COLOR: always
   QIRLIB_DOWNLOAD_LLVM: false
-  CCACHE_DIR: ~/.ccache
-  SCCACHE_DIR: C:\sccache
 
 jobs:
   build:
@@ -55,9 +53,19 @@ jobs:
               env: {},
             }
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - name: Setup rust toolchain
+        uses: ./.github/actions/rust-toolchain
+        with:
+          toolchain: 1.64.0
+          components: rustfmt clippy
+      - name: Install LLVM
+        uses: ./.github/actions/install-llvm
         with:
-          submodules: 'recursive'
+          version: "14"
+          os: ${{ matrix.config.os }}
+          directory: ${{ github.workspace }}/target/llvm
+          target: ${{ matrix.config.target }}
       - name: Linux - Install build dependencies, ccache, ninja
         run: sudo apt-get install -y ccache ninja-build
         if: ${{ matrix.config.os == 'ubuntu-20.04' }}
@@ -67,31 +75,6 @@ jobs:
       - name: MacOS - Install build dependencies, ccache, ninja
         run: brew install ccache ninja
         if: ${{ matrix.config.os == 'macos-11' }}
-      - name: Windows - Install LLVM 11.1.0
-        run: choco install llvm --version=11.1.0 --allow-downgrade
-        if: ${{ matrix.config.os == 'windows-2019' }}
-      - name: Get Timestamp
-        id: timestamp
-        run: Write-Host "::set-output name=timestamp::$([DateTime]::UtcNow.ToString('o'))"
-        shell: pwsh
-      - name: CCACHE cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ env.CCACHE_DIR }}
-          key: ${{ matrix.config.os }}-${{ matrix.config.target }}-ccache-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-              ${{ matrix.config.os }}-${{ matrix.config.target }}-ccache-
-        if: ${{ matrix.config.os != 'windows-2019' }}
-      - name: SCCACHE cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ env.SCCACHE_DIR }}
-          key: ${{ matrix.config.os }}-sccache-${{ steps.timestamp.outputs.timestamp }}
-          restore-keys: |
-              ${{ matrix.config.os }}-sccache-
-        if: ${{ matrix.config.os == 'windows-2019' }}
       - name: "Build ${{ matrix.config.target }}"
         run: ./build.ps1 -t ${{ matrix.config.target }}
         shell: pwsh

eng/Dockerfile.manylinux

@@ -6,7 +6,7 @@ FROM quay.io/pypa/manylinux2014_x86_64
 ARG USERNAME=runner
 ARG USER_UID=1000
 ARG USER_GID=${USER_UID}
-ARG RUST_VERSION=1.64.0
+ARG RUST_TOOLCHAIN=1.64.0
 
 RUN groupadd --gid ${USER_GID} ${USERNAME}
 RUN useradd --uid ${USER_UID} --gid ${USER_GID} -m ${USERNAME}
@@ -16,13 +16,13 @@ RUN chmod 0440 /etc/sudoers.d/${USERNAME}
 
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:${PATH} \
-    RUST_VERSION=${RUST_VERSION}
+    PATH=/usr/local/cargo/bin:${PATH}
 
-RUN curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --profile minimal --default-toolchain ${RUST_VERSION} -y
+RUN curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --profile minimal --default-toolchain none -y
+RUN chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME};
 
-RUN chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME}; \
-    rustup --version; \
+RUN rustup override set ${RUST_TOOLCHAIN}
+RUN rustup --version; \
     cargo --version; \
     rustc --version;
 
@@ -56,8 +56,6 @@ USER $USERNAME
 
 ENV PATH="/usr/local/miniconda3/bin:${PATH}"
 
-ENV PATH="/usr/lib/ccache:${PATH}"
-
 RUN conda init && \
     conda install -y -c conda-forge clang-11 libstdcxx-devel_linux-64 libgcc-devel_linux-64 && \
     cp /usr/local/miniconda3/bin/clang-11 /usr/local/miniconda3/bin/clang++-11

eng/psakefile.ps1

@@ -14,7 +14,6 @@ Properties {
     $VscodeSettingsJson = Join-Path $Root .vscode settings.json
     $DocsRoot = Join-Path $Root docs
     $DocsBuild = Join-Path $DocsRoot _build
-    $RustVersion = "1.64.0"
     $ManylinuxTag = "manylinux2014_x86_64_maturin"
     $ManylinuxRoot = "/io"
     $Python = Resolve-Python
@@ -26,19 +25,16 @@ task checks -depends cargo-fmt, cargo-clippy, black, mypy
 task manylinux -depends build-manylinux-container-image, run-manylinux-container-image, run-examples-in-containers 
 
 task run-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } {
-    # For any of the volumes mapped, if the dir doesn't exist,
-    # docker will create it and it will be owned by root and
-    # the caching/install breaks with permission errors.
-    # New-Item is idempotent so we don't need to check for existence
-    $cacheMount, $cacheEnv = Get-CCacheParams
+    $llvmDir = Resolve-InstallationDirectory
+    $llvmMount = @("-v", "$($llvmDir):/tmp/llvm")
     Write-BuildLog "Running container image: $ManylinuxTag"
     $ioVolume = "${Root}:$ManylinuxRoot"
     $userName = Get-LinuxContainerUserName
 
     Invoke-LoggedCommand {
         docker run --rm `
             --user $userName `
-            --volume $ioVolume @cacheMount @cacheEnv `
+            --volume $ioVolume @llvmMount `
             --env QIRLIB_CACHE_DIR=/tmp/llvm `
             --workdir $ManylinuxRoot `
             $ManylinuxTag `
@@ -183,20 +179,34 @@ task install-llvm-from-source -depends configure-sccache -postaction { Write-Cac
     Assert (Test-LlvmConfig $installationDirectory) "install-llvm-from-source failed to install a usable LLVM installation"
 }
 
+task manylinux-install-llvm-from-source -depends build-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } {
+    $llvmDir = Resolve-InstallationDirectory
+    $llvmMount = @("-v", "$($llvmDir):/tmp/llvm")
+    Write-BuildLog "Running container image: $ManylinuxTag"
+    $ioVolume = "${Root}:$ManylinuxRoot"
+    $userName = Get-LinuxContainerUserName
+
+    Invoke-LoggedCommand {
+        docker run --rm `
+            --user $userName `
+            --volume $ioVolume @llvmMount `
+            --workdir $ManylinuxRoot `
+            $ManylinuxTag `
+            conda run --no-capture-output pwsh build.ps1 -t install-llvm-from-source
+    }
+}
+
 task package-manylinux-llvm -depends build-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } {
-    # For any of the volumes mapped, if the dir doesn't exist,
-    # docker will create it and it will be owned by root and
-    # the caching/install breaks with permission errors.
-    # New-Item is idempotent so we don't need to check for existence
-    $cacheMount, $cacheEnv = Get-CCacheParams
+    $llvmDir = Resolve-InstallationDirectory
+    $llvmMount = @("-v", "$($llvmDir):/tmp/llvm")
     Write-BuildLog "Running container image: $ManylinuxTag"
     $ioVolume = "${Root}:$ManylinuxRoot"
     $userName = Get-LinuxContainerUserName
 
     Invoke-LoggedCommand {
         docker run --rm `
             --user $userName `
-            --volume $ioVolume @cacheMount @cacheEnv `
+            --volume $ioVolume @llvmMount `
             --workdir $ManylinuxRoot `
             --env QIRLIB_PKG_DEST=$ManylinuxRoot/target/manylinux `
             $ManylinuxTag `
@@ -228,17 +238,20 @@ task package-llvm {
 
 task build-manylinux-container-image {
     Write-BuildLog "Building container image manylinux-llvm-builder"
+    $RustVersion = (rustc --version) -split " " -match "^(\d+\.)?(\d+\.)?(\*|\d+)$"
+    Write-BuildLog "Found rustc version $RustVersion"
     Invoke-LoggedCommand -workingDirectory (Join-Path $Root eng) {
         $user = Get-LinuxContainerUserName
         $uid = Get-LinuxContainerUserId
         $gid = Get-LinuxContainerGroupId
-        Get-Content Dockerfile.manylinux | docker build `
+        docker build `
             --build-arg USERNAME=$user `
             --build-arg USER_UID=$uid `
             --build-arg USER_GID=$gid `
-            --build-arg RUST_VERSION=$RustVersion `
+            --build-arg RUST_TOOLCHAIN=$RustVersion `
             --tag $ManylinuxTag `
-            -
+            -f Dockerfile.manylinux `
+            $Root
     }
 }
 

eng/utils.ps1

@@ -107,13 +107,17 @@ function Test-LlvmConfig {
 }
 
 function Resolve-InstallationDirectory {
-    if (Test-Path env:\QIRLIB_LLVM_EXTERNAL_DIR) {
-        return $env:QIRLIB_LLVM_EXTERNAL_DIR
+    $result = if (Test-Path env:\QIRLIB_LLVM_EXTERNAL_DIR) {
+        $env:QIRLIB_LLVM_EXTERNAL_DIR
     }
     else {
         $packagePath = Get-DefaultInstallDirectory
-        return $packagePath
+        $packagePath
     }
+    if (!(Test-Path $result)) {
+        New-Item -ItemType Directory -Force $result | Out-Null
+    }
+    return $result
 }
 
 function Get-DefaultInstallDirectory {
@@ -283,7 +287,6 @@ function install-llvm {
     )
 
     $installationDirectory = Resolve-InstallationDirectory
-    New-Item -ItemType Directory -Force $installationDirectory | Out-Null
     $clear_cache_var = $false
     if (!(Test-Path env:\QIRLIB_CACHE_DIR)) {
         $clear_cache_var = $true
@@ -300,30 +303,3 @@ function install-llvm {
         }
     }   
 }
-
-function Get-CCacheParams {
-    # only ccache is supported in the container for now.
-    # we would need a way to specify which cache is used to
-    # support both.
-    if (Test-CommandExists ccache) {
-        # we need to map the local cache dir into the
-        # container. If the env var isn't set, ask ccache
-        $cacheDir = ""
-        if (Test-Path env:\CCACHE_DIR) {
-            $cacheDir = $Env:CCACHE_DIR
-        }
-        else {
-            $cacheDir = exec { ccache -k cache_dir }
-        }
-        if (![string]::IsNullOrWhiteSpace($cacheDir)) {
-            New-Item -ItemType Directory -Force $cacheDir | Out-Null
-
-            $cacheDir = Resolve-Path $cacheDir
-            # mount the cache outside of any runner mappings
-            $cacheMount = @("-v", "${cacheDir}:/ccache")
-            $cacheEnv = @("-e", "CCACHE_DIR=`"/ccache`"")
-            return $cacheMount, $cacheEnv
-        }
-    }
-    return "", ""
-}

pyqir/src/metadata.rs

@@ -90,8 +90,8 @@ impl MetadataString {
         let owner = context.clone_ref(py).into();
         let c_string = CString::new(string).unwrap();
         let context = context.borrow(py).as_ptr();
-        let md = unsafe { LLVMMDStringInContext2(context, c_string.as_ptr(), string.len()) };
-        unsafe { MetadataString::from_raw(py, owner, md) }
+        let md = LLVMMDStringInContext2(context, c_string.as_ptr(), string.len());
+        MetadataString::from_raw(py, owner, md)
     }
 
     /// The underlying metadata string value.

qirlib/src/metadata.rs

@@ -6,7 +6,7 @@ use llvm_sys::prelude::LLVMValueRef;
 use crate::llvm_wrapper::LLVMRustExtractMDConstant;
 
 pub unsafe fn extract_constant(value: LLVMValueRef) -> Option<LLVMValueRef> {
-    let constant_value = unsafe { LLVMRustExtractMDConstant(value) };
+    let constant_value = LLVMRustExtractMDConstant(value);
     if constant_value.is_null() {
         None
     } else {

qirlib/src/module.rs

@@ -49,15 +49,13 @@ pub unsafe fn add_flag(
     id: &str,
     md: LLVMMetadataRef,
 ) {
-    unsafe {
-        LLVMRustAddModuleFlag(
-            module,
-            behavior
-                .try_into()
-                .expect("Could not convert behavior for the current version of LLVM"),
-            id.as_ptr() as *mut std::ffi::c_char,
-            id.len().try_into().unwrap(),
-            md,
-        );
-    }
+    LLVMRustAddModuleFlag(
+        module,
+        behavior
+            .try_into()
+            .expect("Could not convert behavior for the current version of LLVM"),
+        id.as_ptr() as *mut std::ffi::c_char,
+        id.len().try_into().unwrap(),
+        md,
+    );
 }