diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..f6fdc67a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.dockerignore +.git +.gitignore +/target diff --git a/.github/actions/install-llvm/action.yml b/.github/actions/install-llvm/action.yml new file mode 100644 index 00000000..c836d3f3 --- /dev/null +++ b/.github/actions/install-llvm/action.yml @@ -0,0 +1,61 @@ +name: install-llvm +description: "Builds and installs LLVM from source using install-llvm-from-source tasks" + +inputs: + version: + description: "The version of LLVM to install." + required: true + os: + description: "The OS being built upon." + required: true + directory: + description: "The directory to install LLVM binaries to." + required: true + target: + description: "The build script target." + required: true + +runs: + using: composite + steps: + - name: LLVM install cache + id: cache-llvm + uses: actions/cache@v3 + with: + path: ${{ inputs.directory }} + key: llvm-${{ inputs.version }}-${{ inputs.arch }}-${{ inputs.os }}-${{ inputs.target }} + restore-keys: llvm-${{ inputs.version }}-${{ inputs.arch }}-${{ inputs.os }}-${{ inputs.target }} + + - name: Linux - Install build dependencies, ninja + run: sudo apt-get install -y ninja-build + shell: pwsh + if: ${{ (inputs.os == 'ubuntu-20.04') && (steps.cache-llvm.outputs.cache-hit != 'true') }} + - name: Windows - Install build dependencies, ninja + run: | + choco install --accept-license -y ninja + choco uninstall -y llvm + shell: pwsh + if: ${{ (inputs.os == 'windows-2019') && (steps.cache-llvm.outputs.cache-hit != 'true') }} + - name: MacOS - Install build dependencies, ninja + run: brew install ninja + shell: pwsh + if: ${{ (inputs.os == 'macos-11') && (steps.cache-llvm.outputs.cache-hit != 'true') }} + + - name: Configure long paths + run: git config --global core.longpaths true + shell: pwsh + if: ${{ steps.cache-llvm.outputs.cache-hit != 'true' }} + + - name: Configure LLVM Environment + run: | + Write-Output "QIRLIB_CACHE_DIR=${{ inputs.directory }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "LLVM_SYS_${{ inputs.version }}0_PREFIX=${{ inputs.directory }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "PYQIR_LLVM_FEATURE_VERSION=llvm${{ inputs.version }}-0" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + shell: pwsh + + - name: Install LLVM + run: | + $target_prefix = if("${{ inputs.target }}" -eq "manylinux") {"manylinux-"} else {""} + ./build.ps1 -t "$($target_prefix)install-llvm-from-source" + shell: pwsh + if: ${{ steps.cache-llvm.outputs.cache-hit != 'true' }} diff --git a/.github/actions/rust-toolchain/action.yml b/.github/actions/rust-toolchain/action.yml new file mode 100644 index 00000000..d0107029 --- /dev/null +++ b/.github/actions/rust-toolchain/action.yml @@ -0,0 +1,16 @@ +name: rust-toolchain +inputs: + toolchain: + required: false + components: + required: false + +runs: + using: composite + steps: + - shell: pwsh + run: | + rustup override set ${{ inputs.toolchain || 'stable' }} + ${{ inputs.components && format('rustup component add {0}', inputs.components) }} + cargo --version + rustc --version \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 109dcb7e..838fb9c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,6 @@ on: env: CARGO_TERM_COLOR: always QIRLIB_DOWNLOAD_LLVM: false - CCACHE_DIR: ~/.ccache - SCCACHE_DIR: C:\sccache jobs: build: @@ -55,9 +53,19 @@ jobs: env: {}, } steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Setup rust toolchain + uses: ./.github/actions/rust-toolchain + with: + toolchain: 1.64.0 + components: rustfmt clippy + - name: Install LLVM + uses: ./.github/actions/install-llvm with: - submodules: 'recursive' + version: "14" + os: ${{ matrix.config.os }} + directory: ${{ github.workspace }}/target/llvm + target: ${{ matrix.config.target }} - name: Linux - Install build dependencies, ccache, ninja run: sudo apt-get install -y ccache ninja-build if: ${{ matrix.config.os == 'ubuntu-20.04' }} @@ -67,31 +75,6 @@ jobs: - name: MacOS - Install build dependencies, ccache, ninja run: brew install ccache ninja if: ${{ matrix.config.os == 'macos-11' }} - - name: Windows - Install LLVM 11.1.0 - run: choco install llvm --version=11.1.0 --allow-downgrade - if: ${{ matrix.config.os == 'windows-2019' }} - - name: Get Timestamp - id: timestamp - run: Write-Host "::set-output name=timestamp::$([DateTime]::UtcNow.ToString('o'))" - shell: pwsh - - name: CCACHE cache - uses: actions/cache@v3 - with: - path: | - ${{ env.CCACHE_DIR }} - key: ${{ matrix.config.os }}-${{ matrix.config.target }}-ccache-${{ steps.timestamp.outputs.timestamp }} - restore-keys: | - ${{ matrix.config.os }}-${{ matrix.config.target }}-ccache- - if: ${{ matrix.config.os != 'windows-2019' }} - - name: SCCACHE cache - uses: actions/cache@v3 - with: - path: | - ${{ env.SCCACHE_DIR }} - key: ${{ matrix.config.os }}-sccache-${{ steps.timestamp.outputs.timestamp }} - restore-keys: | - ${{ matrix.config.os }}-sccache- - if: ${{ matrix.config.os == 'windows-2019' }} - name: "Build ${{ matrix.config.target }}" run: ./build.ps1 -t ${{ matrix.config.target }} shell: pwsh diff --git a/eng/Dockerfile.manylinux b/eng/Dockerfile.manylinux index bc9ce509..911d838b 100644 --- a/eng/Dockerfile.manylinux +++ b/eng/Dockerfile.manylinux @@ -6,7 +6,7 @@ FROM quay.io/pypa/manylinux2014_x86_64 ARG USERNAME=runner ARG USER_UID=1000 ARG USER_GID=${USER_UID} -ARG RUST_VERSION=1.64.0 +ARG RUST_TOOLCHAIN=1.64.0 RUN groupadd --gid ${USER_GID} ${USERNAME} RUN useradd --uid ${USER_UID} --gid ${USER_GID} -m ${USERNAME} @@ -16,13 +16,13 @@ RUN chmod 0440 /etc/sudoers.d/${USERNAME} ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:${PATH} \ - RUST_VERSION=${RUST_VERSION} + PATH=/usr/local/cargo/bin:${PATH} -RUN curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --profile minimal --default-toolchain ${RUST_VERSION} -y +RUN curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --profile minimal --default-toolchain none -y +RUN chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME}; -RUN chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME}; \ - rustup --version; \ +RUN rustup override set ${RUST_TOOLCHAIN} +RUN rustup --version; \ cargo --version; \ rustc --version; @@ -56,8 +56,6 @@ USER $USERNAME ENV PATH="/usr/local/miniconda3/bin:${PATH}" -ENV PATH="/usr/lib/ccache:${PATH}" - RUN conda init && \ conda install -y -c conda-forge clang-11 libstdcxx-devel_linux-64 libgcc-devel_linux-64 && \ cp /usr/local/miniconda3/bin/clang-11 /usr/local/miniconda3/bin/clang++-11 diff --git a/eng/psakefile.ps1 b/eng/psakefile.ps1 index f044a344..2ac5d2e7 100644 --- a/eng/psakefile.ps1 +++ b/eng/psakefile.ps1 @@ -14,7 +14,6 @@ Properties { $VscodeSettingsJson = Join-Path $Root .vscode settings.json $DocsRoot = Join-Path $Root docs $DocsBuild = Join-Path $DocsRoot _build - $RustVersion = "1.64.0" $ManylinuxTag = "manylinux2014_x86_64_maturin" $ManylinuxRoot = "/io" $Python = Resolve-Python @@ -26,11 +25,8 @@ task checks -depends cargo-fmt, cargo-clippy, black, mypy task manylinux -depends build-manylinux-container-image, run-manylinux-container-image, run-examples-in-containers task run-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } { - # For any of the volumes mapped, if the dir doesn't exist, - # docker will create it and it will be owned by root and - # the caching/install breaks with permission errors. - # New-Item is idempotent so we don't need to check for existence - $cacheMount, $cacheEnv = Get-CCacheParams + $llvmDir = Resolve-InstallationDirectory + $llvmMount = @("-v", "$($llvmDir):/tmp/llvm") Write-BuildLog "Running container image: $ManylinuxTag" $ioVolume = "${Root}:$ManylinuxRoot" $userName = Get-LinuxContainerUserName @@ -38,7 +34,7 @@ task run-manylinux-container-image -preaction { Write-CacheStats } -postaction { Invoke-LoggedCommand { docker run --rm ` --user $userName ` - --volume $ioVolume @cacheMount @cacheEnv ` + --volume $ioVolume @llvmMount ` --env QIRLIB_CACHE_DIR=/tmp/llvm ` --workdir $ManylinuxRoot ` $ManylinuxTag ` @@ -183,12 +179,26 @@ task install-llvm-from-source -depends configure-sccache -postaction { Write-Cac Assert (Test-LlvmConfig $installationDirectory) "install-llvm-from-source failed to install a usable LLVM installation" } +task manylinux-install-llvm-from-source -depends build-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } { + $llvmDir = Resolve-InstallationDirectory + $llvmMount = @("-v", "$($llvmDir):/tmp/llvm") + Write-BuildLog "Running container image: $ManylinuxTag" + $ioVolume = "${Root}:$ManylinuxRoot" + $userName = Get-LinuxContainerUserName + + Invoke-LoggedCommand { + docker run --rm ` + --user $userName ` + --volume $ioVolume @llvmMount ` + --workdir $ManylinuxRoot ` + $ManylinuxTag ` + conda run --no-capture-output pwsh build.ps1 -t install-llvm-from-source + } +} + task package-manylinux-llvm -depends build-manylinux-container-image -preaction { Write-CacheStats } -postaction { Write-CacheStats } { - # For any of the volumes mapped, if the dir doesn't exist, - # docker will create it and it will be owned by root and - # the caching/install breaks with permission errors. - # New-Item is idempotent so we don't need to check for existence - $cacheMount, $cacheEnv = Get-CCacheParams + $llvmDir = Resolve-InstallationDirectory + $llvmMount = @("-v", "$($llvmDir):/tmp/llvm") Write-BuildLog "Running container image: $ManylinuxTag" $ioVolume = "${Root}:$ManylinuxRoot" $userName = Get-LinuxContainerUserName @@ -196,7 +206,7 @@ task package-manylinux-llvm -depends build-manylinux-container-image -preaction Invoke-LoggedCommand { docker run --rm ` --user $userName ` - --volume $ioVolume @cacheMount @cacheEnv ` + --volume $ioVolume @llvmMount ` --workdir $ManylinuxRoot ` --env QIRLIB_PKG_DEST=$ManylinuxRoot/target/manylinux ` $ManylinuxTag ` @@ -228,17 +238,20 @@ task package-llvm { task build-manylinux-container-image { Write-BuildLog "Building container image manylinux-llvm-builder" + $RustVersion = (rustc --version) -split " " -match "^(\d+\.)?(\d+\.)?(\*|\d+)$" + Write-BuildLog "Found rustc version $RustVersion" Invoke-LoggedCommand -workingDirectory (Join-Path $Root eng) { $user = Get-LinuxContainerUserName $uid = Get-LinuxContainerUserId $gid = Get-LinuxContainerGroupId - Get-Content Dockerfile.manylinux | docker build ` + docker build ` --build-arg USERNAME=$user ` --build-arg USER_UID=$uid ` --build-arg USER_GID=$gid ` - --build-arg RUST_VERSION=$RustVersion ` + --build-arg RUST_TOOLCHAIN=$RustVersion ` --tag $ManylinuxTag ` - - + -f Dockerfile.manylinux ` + $Root } } diff --git a/eng/utils.ps1 b/eng/utils.ps1 index ad0ed7f0..7cd10775 100644 --- a/eng/utils.ps1 +++ b/eng/utils.ps1 @@ -107,13 +107,17 @@ function Test-LlvmConfig { } function Resolve-InstallationDirectory { - if (Test-Path env:\QIRLIB_LLVM_EXTERNAL_DIR) { - return $env:QIRLIB_LLVM_EXTERNAL_DIR + $result = if (Test-Path env:\QIRLIB_LLVM_EXTERNAL_DIR) { + $env:QIRLIB_LLVM_EXTERNAL_DIR } else { $packagePath = Get-DefaultInstallDirectory - return $packagePath + $packagePath } + if (!(Test-Path $result)) { + New-Item -ItemType Directory -Force $result | Out-Null + } + return $result } function Get-DefaultInstallDirectory { @@ -283,7 +287,6 @@ function install-llvm { ) $installationDirectory = Resolve-InstallationDirectory - New-Item -ItemType Directory -Force $installationDirectory | Out-Null $clear_cache_var = $false if (!(Test-Path env:\QIRLIB_CACHE_DIR)) { $clear_cache_var = $true @@ -300,30 +303,3 @@ function install-llvm { } } } - -function Get-CCacheParams { - # only ccache is supported in the container for now. - # we would need a way to specify which cache is used to - # support both. - if (Test-CommandExists ccache) { - # we need to map the local cache dir into the - # container. If the env var isn't set, ask ccache - $cacheDir = "" - if (Test-Path env:\CCACHE_DIR) { - $cacheDir = $Env:CCACHE_DIR - } - else { - $cacheDir = exec { ccache -k cache_dir } - } - if (![string]::IsNullOrWhiteSpace($cacheDir)) { - New-Item -ItemType Directory -Force $cacheDir | Out-Null - - $cacheDir = Resolve-Path $cacheDir - # mount the cache outside of any runner mappings - $cacheMount = @("-v", "${cacheDir}:/ccache") - $cacheEnv = @("-e", "CCACHE_DIR=`"/ccache`"") - return $cacheMount, $cacheEnv - } - } - return "", "" -} diff --git a/pyqir/src/metadata.rs b/pyqir/src/metadata.rs index 2c55174b..49bb44de 100644 --- a/pyqir/src/metadata.rs +++ b/pyqir/src/metadata.rs @@ -90,8 +90,8 @@ impl MetadataString { let owner = context.clone_ref(py).into(); let c_string = CString::new(string).unwrap(); let context = context.borrow(py).as_ptr(); - let md = unsafe { LLVMMDStringInContext2(context, c_string.as_ptr(), string.len()) }; - unsafe { MetadataString::from_raw(py, owner, md) } + let md = LLVMMDStringInContext2(context, c_string.as_ptr(), string.len()); + MetadataString::from_raw(py, owner, md) } /// The underlying metadata string value. diff --git a/qirlib/src/metadata.rs b/qirlib/src/metadata.rs index 077c012d..dc4742a4 100644 --- a/qirlib/src/metadata.rs +++ b/qirlib/src/metadata.rs @@ -6,7 +6,7 @@ use llvm_sys::prelude::LLVMValueRef; use crate::llvm_wrapper::LLVMRustExtractMDConstant; pub unsafe fn extract_constant(value: LLVMValueRef) -> Option { - let constant_value = unsafe { LLVMRustExtractMDConstant(value) }; + let constant_value = LLVMRustExtractMDConstant(value); if constant_value.is_null() { None } else { diff --git a/qirlib/src/module.rs b/qirlib/src/module.rs index 342a5817..1da69990 100644 --- a/qirlib/src/module.rs +++ b/qirlib/src/module.rs @@ -49,15 +49,13 @@ pub unsafe fn add_flag( id: &str, md: LLVMMetadataRef, ) { - unsafe { - LLVMRustAddModuleFlag( - module, - behavior - .try_into() - .expect("Could not convert behavior for the current version of LLVM"), - id.as_ptr() as *mut std::ffi::c_char, - id.len().try_into().unwrap(), - md, - ); - } + LLVMRustAddModuleFlag( + module, + behavior + .try_into() + .expect("Could not convert behavior for the current version of LLVM"), + id.as_ptr() as *mut std::ffi::c_char, + id.len().try_into().unwrap(), + md, + ); } diff --git a/qirlib/src/rt.rs b/qirlib/src/rt.rs index da96b0f5..ae3c172b 100644 --- a/qirlib/src/rt.rs +++ b/qirlib/src/rt.rs @@ -27,9 +27,7 @@ pub unsafe fn build_array_record_output( } pub unsafe fn build_initialize(builder: LLVMBuilderRef, data: LLVMValueRef) { - unsafe { - build_call(builder, initialize(builder_module(builder)), &mut [data]); - } + build_call(builder, initialize(builder_module(builder)), &mut [data]); } pub unsafe fn build_result_record_output( @@ -37,13 +35,11 @@ pub unsafe fn build_result_record_output( result: LLVMValueRef, label: LLVMValueRef, ) { - unsafe { - build_call( - builder, - result_record_output(builder_module(builder)), - &mut [result, label], - ); - } + build_call( + builder, + result_record_output(builder_module(builder)), + &mut [result, label], + ); } pub unsafe fn build_tuple_record_output( @@ -51,13 +47,11 @@ pub unsafe fn build_tuple_record_output( num_elements: LLVMValueRef, label: LLVMValueRef, ) { - unsafe { - build_call( - builder, - tuple_record_output(builder_module(builder)), - &mut [num_elements, label], - ); - } + build_call( + builder, + tuple_record_output(builder_module(builder)), + &mut [num_elements, label], + ); } unsafe fn array_record_output(module: LLVMModuleRef) -> LLVMValueRef {