From 46f58a333031293fd31bb91ab8c8fff16737f146 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 12:33:06 -0500 Subject: [PATCH 01/34] try using zig to cross compile everything --- .github/workflows/build-and-test-make.yml | 103 +++++++++++++++++++++ .github/workflows/publish-docker-image.yml | 1 + 2 files changed, 104 insertions(+) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index f8d5fc91..ac471d49 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -18,7 +18,109 @@ env: BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767 jobs: + build-and-test-cross: + name: Cross compile + runs-on: ubuntu-latest + env: + CC: zig cc + CXX: zig c++ + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-linux-gnu.2.17 + - target: aarch64-linux-gnu.2.17 + - target: arm-linux-gnu.2.17 + - target: x86_64-windows-gnu + - target: x86_64-macos-none + - target: aarch64-macos-none + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Set up zig + uses: goto-bus-stop/setup-zig@v2 + + - name: Build zlib + run: | + curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz + tar xf zlib-1.3.tar.gz + cd zlib-1.3 + ./configure --prefix=${{ runner.temp }}/zlib + make + make install + # Make sure curl will link with libz.so.1 and not libz.so + rm -f ${{ runner.temp }}/zlib/lib/libz.so + + - name: Run configure script + run: | + mkdir ${{ runner.temp }}/install + ./configure --prefix=${{ runner.temp }}/install + + # Cache the build of BoringSSL, which is the longest part of the build + # We must cache the .zip as well, otherwise the Makefile will + # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to + # reduce the insanely long build times. + - name: Cache BoringSSL source + uses: actions/cache@v3 + with: + path: boringssl.zip + key: ${{ matrix.target }}-boring-source-${{ env.BORING_SSL_COMMIT }} + + - name: Cache BoringSSL build + id: cache-boringssl + uses: actions/cache@v3 + with: + path: boringssl/build + key: ${{ matrix.target }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} + + # Trick the Makefile into skipping the BoringSSL build step + # if it was found in the cache. See Makefile.in + - name: Post BoringSSL cache restore + if: ${{ steps.cache-boringssl.outputs.cache-hit != false }} + run: | + touch boringssl.zip + touch boringssl/.patched + find boringssl/build -type f | xargs touch + + - name: Build the Chrome version of curl-impersonate + run: | + make chrome-build + make chrome-checkbuild + make chrome-install + + # Cache the build of NSS, which is the longest part of the build + # We must cache the .tar.gz as well, otherwise the Makefile will + # rebuild NSS. + - name: Cache NSS source + uses: actions/cache@v3 + with: + path: ${{ env.NSS_VERSION }}.tar.gz + key: ${{ matrix.target }}-nss-source-${{ env.NSS_VERSION }} + + - name: Cache NSS build + id: cache-nss + uses: actions/cache@v3 + with: + path: ${{ env.NSS_VERSION }}/dist + key: ${{ matrix.target }}-nss-build-${{ env.NSS_VERSION }} + + # Trick the Makefile into skipping the NSS build step + # if it was found in the cache. + - name: Post NSS cache restore + if: ${{ steps.cache-nss.outputs.cache-hit != false }} + run: | + touch ${{ env.NSS_VERSION }}.tar.gz + find ${{ env.NSS_VERSION }}/dist -type f | xargs touch + + - name: Build the Firefox version of curl-impersonate + run: | + make firefox-build + make firefox-checkbuild + make firefox-install + build-and-test-linux: + if: false name: (Linux ${{ matrix.arch }}) Build curl-impersonate and run the tests runs-on: ubuntu-latest strategy: @@ -279,6 +381,7 @@ jobs: ${{ env.release_file_bin }} build-and-test-macos: + if: false name: (MacOS ${{ matrix.arch }}) Build curl-impersonate and run the tests runs-on: ${{ matrix.os }} strategy: diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 1529f989..f43f7541 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -9,6 +9,7 @@ on: jobs: push-docker-image: + if: false name: Push Docker image to Docker Hub runs-on: ubuntu-latest steps: From 3eba8ca5127f05d0bea593301caad18e094a13d8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 12:33:58 -0500 Subject: [PATCH 02/34] skip windows build for now --- .github/workflows/build-win.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-win.yaml b/.github/workflows/build-win.yaml index e8f644ca..1aa36495 100644 --- a/.github/workflows/build-win.yaml +++ b/.github/workflows/build-win.yaml @@ -19,6 +19,7 @@ env: jobs: build-windows: + if: false name: Build windows binaries runs-on: windows-2019 strategy: From 4b2f36e277b1a597c50afe65874258ef7101f6ed Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 12:38:09 -0500 Subject: [PATCH 03/34] add zig shims --- .github/workflows/build-and-test-make.yml | 4 ++-- zigshim/cc | 2 ++ zigshim/cxx | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100755 zigshim/cc create mode 100755 zigshim/cxx diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index ac471d49..eec3b136 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -22,8 +22,8 @@ jobs: name: Cross compile runs-on: ubuntu-latest env: - CC: zig cc - CXX: zig c++ + CC: ${{ github.workspace }}/zigshim/cc + CXX: ${{ github.workspace }}/zigshim/cxx strategy: fail-fast: false matrix: diff --git a/zigshim/cc b/zigshim/cc new file mode 100755 index 00000000..a4a207fc --- /dev/null +++ b/zigshim/cc @@ -0,0 +1,2 @@ +#!/bin/bash +zig cc "$@" \ No newline at end of file diff --git a/zigshim/cxx b/zigshim/cxx new file mode 100755 index 00000000..7cc320b0 --- /dev/null +++ b/zigshim/cxx @@ -0,0 +1,2 @@ +#!/bin/bash +zig c++ "$@" \ No newline at end of file From 57f71f9501e2e6c5febf8de47a1efee9ef9b56f2 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 12:38:39 -0500 Subject: [PATCH 04/34] disable docker build for now --- .github/workflows/build-and-test-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-test-docker.yml b/.github/workflows/build-and-test-docker.yml index 3c53d2b2..cdac6ec4 100644 --- a/.github/workflows/build-and-test-docker.yml +++ b/.github/workflows/build-and-test-docker.yml @@ -11,6 +11,7 @@ on: jobs: build-docker-and-test: + if: false name: Build curl-impersonate Docker images and run the tests runs-on: ubuntu-latest services: From d0ff26a934d914e1d6a4def26fb30bc873b6f5db Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 12:41:46 -0500 Subject: [PATCH 05/34] copy over old deps install --- .github/workflows/build-and-test-make.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index eec3b136..66080e44 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -41,6 +41,22 @@ jobs: - name: Set up zig uses: goto-bus-stop/setup-zig@v2 + - name: Install Ubuntu dependencies + run: | + sudo apt-get update + sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool + # Chrome version dependencies + sudo apt-get install golang-go + # Needed to compile 'minicurl' + sudo apt-get install libcurl4-openssl-dev + # More dependencies for the tests + sudo apt-get install tcpdump nghttp2-server libnss3 + + - name: Install common dependencies + run: | + # Firefox version dependencies + pip3 install gyp-next + - name: Build zlib run: | curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz From ce1367c2e111fe440b8d6c6728570fea38830926 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 12:55:48 -0500 Subject: [PATCH 06/34] properly set cross targets --- .github/workflows/build-and-test-make.yml | 22 ++++++++++++++++++++++ zigshim/cc | 2 +- zigshim/cxx | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 66080e44..776f0198 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -24,16 +24,23 @@ jobs: env: CC: ${{ github.workspace }}/zigshim/cc CXX: ${{ github.workspace }}/zigshim/cxx + ZIG_TARGET: ${{ matrix.target }} strategy: fail-fast: false matrix: include: - target: x86_64-linux-gnu.2.17 + host: x86_64-linux-gnu - target: aarch64-linux-gnu.2.17 + host: aarch64-linux-gnu - target: arm-linux-gnu.2.17 + host: arm-linux-gnu - target: x86_64-windows-gnu + host: x86_64-windows-gnu - target: x86_64-macos-none + host: x86_64-macos-none - target: aarch64-macos-none + host: aarch64-macos-none steps: - name: Check out the repo uses: actions/checkout@v2 @@ -69,10 +76,25 @@ jobs: rm -f ${{ runner.temp }}/zlib/lib/libz.so - name: Run configure script + if: matrix.host == 'x86_64-linux-gnu' run: | mkdir ${{ runner.temp }}/install ./configure --prefix=${{ runner.temp }}/install + # When cross compiling a more complicated configuration is needed, since + # curl's configure script can't figure out where some files and libraries + # are located. The locations used here are the ones used by Ubuntu. + - name: Run configure script (cross compiling) + if: matrix.host != 'x86_64-linux-gnu' + run: | + mkdir ${{ runner.temp }}/install + ./configure --prefix=${{ runner.temp }}/install \ + --host=${{ matrix.host }} \ + --with-zlib=${{ runner.temp }}/zlib \ + --with-ca-path=/etc/ssl/certs \ + --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ + --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss + # Cache the build of BoringSSL, which is the longest part of the build # We must cache the .zip as well, otherwise the Makefile will # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to diff --git a/zigshim/cc b/zigshim/cc index a4a207fc..5e5302c2 100755 --- a/zigshim/cc +++ b/zigshim/cc @@ -1,2 +1,2 @@ #!/bin/bash -zig cc "$@" \ No newline at end of file +zig cc -target $ZIG_TARGET "$@" \ No newline at end of file diff --git a/zigshim/cxx b/zigshim/cxx index 7cc320b0..15fd2e9f 100755 --- a/zigshim/cxx +++ b/zigshim/cxx @@ -1,2 +1,2 @@ #!/bin/bash -zig c++ "$@" \ No newline at end of file +zig c++ -target $ZIG_TARGET "$@" \ No newline at end of file From 1c8f20e2d9ca45f0ddb5bc84e164a6f345fd5dd4 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:02:00 -0500 Subject: [PATCH 07/34] configur zlib host --- .github/workflows/build-and-test-make.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 776f0198..183b1ae0 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -33,8 +33,8 @@ jobs: host: x86_64-linux-gnu - target: aarch64-linux-gnu.2.17 host: aarch64-linux-gnu - - target: arm-linux-gnu.2.17 - host: arm-linux-gnu + - target: arm-linux-gnueabi.2.17 + host: arm-linux-gnueabi - target: x86_64-windows-gnu host: x86_64-windows-gnu - target: x86_64-macos-none @@ -69,7 +69,7 @@ jobs: curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz tar xf zlib-1.3.tar.gz cd zlib-1.3 - ./configure --prefix=${{ runner.temp }}/zlib + CHOST=${{ matrix.host }} ./configure --prefix=${{ runner.temp }}/zlib make make install # Make sure curl will link with libz.so.1 and not libz.so From 0943632c703c58827cace9ac1d282d7e5d3312fe Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:04:01 -0500 Subject: [PATCH 08/34] remove centos7 hacks --- chrome/patches/boringssl-old-ciphers.patch | 31 ---------------------- 1 file changed, 31 deletions(-) diff --git a/chrome/patches/boringssl-old-ciphers.patch b/chrome/patches/boringssl-old-ciphers.patch index b3821999..2b278299 100644 --- a/chrome/patches/boringssl-old-ciphers.patch +++ b/chrome/patches/boringssl-old-ciphers.patch @@ -207,34 +207,3 @@ index 57116cd6c..fa1652832 100644 } // Check for invalid algorithms, and filter out |SSL_SIGN_RSA_PKCS1_MD5_SHA1|. -diff --git a/crypto/fipsmodule/rand/urandom_test.cc b/crypto/fipsmodule/rand/urandom_test.cc -index 08e4183..d486c01 100644 ---- a/crypto/fipsmodule/rand/urandom_test.cc -+++ b/crypto/fipsmodule/rand/urandom_test.cc -@@ -38,10 +38,26 @@ - #include "fork_detect.h" - #include "getrandom_fillin.h" - -+#if !defined(NT_ARM_SYSTEM_CALL) -+// https://elixir.bootlin.com/linux/v6.6.8/source/include/uapi/linux/elf.h#L433 -+#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */ -+#endif -+ - #if !defined(PTRACE_O_EXITKILL) - #define PTRACE_O_EXITKILL (1 << 20) - #endif - -+#if defined(OPENSSL_AARCH64) && defined(__linux__) -+// https://elixir.bootlin.com/glibc/glibc-2.38/source/sysdeps/unix/sysv/linux/aarch64/sys/user.h#L22 -+struct user_regs_struct -+{ -+ unsigned long long regs[31]; -+ unsigned long long sp; -+ unsigned long long pc; -+ unsigned long long pstate; -+}; -+#endif -+ - #if defined(OPENSSL_ANDROID) - static const bool kIsAndroid = true; - #else From 5a341b01e5f9143292b12ad3ee32e7d9f4202a23 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:05:39 -0500 Subject: [PATCH 09/34] use shared configure script --- .github/workflows/build-and-test-make.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 183b1ae0..0917da76 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -76,16 +76,6 @@ jobs: rm -f ${{ runner.temp }}/zlib/lib/libz.so - name: Run configure script - if: matrix.host == 'x86_64-linux-gnu' - run: | - mkdir ${{ runner.temp }}/install - ./configure --prefix=${{ runner.temp }}/install - - # When cross compiling a more complicated configuration is needed, since - # curl's configure script can't figure out where some files and libraries - # are located. The locations used here are the ones used by Ubuntu. - - name: Run configure script (cross compiling) - if: matrix.host != 'x86_64-linux-gnu' run: | mkdir ${{ runner.temp }}/install ./configure --prefix=${{ runner.temp }}/install \ From e07345d4ae1af012de5fee03f1ff804e90ce7974 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:07:40 -0500 Subject: [PATCH 10/34] fix CHOST? --- .github/workflows/build-and-test-make.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 0917da76..19d6dcb4 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -36,11 +36,11 @@ jobs: - target: arm-linux-gnueabi.2.17 host: arm-linux-gnueabi - target: x86_64-windows-gnu - host: x86_64-windows-gnu + host: x86_64-windows - target: x86_64-macos-none - host: x86_64-macos-none + host: x86_64-macos - target: aarch64-macos-none - host: aarch64-macos-none + host: aarch64-macos steps: - name: Check out the repo uses: actions/checkout@v2 From 71b7b35bc17613e71b30c5b64e477907ebfedf81 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:30:52 -0500 Subject: [PATCH 11/34] use armhf --- .github/workflows/build-and-test-make.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 19d6dcb4..9453595d 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -33,8 +33,8 @@ jobs: host: x86_64-linux-gnu - target: aarch64-linux-gnu.2.17 host: aarch64-linux-gnu - - target: arm-linux-gnueabi.2.17 - host: arm-linux-gnueabi + - target: arm-linux-gnueabihf.2.17 + host: arm-linux-gnueabihf - target: x86_64-windows-gnu host: x86_64-windows - target: x86_64-macos-none From b84fb3915dcf6a99a95da9f410947d3c42407439 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:34:13 -0500 Subject: [PATCH 12/34] add ar shim --- .github/workflows/build-and-test-make.yml | 1 + zigshim/ar | 2 ++ 2 files changed, 3 insertions(+) create mode 100755 zigshim/ar diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 9453595d..e4278be1 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -24,6 +24,7 @@ jobs: env: CC: ${{ github.workspace }}/zigshim/cc CXX: ${{ github.workspace }}/zigshim/cxx + AR: ${{ github.workspace }}/zigshim/ar ZIG_TARGET: ${{ matrix.target }} strategy: fail-fast: false diff --git a/zigshim/ar b/zigshim/ar new file mode 100755 index 00000000..aea95b6c --- /dev/null +++ b/zigshim/ar @@ -0,0 +1,2 @@ +#!/bin/bash +zig ar -target $ZIG_TARGET "$@" \ No newline at end of file From fe861ed33f248915cdc24d161cea6994ace8f398 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:38:25 -0500 Subject: [PATCH 13/34] use windows msvc --- .github/workflows/build-and-test-make.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index e4278be1..9fc36922 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -36,7 +36,7 @@ jobs: host: aarch64-linux-gnu - target: arm-linux-gnueabihf.2.17 host: arm-linux-gnueabihf - - target: x86_64-windows-gnu + - target: x86_64-windows-msvc host: x86_64-windows - target: x86_64-macos-none host: x86_64-macos From 9cc428fd4fbeb60cf6893401d1c83384b27410d5 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:42:27 -0500 Subject: [PATCH 14/34] don't use target for ar --- zigshim/ar | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zigshim/ar b/zigshim/ar index aea95b6c..80b2e84f 100755 --- a/zigshim/ar +++ b/zigshim/ar @@ -1,2 +1,2 @@ #!/bin/bash -zig ar -target $ZIG_TARGET "$@" \ No newline at end of file +zig ar "$@" \ No newline at end of file From e455943189778a1bc60a743b89c56e511b65f977 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:50:37 -0500 Subject: [PATCH 15/34] use clang linker --- .github/workflows/build-and-test-make.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 9fc36922..67e2c342 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -25,6 +25,7 @@ jobs: CC: ${{ github.workspace }}/zigshim/cc CXX: ${{ github.workspace }}/zigshim/cxx AR: ${{ github.workspace }}/zigshim/ar + LD: lld ZIG_TARGET: ${{ matrix.target }} strategy: fail-fast: false @@ -52,7 +53,7 @@ jobs: - name: Install Ubuntu dependencies run: | sudo apt-get update - sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool + sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool lld # Chrome version dependencies sudo apt-get install golang-go # Needed to compile 'minicurl' From 69d8d3dd3c4354b561dd19ba8b3d719cf442e6f5 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 13:51:09 -0500 Subject: [PATCH 16/34] use gnu windows --- .github/workflows/build-and-test-make.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 67e2c342..9a404471 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -37,7 +37,7 @@ jobs: host: aarch64-linux-gnu - target: arm-linux-gnueabihf.2.17 host: arm-linux-gnueabihf - - target: x86_64-windows-msvc + - target: x86_64-windows-gnu host: x86_64-windows - target: x86_64-macos-none host: x86_64-macos From d7705b7da53da2e75f6ae211fd5e0aa9bb2f327d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 14:07:50 -0500 Subject: [PATCH 17/34] Revert "use clang linker" This reverts commit e455943189778a1bc60a743b89c56e511b65f977. --- .github/workflows/build-and-test-make.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 9a404471..e4278be1 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -25,7 +25,6 @@ jobs: CC: ${{ github.workspace }}/zigshim/cc CXX: ${{ github.workspace }}/zigshim/cxx AR: ${{ github.workspace }}/zigshim/ar - LD: lld ZIG_TARGET: ${{ matrix.target }} strategy: fail-fast: false @@ -53,7 +52,7 @@ jobs: - name: Install Ubuntu dependencies run: | sudo apt-get update - sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool lld + sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool # Chrome version dependencies sudo apt-get install golang-go # Needed to compile 'minicurl' From 40fc70f46a4108e7d2f9c14bb2f045c2402438c8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 16:04:16 -0500 Subject: [PATCH 18/34] windows pthreads --- .github/workflows/build-and-test-make.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index e4278be1..53032d8c 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -25,6 +25,8 @@ jobs: CC: ${{ github.workspace }}/zigshim/cc CXX: ${{ github.workspace }}/zigshim/cxx AR: ${{ github.workspace }}/zigshim/ar + CFLAGS: ${{ matrix.cflags }} + LDFLAGS: ${{ matrix.ldflags }} ZIG_TARGET: ${{ matrix.target }} strategy: fail-fast: false @@ -32,16 +34,28 @@ jobs: include: - target: x86_64-linux-gnu.2.17 host: x86_64-linux-gnu + cflags: + ldflags: - target: aarch64-linux-gnu.2.17 host: aarch64-linux-gnu + cflags: + ldflags: - target: arm-linux-gnueabihf.2.17 host: arm-linux-gnueabihf + cflags: + ldflags: - target: x86_64-windows-gnu host: x86_64-windows + cflags: -I/usr/x86_64-w64-mingw32/include + ldflags: -L/usr/x86_64-w64-mingw32/lib - target: x86_64-macos-none host: x86_64-macos + cflags: + ldflags: - target: aarch64-macos-none host: aarch64-macos + cflags: + ldflags: steps: - name: Check out the repo uses: actions/checkout@v2 @@ -60,6 +74,10 @@ jobs: # More dependencies for the tests sudo apt-get install tcpdump nghttp2-server libnss3 + - name: Install Windows dependencies + run: | + sudo apt-get install mingw-w64 + - name: Install common dependencies run: | # Firefox version dependencies From d55d6ede0812f0816b0025b331a0a3a44db3a650 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 23 Jan 2024 12:11:22 -0500 Subject: [PATCH 19/34] add cflags to makefile --- Makefile.in | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile.in b/Makefile.in index 95ba5182..a4c9eb30 100644 --- a/Makefile.in +++ b/Makefile.in @@ -198,6 +198,7 @@ $(brotli_static_libs): brotli-$(BROTLI_VERSION).tar.gz -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_CXX_COMPILER=$(CXX) \ -DCMAKE_C_COMPILER=$(CC) \ + -DCMAKE_C_FLAGS=$(CFLAGS) \ -DCMAKE_SYSTEM_NAME=$$system_name \ -DCMAKE_SYSTEM_PROCESSOR=$(host_cpu) \ .. @@ -288,7 +289,7 @@ $(boringssl_static_libs): boringssl.zip boringssl/.patched # prevent clang from failing on them. @cmake@ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_POSITION_INDEPENDENT_CODE=on \ - -DCMAKE_C_FLAGS="-Wno-unknown-warning-option -Wno-stringop-overflow -Wno-array-bounds" \ + -DCMAKE_C_FLAGS="$(CFLAGS) -Wno-unknown-warning-option -Wno-stringop-overflow -Wno-array-bounds" \ -DCMAKE_CXX_COMPILER=$(CXX) \ -DCMAKE_C_COMPILER=$(CC) \ -DCMAKE_SYSTEM_NAME=$$system_name \ @@ -388,7 +389,7 @@ $(CURL_VERSION)/.firefox: $(firefox_libs) $(CURL_VERSION).tar.xz $(CURL_VERSION) echo "Configuring curl with: $$config_flags" - ./configure $$config_flags CFLAGS="$$add_cflags" + ./configure $$config_flags CFLAGS="$(CFLAGS) $$add_cflags" # Remove possible leftovers from a previous compilation $(MAKE) clean MAKEFLAGS= touch .firefox From d0849f5d1353241ce9f34aae2ed77dbb036469f4 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Tue, 23 Jan 2024 12:20:59 -0500 Subject: [PATCH 20/34] quote cflags --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index a4c9eb30..af3cab6c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -198,7 +198,7 @@ $(brotli_static_libs): brotli-$(BROTLI_VERSION).tar.gz -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_CXX_COMPILER=$(CXX) \ -DCMAKE_C_COMPILER=$(CC) \ - -DCMAKE_C_FLAGS=$(CFLAGS) \ + -DCMAKE_C_FLAGS="$(CFLAGS)" \ -DCMAKE_SYSTEM_NAME=$$system_name \ -DCMAKE_SYSTEM_PROCESSOR=$(host_cpu) \ .. From bdc0d9cab9809aafc4a26d9c83869bef9ff4794b Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 16:36:11 -0500 Subject: [PATCH 21/34] pin version --- .github/workflows/build-and-test-make.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 53032d8c..a3114aa8 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -62,6 +62,8 @@ jobs: - name: Set up zig uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 - name: Install Ubuntu dependencies run: | From 67fca8b0937abf763af6c41803361deac6d75758 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 16:41:13 -0500 Subject: [PATCH 22/34] try older zig --- .github/workflows/build-and-test-make.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index a3114aa8..2257c782 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -63,7 +63,7 @@ jobs: - name: Set up zig uses: goto-bus-stop/setup-zig@v2 with: - version: 0.11.0 + version: 0.10.0 - name: Install Ubuntu dependencies run: | From d72fc8402be2bd05f72094ddee944b250adfcdbb Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 16:51:36 -0500 Subject: [PATCH 23/34] only use zig for linux --- .github/workflows/build-and-test-make.yml | 24 ----------------------- 1 file changed, 24 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 2257c782..c54ae196 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -25,8 +25,6 @@ jobs: CC: ${{ github.workspace }}/zigshim/cc CXX: ${{ github.workspace }}/zigshim/cxx AR: ${{ github.workspace }}/zigshim/ar - CFLAGS: ${{ matrix.cflags }} - LDFLAGS: ${{ matrix.ldflags }} ZIG_TARGET: ${{ matrix.target }} strategy: fail-fast: false @@ -34,28 +32,10 @@ jobs: include: - target: x86_64-linux-gnu.2.17 host: x86_64-linux-gnu - cflags: - ldflags: - target: aarch64-linux-gnu.2.17 host: aarch64-linux-gnu - cflags: - ldflags: - target: arm-linux-gnueabihf.2.17 host: arm-linux-gnueabihf - cflags: - ldflags: - - target: x86_64-windows-gnu - host: x86_64-windows - cflags: -I/usr/x86_64-w64-mingw32/include - ldflags: -L/usr/x86_64-w64-mingw32/lib - - target: x86_64-macos-none - host: x86_64-macos - cflags: - ldflags: - - target: aarch64-macos-none - host: aarch64-macos - cflags: - ldflags: steps: - name: Check out the repo uses: actions/checkout@v2 @@ -76,10 +56,6 @@ jobs: # More dependencies for the tests sudo apt-get install tcpdump nghttp2-server libnss3 - - name: Install Windows dependencies - run: | - sudo apt-get install mingw-w64 - - name: Install common dependencies run: | # Firefox version dependencies From 236afe3d418b2c44c3098a61015756e7326391b9 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Thu, 25 Jan 2024 16:51:48 -0500 Subject: [PATCH 24/34] use latest zig --- .github/workflows/build-and-test-make.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index c54ae196..74cac210 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -43,7 +43,7 @@ jobs: - name: Set up zig uses: goto-bus-stop/setup-zig@v2 with: - version: 0.10.0 + version: 0.11.0 - name: Install Ubuntu dependencies run: | From 443a23f4c7797b2c1839108a8ec6a6844974c502 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 21:53:05 -0500 Subject: [PATCH 25/34] merge together linux pipelines --- .github/workflows/build-and-test-make.yml | 217 ++-------------------- 1 file changed, 18 insertions(+), 199 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 860e5d8a..79fbd01a 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -55,11 +55,6 @@ jobs: # More dependencies for the tests sudo apt-get install tcpdump nghttp2-server libnss3 - - name: Install common dependencies - run: | - # Firefox version dependencies - pip3 install gyp-next - - name: Build zlib run: | curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz @@ -113,181 +108,26 @@ jobs: make chrome-checkbuild make chrome-install - # Cache the build of NSS, which is the longest part of the build - # We must cache the .tar.gz as well, otherwise the Makefile will - # rebuild NSS. - - name: Cache NSS source - uses: actions/cache@v3 - with: - path: ${{ env.NSS_VERSION }}.tar.gz - key: ${{ matrix.target }}-nss-source-${{ env.NSS_VERSION }} - - - name: Cache NSS build - id: cache-nss - uses: actions/cache@v3 - with: - path: ${{ env.NSS_VERSION }}/dist - key: ${{ matrix.target }}-nss-build-${{ env.NSS_VERSION }} - - # Trick the Makefile into skipping the NSS build step - # if it was found in the cache. - - name: Post NSS cache restore - if: ${{ steps.cache-nss.outputs.cache-hit != false }} - run: | - touch ${{ env.NSS_VERSION }}.tar.gz - find ${{ env.NSS_VERSION }}/dist -type f | xargs touch - - - name: Build the Firefox version of curl-impersonate - run: | - make firefox-build - make firefox-checkbuild - make firefox-install - - build-and-test-linux: - if: false - name: (Linux ${{ matrix.arch }}) Build curl-impersonate and run the tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arch: [x86_64] - include: - - arch: x86_64 - go_arch: amd64 - docker_arch: linux/amd64 - host: x86_64-linux-gnu - capture_interface: eth0 - image: quay.io/pypa/manylinux2014_x86_64:latest - - arch: aarch64 - go_arch: arm64 - docker_arch: linux/arm/v8 - host: aarch64-linux-gnu - capture_interface: eth0 - image: quay.io/pypa/manylinux2014_aarch64:latest - - arch: arm - go_arch: armv6l - docker_arch: linux/arm/v7 - host: arm-linux-gnu - capture_interface: eth0 - image: ghcr.io/bjia56/armv7l-wheel-builder:main - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Install dependencies - run: | - python_bin_dir=$(docker run ${{ matrix.image }} python3.10 -c 'import sys; import os; print(os.path.dirname(os.path.realpath(sys.executable)))') - echo "runner_uid=$(id -u)" >> $GITHUB_ENV - echo "runner_gid=$(id -g)" >> $GITHUB_ENV - echo "runner_home=$HOME" >> $GITHUB_ENV - docker build -t curl-impersonate-builder -f - . < - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - mkdir ${{ runner.temp }}/install - ./configure --prefix=${{ runner.temp }}/install \ - --with-ca-path=/etc/ssl/certs \ - --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt - - # Cache the build of BoringSSL, which is the longest part of the build - # We must cache the .zip as well, otherwise the Makefile will - # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to - # reduce the insanely long build times. - - name: Cache BoringSSL source - uses: actions/cache@v3 - with: - path: boringssl.zip - key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }} - - - name: Cache BoringSSL build - id: cache-boringssl - uses: actions/cache@v3 - with: - path: boringssl/build - key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} - - # Trick the Makefile into skipping the BoringSSL build step - # if it was found in the cache. See Makefile.in - - name: Post BoringSSL cache restore - if: ${{ steps.cache-boringssl.outputs.cache-hit != false }} - run: | - touch boringssl.zip - touch boringssl/.patched - find boringssl/build -type f | xargs touch - - - name: Build the Chrome version of curl-impersonate - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - make chrome-build - make chrome-checkbuild - make chrome-install - - - name: Install test dependencies - if: matrix.arch == 'x86_64' - run: | - sudo apt-get update - sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool - # Chrome version dependencies - sudo apt-get install golang-go - # Needed to compile 'minicurl' - sudo apt-get install libcurl4-openssl-dev - # More dependencies for the tests - sudo apt-get install tcpdump nghttp2-server - - name: Prepare the tests - if: matrix.arch == 'x86_64' + if: ${{ startsWith(matrix.host, 'x86_64') }} run: | # Compile 'minicurl' which is used by the tests gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs` - uses: actions/setup-python@v4 - if: matrix.arch == 'x86_64' + if: ${{ startsWith(matrix.host, 'x86_64') }} with: python-version: '3.10' - name: Install dependencies for the tests script - if: matrix.arch == 'x86_64' + if: ${{ startsWith(matrix.host, 'x86_64') }} run: | pip3 install -r tests/requirements.txt # For now we can only run the tests when native # tests run the curl-impersonate binary locally. - name: Run the tests - if: matrix.arch == 'x86_64' + if: ${{ startsWith(matrix.host, 'x86_64') }} run: | cd tests # sudo is needed for capturing packets @@ -304,47 +144,26 @@ jobs: - name: Clean build if: startsWith(github.ref, 'refs/tags/') - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - make chrome-clean - rm -Rf ${{ runner.temp }}/install - mkdir ${{ runner.temp }}/install + run: | + set -e + make chrome-clean + rm -Rf ${{ runner.temp }}/install + mkdir ${{ runner.temp }}/install # Recompile curl-impersonate statically when doing a release. - name: Reconfigure statically - if: startsWith(github.ref, 'refs/tags/') && matrix.arch == 'x86_64' - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - ./configure --prefix=${{ runner.temp }}/install --enable-static + if: startsWith(github.ref, 'refs/tags/') + run: | + set -e + ./configure --prefix=${{ runner.temp }}/install --enable-static - name: Rebuild statically if: startsWith(github.ref, 'refs/tags/') - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - make chrome-build - make chrome-checkbuild - make chrome-install-strip + run: | + set -e + make chrome-build + make chrome-checkbuild + make chrome-install-strip - name: Create tar release files for curl-impersonate if: startsWith(github.ref, 'refs/tags/') From aed4020e3f22a57216571a261550d4a93431854d Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 21:56:55 -0500 Subject: [PATCH 26/34] add back capture_interface --- .github/workflows/build-and-test-make.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 1f03af1a..06ec417e 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -31,10 +31,13 @@ jobs: include: - target: x86_64-linux-gnu.2.17 host: x86_64-linux-gnu + capture_interface: eth0 - target: aarch64-linux-gnu.2.17 host: aarch64-linux-gnu + capture_interface: eth0 - target: arm-linux-gnueabihf.2.17 host: arm-linux-gnueabihf + capture_interface: eth0 steps: - name: Check out the repo uses: actions/checkout@v2 From b6f14d1686bf115d2f95960dd2e59675d0cd9c25 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 21:58:56 -0500 Subject: [PATCH 27/34] names --- .github/workflows/build-and-test-make.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 06ec417e..1c0047b3 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -17,8 +17,8 @@ env: BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767 jobs: - build-and-test-cross: - name: Cross compile + build-and-test-linux: + name: (${{ matrix.host }}) Build curl-impersonate and run the tests runs-on: ubuntu-latest env: CC: ${{ github.workspace }}/zigshim/cc From e0d7697262f1ea3fe215a555c4f271fb7df314a9 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 22:00:30 -0500 Subject: [PATCH 28/34] arm targets raspberry pi 1 --- .github/workflows/build-and-test-make.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 1c0047b3..c5550cf2 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -25,6 +25,7 @@ jobs: CXX: ${{ github.workspace }}/zigshim/cxx AR: ${{ github.workspace }}/zigshim/ar ZIG_TARGET: ${{ matrix.target }} + CFLAGS: ${{ matrix.cflags }} strategy: fail-fast: false matrix: @@ -32,12 +33,15 @@ jobs: - target: x86_64-linux-gnu.2.17 host: x86_64-linux-gnu capture_interface: eth0 + cflags: - target: aarch64-linux-gnu.2.17 host: aarch64-linux-gnu capture_interface: eth0 + cflags: - target: arm-linux-gnueabihf.2.17 host: arm-linux-gnueabihf capture_interface: eth0 + cflags: "-mfpu=vfp -mfloat-abi=hard -mcpu=arm1176jzf_s" steps: - name: Check out the repo uses: actions/checkout@v2 From 5fd7a8fbf85c398ceba30236b31f4c4b271d6eff Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 22:01:43 -0500 Subject: [PATCH 29/34] Revert "names" This reverts commit b6f14d1686bf115d2f95960dd2e59675d0cd9c25. --- .github/workflows/build-and-test-make.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index c5550cf2..1fe084f8 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -17,8 +17,8 @@ env: BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767 jobs: - build-and-test-linux: - name: (${{ matrix.host }}) Build curl-impersonate and run the tests + build-and-test-cross: + name: Cross compile runs-on: ubuntu-latest env: CC: ${{ github.workspace }}/zigshim/cc From 00b9eac48dceb5bf80ae8a110d04a52d4cd2f7ce Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 22:05:34 -0500 Subject: [PATCH 30/34] bring back matrix.arch --- .github/workflows/build-and-test-make.yml | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 1fe084f8..3895cceb 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -17,8 +17,8 @@ env: BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767 jobs: - build-and-test-cross: - name: Cross compile + build-and-test-linux: + name: (Linux ${{ matrix.arch }}) Build curl-impersonate and run the tests runs-on: ubuntu-latest env: CC: ${{ github.workspace }}/zigshim/cc @@ -30,15 +30,18 @@ jobs: fail-fast: false matrix: include: - - target: x86_64-linux-gnu.2.17 + - arch: x86_64 + target: x86_64-linux-gnu.2.17 host: x86_64-linux-gnu capture_interface: eth0 cflags: - - target: aarch64-linux-gnu.2.17 + - arch: aarch64 + target: aarch64-linux-gnu.2.17 host: aarch64-linux-gnu capture_interface: eth0 cflags: - - target: arm-linux-gnueabihf.2.17 + - arch: arm + target: arm-linux-gnueabihf.2.17 host: arm-linux-gnueabihf capture_interface: eth0 cflags: "-mfpu=vfp -mfloat-abi=hard -mcpu=arm1176jzf_s" @@ -91,14 +94,14 @@ jobs: uses: actions/cache@v3 with: path: boringssl.zip - key: ${{ matrix.target }}-boring-source-${{ env.BORING_SSL_COMMIT }} + key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }} - name: Cache BoringSSL build id: cache-boringssl uses: actions/cache@v3 with: path: boringssl/build - key: ${{ matrix.target }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} + key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} # Trick the Makefile into skipping the BoringSSL build step # if it was found in the cache. See Makefile.in @@ -116,25 +119,25 @@ jobs: make chrome-install - name: Prepare the tests - if: ${{ startsWith(matrix.host, 'x86_64') }} + if: matrix.arch == 'x86_64' run: | # Compile 'minicurl' which is used by the tests gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs` - uses: actions/setup-python@v4 - if: ${{ startsWith(matrix.host, 'x86_64') }} + if: matrix.arch == 'x86_64' with: python-version: '3.10' - name: Install dependencies for the tests script - if: ${{ startsWith(matrix.host, 'x86_64') }} + if: matrix.arch == 'x86_64' run: | pip3 install -r tests/requirements.txt # For now we can only run the tests when native # tests run the curl-impersonate binary locally. - name: Run the tests - if: ${{ startsWith(matrix.host, 'x86_64') }} + if: matrix.arch == 'x86_64' run: | cd tests # sudo is needed for capturing packets From 450991553cf3a2041376c231e2ffe45e3e0d5606 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 22:07:16 -0500 Subject: [PATCH 31/34] reconfigure with same parameters as original configure --- .github/workflows/build-and-test-make.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 3895cceb..52faebd3 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -172,7 +172,12 @@ jobs: - name: Reconfigure statically run: | set -e - ./configure --prefix=${{ runner.temp }}/install --enable-static + ./configure --prefix=${{ runner.temp }}/install --enable-static \ + --host=${{ matrix.host }} \ + --with-zlib=${{ runner.temp }}/zlib \ + --with-ca-path=/etc/ssl/certs \ + --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ + --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss - name: Rebuild statically run: | From 4986ee9c9fad65b0c5ffc137099ff3c965af5dae Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 22:22:50 -0500 Subject: [PATCH 32/34] add cross compilers for architecture-specific strip --- .github/workflows/build-and-test-make.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 52faebd3..763187c7 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -58,6 +58,7 @@ jobs: run: | sudo apt-get update sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool + sudo apt-get install gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf # Chrome version dependencies sudo apt-get install golang-go # Needed to compile 'minicurl' From 65dc31b1377cb75e3c328bc5398752487cd442f9 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 22:34:51 -0500 Subject: [PATCH 33/34] un-disable --- .github/workflows/build-and-test-docker.yml | 1 - .github/workflows/build-win.yaml | 1 - .github/workflows/publish-docker-image.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/build-and-test-docker.yml b/.github/workflows/build-and-test-docker.yml index aaf6f3c9..19596d85 100644 --- a/.github/workflows/build-and-test-docker.yml +++ b/.github/workflows/build-and-test-docker.yml @@ -11,7 +11,6 @@ on: jobs: build-docker-and-test: - if: false name: Build curl-impersonate Docker images and run the tests runs-on: ubuntu-latest services: diff --git a/.github/workflows/build-win.yaml b/.github/workflows/build-win.yaml index 1aa36495..e8f644ca 100644 --- a/.github/workflows/build-win.yaml +++ b/.github/workflows/build-win.yaml @@ -19,7 +19,6 @@ env: jobs: build-windows: - if: false name: Build windows binaries runs-on: windows-2019 strategy: diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 437b6209..b576fc3a 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -9,7 +9,6 @@ on: jobs: push-docker-image: - if: false name: Push Docker image to Docker Hub runs-on: ubuntu-latest steps: From 143f6c2fa0d2ec9da2dea73ec536b8d5b1d104e8 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Fri, 2 Feb 2024 22:44:47 -0500 Subject: [PATCH 34/34] un-disable --- .github/workflows/build-and-test-make.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 763187c7..b1073fdc 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -212,7 +212,6 @@ jobs: ${{ env.release_file_bin }} build-and-test-macos: - if: false name: (MacOS ${{ matrix.arch }}) Build curl-impersonate and run the tests runs-on: ${{ matrix.os }} strategy: