diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..e44b810 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E501 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9db2937..5a2f6c2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,16 +23,23 @@ jobs: echo "Image not yet published. Will publish." fi + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log in to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . push: true + platforms: linux/amd64,linux/arm64 file: ./Dockerfile tags: elrondnetwork/build-contract-rust:${{ github.ref_name }} diff --git a/Dockerfile b/Dockerfile index 817b6d2..d1a0df3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 # Constants ARG VERSION_RUST="nightly-2022-08-23" -ARG VERSION_BINARYEN="version_105" -ARG VERSION_WABT="1.0.27" +ARG VERSION_BINARYEN="105-1" +ARG VERSION_WABT="1.0.27-1" RUN apt-get update && apt-get install wget -y -RUN apt-get update && apt-get install python3.8 python-is-python3 -y +RUN apt-get update && apt-get install python3.10 python-is-python3 -y RUN apt-get update && apt-get install build-essential -y # Install rust @@ -16,17 +16,10 @@ RUN wget -O rustup.sh https://sh.rustup.rs && \ rm rustup.sh # Install wasm-opt -RUN wget -O binaryen.tar.gz https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz && \ - tar -xf binaryen.tar.gz && mv binaryen-${VERSION_BINARYEN}/bin/wasm-opt /usr/bin && \ - rm binaryen.tar.gz && \ - rm -rf binaryen-${VERSION_BINARYEN} +RUN apt-get update && apt-get install binaryen=${VERSION_BINARYEN} # Install wabt -RUN wget -O wabt.tar.gz https://github.com/WebAssembly/wabt/releases/download/${VERSION_WABT}/wabt-${VERSION_WABT}-ubuntu.tar.gz && \ - tar -xf wabt.tar.gz && mv wabt-${VERSION_WABT}/bin/wasm2wat /usr/bin && mv wabt-${VERSION_WABT}/bin/wasm-objdump /usr/bin && \ - rm wabt.tar.gz && \ - rm -rf wabt-${VERSION_WABT} - +RUN apt-get update && apt-get install wabt=${VERSION_WABT} COPY "./build_within_docker.py" "/build.py" @@ -47,4 +40,3 @@ LABEL frozen="yes" LABEL rust=${VERSION_RUST} LABEL wasm-opt-binaryen=${VERSION_BINARYEN} LABEL wabt=${VERSION_WABT} - diff --git a/README.md b/README.md index c43903e..ec9ac49 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Docker image (and wrappers) for reproducible contract builds (Rust). ## Build the Docker image ``` -docker image build --no-cache . -t build-contract-rust:experimental -f ./Dockerfile +docker buildx build --no-cache . -t build-contract-rust:experimental -f ./Dockerfile ``` ## Build contract using the wrapper @@ -33,7 +33,6 @@ This is useful for useful for testing, debugging and reviewing the script. ``` export PROJECT=${HOME}/contracts/reproducible-contract-build-example -export PROJECT_ARCHIVE=${HOME}/contracts/adder.tar export OUTPUT=${HOME}/contracts/output export CARGO_TARGET_DIR=${HOME}/cargo-target-dir export OWNER_ID=$(id -u) @@ -43,7 +42,7 @@ export RUSTUP_HOME=${HOME}/elrondsdk/vendor-rust export CARGO_HOME=${HOME}/elrondsdk/vendor-rust ``` -Build a project directory: +Build a project: ``` python3 ./build_within_docker.py --project=${PROJECT} --output=${OUTPUT} \ @@ -51,12 +50,3 @@ python3 ./build_within_docker.py --project=${PROJECT} --output=${OUTPUT} \ --output-owner-id=${OWNER_ID} \ --output-group-id=${GROUP_ID} ``` - -Build a project archive: - -``` -python3 ./build_within_docker.py --project=${PROJECT_ARCHIVE} --output=${OUTPUT} \ - --cargo-target-dir=${CARGO_TARGET_DIR} \ - --output-owner-id=${OWNER_ID} \ - --output-group-id=${GROUP_ID} -``` diff --git a/build_with_docker.py b/build_with_docker.py index cefd504..7e7d9cc 100644 --- a/build_with_docker.py +++ b/build_with_docker.py @@ -14,43 +14,50 @@ def main(cli_args: List[str]): parser = ArgumentParser() parser.add_argument("--image", type=str, required=True) + parser.add_argument("--no-docker-interactive", action="store_true", default=False) + parser.add_argument("--no-docker-tty", action="store_true", default=False) parser.add_argument("--project", type=str) - parser.add_argument("--output", type=str, - default=Path(os.getcwd()) / "output") + parser.add_argument("--output", type=str, default=Path(os.getcwd()) / "output") # Providing this parameter # (a) *might* (should, but it doesn't) speed up (subsequent) builds, but # (b) *might* (with a *very low* probability) break build determinism. # As of September 2022, both (a) and (b) are still open points. parser.add_argument("--cargo-target-dir", type=str) - parser.add_argument("--no-wasm-opt", action="store_true", default=False, - help="do not optimize wasm files after the build (default: %(default)s)") + parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)") parsed_args = parser.parse_args(cli_args) image = parsed_args.image + docker_interactive = not parsed_args.no_docker_interactive + docker_tty = not parsed_args.no_docker_tty project_path = Path(parsed_args.project).expanduser().resolve() output_path = Path(parsed_args.output).expanduser().resolve() - cargo_target_dir = Path(parsed_args.cargo_target_dir).expanduser( - ).resolve() if parsed_args.cargo_target_dir else None + cargo_target_dir = Path(parsed_args.cargo_target_dir).expanduser().resolve() if parsed_args.cargo_target_dir else None no_wasm_opt = parsed_args.no_wasm_opt output_path.mkdir(parents=True, exist_ok=True) - return_code = run_docker( - image, project_path, output_path, cargo_target_dir, no_wasm_opt) + return_code = run_docker(image, docker_interactive, docker_tty, project_path, output_path, cargo_target_dir, no_wasm_opt) return return_code -def run_docker(image: str, project_path: Path, output_path: Path, cargo_target_dir: Union[Path, None], no_wasm_opt: bool): +def run_docker(image: str, docker_interactive: bool, docker_tty: bool, project_path: Path, output_path: Path, cargo_target_dir: Union[Path, None], no_wasm_opt: bool): docker_mount_args = [ "--mount", f"type=bind,source={project_path},destination=/project", "--mount", f"type=bind,source={output_path},destination=/output" ] if cargo_target_dir: - docker_mount_args += ["--mount", - f"type=bind,source={cargo_target_dir},destination=/cargo-target-dir"] + docker_mount_args += ["--mount", f"type=bind,source={cargo_target_dir},destination=/cargo-target-dir"] - docker_args = ["docker", "run"] + docker_mount_args + ["--rm", image] + docker_args = ["docker", "run"] + + if docker_interactive: + docker_args += ["--interactive"] + if docker_tty: + docker_args += ["--tty"] + + docker_args += docker_mount_args + docker_args += ["--rm", image] entrypoint_args = [ "--output-owner-id", str(os.getuid()), diff --git a/build_within_docker.py b/build_within_docker.py index 613f4be..3459f20 100644 --- a/build_within_docker.py +++ b/build_within_docker.py @@ -4,12 +4,10 @@ import shutil import subprocess import sys -import tarfile -import tempfile from argparse import ArgumentParser from hashlib import blake2b from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Tuple logger = logging.getLogger("build-within-docker") @@ -19,12 +17,12 @@ class BuildContext: def __init__(self, - contract_informal_name: str, + contract_name: str, build_directory: Path, output_directory: Path, no_wasm_opt: bool, cargo_target_dir: str) -> None: - self.contract_informal_name = contract_informal_name + self.contract_name = contract_name self.build_directory = build_directory self.output_directory = output_directory self.no_wasm_opt = no_wasm_opt @@ -35,25 +33,19 @@ class BuildArtifactsAccumulator: def __init__(self): self.contracts: Dict[str, Dict[str, str]] = dict() - def gather_artifacts(self, contract_informal_name: str, output_subdirectory: Path): - self.add_artifact(contract_informal_name, "bytecode", - find_file_in_folder(output_subdirectory, "*.wasm").name) - self.add_artifact(contract_informal_name, "text", find_file_in_folder( - output_subdirectory, "*.wat").name) - self.add_artifact(contract_informal_name, "abi", find_file_in_folder( - output_subdirectory, "*.abi.json").name) - self.add_artifact(contract_informal_name, "imports", find_file_in_folder( - output_subdirectory, "*.imports.json").name) - self.add_artifact(contract_informal_name, "codehash", find_file_in_folder( - output_subdirectory, "*.codehash.txt").name) - self.add_artifact(contract_informal_name, "src", find_file_in_folder( - output_subdirectory, "*.tar").name) - - def add_artifact(self, contract_informal_name: str, kind: str, value: str): - if contract_informal_name not in self.contracts: - self.contracts[contract_informal_name] = dict() - - self.contracts[contract_informal_name][kind] = value + def gather_artifacts(self, contract_name: str, output_subdirectory: Path): + self.add_artifact(contract_name, "bytecode", find_file_in_folder(output_subdirectory, "*.wasm").name) + self.add_artifact(contract_name, "text", find_file_in_folder(output_subdirectory, "*.wat").name) + self.add_artifact(contract_name, "abi", find_file_in_folder(output_subdirectory, "*.abi.json").name) + self.add_artifact(contract_name, "imports", find_file_in_folder(output_subdirectory, "*.imports.json").name) + self.add_artifact(contract_name, "codehash", find_file_in_folder(output_subdirectory, "*.codehash.txt").name) + self.add_artifact(contract_name, "src", find_file_in_folder(output_subdirectory, "*.zip").name) + + def add_artifact(self, contract_name: str, kind: str, value: str): + if contract_name not in self.contracts: + self.contracts[contract_name] = dict() + + self.contracts[contract_name][kind] = value def dump_to_file(self, file: Path): with open(file, "w") as f: @@ -66,17 +58,12 @@ def main(cli_args: List[str]): artifacts_accumulator = BuildArtifactsAccumulator() parser = ArgumentParser() - parser.add_argument("--project", type=str, required=True, - help="source code directory or a *.tar archive of the source code") + parser.add_argument("--project", type=str, required=True, help="source code directory") parser.add_argument("--output", type=str, required=True) - parser.add_argument("--no-wasm-opt", action="store_true", default=False, - help="do not optimize wasm files after the build (default: %(default)s)") - parser.add_argument("--cargo-target-dir", type=str, - required=True, help="Cargo's target-dir") - parser.add_argument("--output-owner-id", type=int, - required=True, help="set owner of output folder") - parser.add_argument("--output-group-id", type=int, - required=True, help="set group of output folder") + parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)") + parser.add_argument("--cargo-target-dir", type=str, required=True, help="Cargo's target-dir") + parser.add_argument("--output-owner-id", type=int, required=True, help="set owner of output folder") + parser.add_argument("--output-group-id", type=int, required=True, help="set group of output folder") parsed_args = parser.parse_args(cli_args) project_path = Path(parsed_args.project).expanduser() @@ -84,20 +71,18 @@ def main(cli_args: List[str]): owner_id = parsed_args.output_owner_id group_id = parsed_args.output_group_id - if project_path.suffix == ".tar": - project_path = extract_project_archive(project_path) - contracts_directories = get_contracts_directories(project_path) for contract_directory in sorted(contracts_directories): - contract_informal_name = contract_directory.name - output_subdirectory = parent_output_directory / contract_informal_name + contract_name, contract_version = get_contract_name_and_version(contract_directory) + logger.info(f"Contract = {contract_name}, version = {contract_version}") + + output_subdirectory = parent_output_directory / f"{contract_name}" output_subdirectory.mkdir(parents=True, exist_ok=True) - build_directory = copy_contract_directory_to_build_directory( - contract_directory) + build_directory = copy_contract_directory_to_build_directory(contract_directory) context = BuildContext( - contract_informal_name=contract_informal_name, + contract_name=contract_name, build_directory=build_directory, output_directory=output_subdirectory, no_wasm_opt=parsed_args.no_wasm_opt, @@ -111,40 +96,40 @@ def main(cli_args: List[str]): # The archive will also include the "output" folder (useful for debugging) clean(build_directory, clean_output=False) - promote_cargo_lock_to_contract_directory( - build_directory, contract_directory, owner_id, group_id) + promote_cargo_lock_to_contract_directory(build_directory, contract_directory, owner_id, group_id) # The archive is created after build, so that Cargo.lock files are included, as well (useful for debugging) - archive_source_code(contract_informal_name, - build_directory, output_subdirectory) + archive_source_code(contract_name, contract_version, build_directory, output_subdirectory) - artifacts_accumulator.gather_artifacts( - contract_informal_name, output_subdirectory) + artifacts_accumulator.gather_artifacts(contract_name, output_subdirectory) - artifacts_accumulator.dump_to_file( - parent_output_directory / "artifacts.json") + artifacts_accumulator.dump_to_file(parent_output_directory / "artifacts.json") adjust_output_ownership(parent_output_directory, owner_id, group_id) -def extract_project_archive(project_archive_path: Path): - tmpdir = Path(tempfile.TemporaryDirectory().name) - tar = tarfile.open(project_archive_path, "r") - tar.extractall(tmpdir) - tar.close() - return tmpdir - - def get_contracts_directories(project_path: Path) -> List[Path]: - directories = [ - elrond_json.parent for elrond_json in project_path.glob("**/elrond.json")] + directories = [elrond_json.parent for elrond_json in project_path.glob("**/elrond.json")] return sorted(directories) +def get_contract_name_and_version(contract_directory: Path) -> Tuple[str, str]: + # For simplicity and less dependencies installed in the Docker image, we do not rely on an external library + # to parse the metadata from Cargo.toml. + with open(contract_directory / "Cargo.toml") as file: + lines = file.readlines() + + line_with_name = next((line for line in lines if line.startswith("name = ")), 'name = "untitled"') + line_with_version = next((line for line in lines if line.startswith("version = ")), 'version = "0.0.0"') + + name = line_with_name.split("=")[1].strip().strip('"') + version = line_with_version.split("=")[1].strip().strip('"') + return name, version + + def copy_contract_directory_to_build_directory(contract_directory: Path): shutil.rmtree(HARDCODED_BUILD_DIRECTORY, ignore_errors=True) HARDCODED_BUILD_DIRECTORY.mkdir() - shutil.copytree(contract_directory, - HARDCODED_BUILD_DIRECTORY, dirs_exist_ok=True) + shutil.copytree(contract_directory, HARDCODED_BUILD_DIRECTORY, dirs_exist_ok=True) return HARDCODED_BUILD_DIRECTORY @@ -180,8 +165,7 @@ def build(context: BuildContext): generate_wabt_artifacts(wasm_file) generate_code_hash_artifact(wasm_file) - shutil.copytree(cargo_output_directory, - context.output_directory, dirs_exist_ok=True) + shutil.copytree(cargo_output_directory, context.output_directory, dirs_exist_ok=True) def promote_cargo_lock_to_contract_directory(build_directory: Path, contract_directory: Path, owner_id: int, group_id: int): @@ -196,12 +180,10 @@ def generate_wabt_artifacts(wasm_file: Path): imports_file = wasm_file.with_suffix(".imports.json") logger.info(f"Convert WASM to WAT: {wasm_file}") - subprocess.check_output(["wasm2wat", str(wasm_file), "-o", str(wat_file)], - shell=False, universal_newlines=True, stderr=subprocess.STDOUT) + subprocess.check_output(["wasm2wat", str(wasm_file), "-o", str(wat_file)], shell=False, universal_newlines=True, stderr=subprocess.STDOUT) logger.info(f"Extract imports: {wasm_file}") - imports_text = subprocess.check_output(["wasm-objdump", str(wasm_file), "--details", - "--section", "Import"], shell=False, universal_newlines=True, stderr=subprocess.STDOUT) + imports_text = subprocess.check_output(["wasm-objdump", str(wasm_file), "--details", "--section", "Import"], shell=False, universal_newlines=True, stderr=subprocess.STDOUT) imports = _parse_imports_text(imports_text) @@ -217,8 +199,7 @@ def generate_code_hash_artifact(wasm_file: Path): def _parse_imports_text(text: str) -> List[str]: - lines = [line for line in text.splitlines( - ) if "func" in line and "env" in line] + lines = [line for line in text.splitlines() if "func" in line and "env" in line] imports = [line.split(".")[-1] for line in lines] return imports @@ -236,29 +217,24 @@ def find_file_in_folder(folder: Path, pattern: str) -> Path: files = list(folder.rglob(pattern)) if len(files) == 0: - raise Exception( - f"No file matches pattern [{pattern}] in folder {folder}") + raise Exception(f"No file matches pattern [{pattern}] in folder {folder}") if len(files) > 1: - logger.warning( - f"More files match pattern [{pattern}] in folder {folder}. Will pick first:\n{files}") + logger.warning(f"More files match pattern [{pattern}] in folder {folder}. Will pick first:\n{files}") file = folder / files[0] return Path(file).resolve() -def archive_source_code(contract_informal_name: str, input_directory: Path, output_directory: Path): - archive_file = output_directory / f"{contract_informal_name}.tar" +def archive_source_code(contract_name: str, contract_version: str, input_directory: Path, output_directory: Path): + archive_file = output_directory / f"{contract_name}-{contract_version}" - tar = tarfile.open(archive_file, "w|") - tar.add(input_directory, arcname=contract_informal_name) - tar.close() + shutil.make_archive(str(archive_file), "zip", input_directory) logger.info(f"Created archive: {archive_file}") def adjust_output_ownership(output_directory: Path, owner_id: int, group_id: int): - logger.info( - f"Adjust ownership of output directory: directory = {output_directory}, owner = {owner_id}, group = {group_id}") + logger.info(f"Adjust ownership of output directory: directory = {output_directory}, owner = {owner_id}, group = {group_id}") for root, dirs, files in os.walk(output_directory): for item in dirs: