Merge pull request #23 from multiversx/whole-project

Include whole project in "*.source.json", if desired

Dockerfile

@@ -4,8 +4,7 @@ FROM ubuntu:22.04
 ARG VERSION_RUST="nightly-2022-10-16"
 ARG VERSION_BINARYEN="105-1"
 ARG VERSION_WABT="1.0.27-1"
-# Normally, this should be "multiversx/sdk-rust-contract-builder:{{pyproject.toml:project:version}}"
-ARG CONTEXT="multiversx/sdk-rust-contract-builder:v4.1.1"
+ARG CONTEXT="multiversx/sdk-rust-contract-builder:v4.1.2"
 
 # Install dependencies (including binaryen and wabt)
 RUN apt-get update && apt-get install -y \
@@ -15,7 +14,7 @@ RUN apt-get update && apt-get install -y \
     binaryen=${VERSION_BINARYEN} \
     wabt=${VERSION_WABT}
 
-RUN pip3 install tomlkit==0.11.6 semver==2.13.0
+RUN pip3 install toml==0.10.2 semver==2.13.0
 
 # Install rust
 RUN wget -O rustup.sh https://sh.rustup.rs && \

integration_tests/shared.py

@@ -31,6 +31,7 @@ def download_packaged_src(json_url: str, name: str) -> Path:
 
 def run_docker(
     project_path: Optional[Path],
+    package_whole_project_src: bool,
     packaged_src_path: Optional[Path],
     contract_name: Optional[str],
     image: str,
@@ -64,6 +65,9 @@ def run_docker(
     if project_path:
         entrypoint_args.extend(["--project", "project"])
 
+    if package_whole_project_src:
+        entrypoint_args.append("--package-whole-project-src")
+
     if packaged_src_path:
         entrypoint_args.extend(["--packaged-src", "packaged-src.json"])
 

integration_tests/test_previous_builds_are_reproducible.py

@@ -39,7 +39,13 @@ def main(cli_args: List[str]):
         if project_path and build.project_relative_path_in_archive:
             project_path = project_path / build.project_relative_path_in_archive
 
-        run_docker(project_path, packaged_src_path, build.contract_name, build.docker_image, output_folder)
+        run_docker(
+            project_path=project_path,
+            package_whole_project_src=False,
+            packaged_src_path=packaged_src_path,
+            contract_name=build.contract_name,
+            image=build.docker_image,
+            output_folder=output_folder)
         check_code_hashes(build, output_folder)
 
 

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -8,50 +8,56 @@
 
 def main(cli_args: List[str]):
     project_path = download_project_repository("https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/main.zip", "mx-exchange-sc-main")
-    output_using_project = PARENT_OUTPUT_FOLDER / "using-project"
-    output_using_packaged_src = PARENT_OUTPUT_FOLDER / "using-packaged-src"
+    parent_output_using_project = PARENT_OUTPUT_FOLDER / "using-project"
+    parent_output_using_packaged_src = PARENT_OUTPUT_FOLDER / "using-packaged-src"
 
-    shutil.rmtree(output_using_project, ignore_errors=True)
-    shutil.rmtree(output_using_packaged_src, ignore_errors=True)
-
-    output_using_project.mkdir(parents=True)
-    output_using_packaged_src.mkdir(parents=True)
+    shutil.rmtree(parent_output_using_project, ignore_errors=True)
+    shutil.rmtree(parent_output_using_packaged_src, ignore_errors=True)
 
     contracts = ['distribution', 'energy-factory', 'energy-update', 'factory', 'farm', 'farm-staking', 'farm-staking-proxy', 'farm-with-locked-rewards', 'fees-collector', 'governance', 'governance-v2', 'lkmex-transfer', 'locked-token-wrapper', 'metabonding-staking', 'pair', 'pause-all', 'price-discovery', 'proxy-deployer', 'proxy_dex', 'router', 'simple-lock', 'simple-lock-whitelist', 'token-unstake']
 
     for contract in contracts:
-        run_docker(
-            project_path=project_path,
-            packaged_src_path=None,
-            contract_name=contract,
-            image="sdk-rust-contract-builder:next",
-            output_folder=output_using_project
-        )
-
-        packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
-
-        run_docker(
-            project_path=None,
-            packaged_src_path=packaged_src_path,
-            contract_name=contract,
-            image="sdk-rust-contract-builder:next",
-            output_folder=output_using_packaged_src
-        )
-
-        # Check that output folders are identical
-        using_project_output_files = sorted((output_using_project / contract).rglob("*"))
-        using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*"))
-
-        assert len(using_project_output_files) == len(using_packaged_src_output_files)
-
-        for index, file in enumerate(using_project_output_files):
-            if not file.is_file() or file.suffix == ".zip":
-                continue
-            using_project_file_content = file.read_bytes()
-            using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
-
-            if using_project_file_content != using_packaged_src_file_content:
-                raise Exception(f"Files differ ({contract}): {file.name}")
+        for package_whole_project_src in [True, False]:
+            output_using_project = parent_output_using_project / ("whole" if package_whole_project_src else "truncated")
+            output_using_packaged_src = parent_output_using_packaged_src / ("whole" if package_whole_project_src else "truncated")
+
+            output_using_packaged_src.mkdir(parents=True, exist_ok=True)
+            output_using_project.mkdir(parents=True, exist_ok=True)
+
+            run_docker(
+                project_path=project_path,
+                package_whole_project_src=package_whole_project_src,
+                packaged_src_path=None,
+                contract_name=contract,
+                image="sdk-rust-contract-builder:next",
+                output_folder=output_using_project
+            )
+
+            packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
+
+            run_docker(
+                project_path=None,
+                package_whole_project_src=package_whole_project_src,
+                packaged_src_path=packaged_src_path,
+                contract_name=contract,
+                image="sdk-rust-contract-builder:next",
+                output_folder=output_using_packaged_src
+            )
+
+            # Check that output folders are identical
+            using_project_output_files = sorted((output_using_project / contract).rglob("*"))
+            using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*"))
+
+            assert len(using_project_output_files) == len(using_packaged_src_output_files)
+
+            for index, file in enumerate(using_project_output_files):
+                if not file.is_file() or file.suffix == ".zip":
+                    continue
+                using_project_file_content = file.read_bytes()
+                using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
+
+                if using_project_file_content != using_packaged_src_file_content:
+                    raise Exception(f"Files differ ({contract}): {file.name}")
 
 
 if __name__ == "__main__":

multiversx_sdk_rust_contract_builder/builder.py

@@ -24,6 +24,7 @@
 
 def build_project(
         project_folder: Path,
+        package_whole_project_src: bool,
         parent_output_folder: Path,
         specific_contract: Optional[str],
         cargo_target_dir: Path,
@@ -39,8 +40,9 @@ def build_project(
     # We copy the whole project folder to the build path, to ensure that all local dependencies are available.
     project_within_build_folder = copy_project_folder_to_build_folder(project_folder)
 
-    cargo_toml.remove_dev_dependencies_sections_from_all(project_within_build_folder)
-    source_code.replace_all_test_content_with_noop(project_within_build_folder, TEST_FILES_PLACEHOLDER)
+    if not package_whole_project_src:
+        cargo_toml.remove_dev_dependencies_sections_from_all(project_within_build_folder)
+        source_code.replace_all_test_content_with_noop(project_within_build_folder, TEST_FILES_PLACEHOLDER)
 
     for contract_folder in sorted(contracts_folders):
         contract_name, contract_version = get_contract_name_and_version(contract_folder)
@@ -67,7 +69,7 @@ def build_project(
 
         # The archives are created after build, so that Cargo.lock files are included (if previously missing).
         create_archives(contract_name, contract_version, contract_build_subfolder, output_subfolder)
-        create_packaged_source_code(project_within_build_folder, contract_name, contract_version, contract_build_subfolder, output_subfolder)
+        create_packaged_source_code(project_within_build_folder, package_whole_project_src, contract_name, contract_version, contract_build_subfolder, output_subfolder)
 
         outcome.gather_artifacts(contract_name, contract_build_subfolder, output_subfolder)
 
@@ -136,8 +138,21 @@ def create_archives(contract_name: str, contract_version: str, input_folder: Pat
         warn_file_too_large(output_artifacts_archive_file, size_of_output_artifacts_archive, MAX_OUTPUT_ARTIFACTS_ARCHIVE_SIZE)
 
 
-def create_packaged_source_code(parent_project_folder: Path, contract_name: str, contract_version: str, contract_folder: Path, output_folder: Path):
-    package = PackagedSourceCode.from_filesystem(parent_project_folder, contract_folder)
+def create_packaged_source_code(
+        parent_project_folder: Path,
+        package_whole_project_src: bool,
+        contract_name: str,
+        contract_version: str,
+        contract_folder: Path,
+        output_folder: Path
+):
+    if package_whole_project_src:
+        files = source_code.get_all_source_code_files(parent_project_folder)
+    else:
+        files = source_code.get_source_code_files_necessary_for_contract(contract_folder, contract_name)
+
+    contract_name, contract_version = get_contract_name_and_version(contract_folder)
+    package = PackagedSourceCode.from_filesystem(parent_project_folder, contract_name, contract_version, files)
     package_path = output_folder / f"{contract_name}-{contract_version}.source.json"
     package.save_to_file(package_path)
 

multiversx_sdk_rust_contract_builder/builder_test.py

@@ -12,6 +12,7 @@
 def test_build_project_adder():
     outcome = builder.build_project(
         project_folder=input_folder / "adder",
+        package_whole_project_src=True,
         parent_output_folder=output_folder,
         specific_contract=None,
         cargo_target_dir=Path("/tmp/cargo-target-dir"),
@@ -33,6 +34,7 @@ def test_build_project_adder():
 def test_build_project_empty():
     outcome = builder.build_project(
         project_folder=input_folder / "empty",
+        package_whole_project_src=True,
         parent_output_folder=output_folder,
         specific_contract=None,
         cargo_target_dir=Path("/tmp/cargo-target-dir"),

multiversx_sdk_rust_contract_builder/cargo_toml.py

@@ -1,20 +1,20 @@
 import logging
 import shutil
 from pathlib import Path
-from typing import Tuple
+from typing import Any, Tuple
 
 import semver
-import tomlkit
+import toml
 
 from multiversx_sdk_rust_contract_builder.filesystem import get_all_files
 
 
 def get_contract_name_and_version(contract_folder: Path) -> Tuple[str, str]:
     file = contract_folder / "Cargo.toml"
-    toml = tomlkit.parse(file.read_text())
+    data = toml.loads(file.read_text())
 
-    name = toml["package"]["name"]
-    version = toml["package"]["version"]
+    name = data["package"]["name"]
+    version = data["package"]["version"]
     return name, version
 
 
@@ -33,27 +33,19 @@ def remove_dev_dependencies_sections_from_all(folder: Path):
 
 
 def remove_dev_dependencies_sections(file: Path):
-    # Workaround: if Cargo.toml contains [dev-dependencies] sections,
-    # then we'll attempt several times to remove them,
-    # because tomlkit does not properly handle a few special cases, such as:
-    # - https://github.com/multiversx/mx-exchange-sc/blob/v2.1-staking/dex/farm/Cargo.toml#L71
-    # - https://github.com/multiversx/mx-exchange-sc/blob/v2.1-staking/dex/farm/Cargo.toml#L84
-    while True:
-        toml = tomlkit.parse(file.read_text())
+    data = toml.loads(file.read_text())
 
-        if "dev-dependencies" not in toml:
-            break
-
-        del toml["dev-dependencies"]
-        file.write_text(tomlkit.dumps(toml))
+    if "dev-dependencies" in data:
+        del data["dev-dependencies"]
+        file.write_text(toml.dumps(data))
 
 
 def does_cargo_build_support_locked(contract_folder: Path) -> bool:
     file = contract_folder / "Cargo.toml"
-    toml = tomlkit.parse(file.read_text())
+    data: Any = toml.loads(file.read_text())
 
-    framework_version_old: str = str(toml.get("dependencies", {}).get("elrond-wasm", {}).get("version", ""))
-    framework_version_new: str = str(toml.get("dependencies", {}).get("multiversx-sc", {}).get("version", ""))
+    framework_version_old: str = str(data.get("dependencies", {}).get("elrond-wasm", {}).get("version", ""))
+    framework_version_new: str = str(data.get("dependencies", {}).get("multiversx-sc", {}).get("version", ""))
     framework_version: str = framework_version_old or framework_version_new
     framework_version = _normalize_rust_framework_version(framework_version)
 

multiversx_sdk_rust_contract_builder/main.py

@@ -22,6 +22,7 @@ def main(cli_args: List[str]):
 
     parser = ArgumentParser()
     parser.add_argument("--project", type=str, required=False, help="source code folder (project)")
+    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="include all project files in *.source.json (default: %(default)s)")
     parser.add_argument("--packaged-src", type=str, required=False, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str, required=False, help="contract to build from within the source code folder; should be relative to the project path")
     parser.add_argument("--output", type=str, required=True)
@@ -31,6 +32,7 @@ def main(cli_args: List[str]):
 
     parsed_args = parser.parse_args(cli_args)
     project_path = Path(parsed_args.project).expanduser().resolve() if parsed_args.project else None
+    package_whole_project_src = parsed_args.package_whole_project_src
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     parent_output_folder = Path(parsed_args.output)
     specific_contract = parsed_args.contract
@@ -50,6 +52,7 @@ def main(cli_args: List[str]):
 
     outcome = builder.build_project(
         project_path,
+        package_whole_project_src,
         parent_output_folder,
         specific_contract,
         cargo_target_dir,

multiversx_sdk_rust_contract_builder/packaged_source_code.py

@@ -4,11 +4,6 @@
 from pathlib import Path
 from typing import Any, Dict, List
 
-from multiversx_sdk_rust_contract_builder.cargo_toml import \
-    get_contract_name_and_version
-from multiversx_sdk_rust_contract_builder.source_code import \
-    get_source_code_files_necessary_for_contract
-
 
 class PackagedSourceCodeEntry:
     def __init__(self, path: Path, content: bytes) -> None:
@@ -54,17 +49,15 @@ def from_dict(cls, data: Dict[str, Any]) -> 'PackagedSourceCode':
         return PackagedSourceCode(name, version, entries)
 
     @classmethod
-    def from_filesystem(cls, project_folder: Path, contract_folder: Path) -> 'PackagedSourceCode':
-        name, version = get_contract_name_and_version(contract_folder)
-        entries = cls._create_entries_from_filesystem(project_folder, contract_folder, name)
-        return PackagedSourceCode(name, version, entries)
+    def from_filesystem(cls, project_folder: Path, contract_name: str, contract_version: str, files: List[Path]) -> 'PackagedSourceCode':
+        entries = cls._create_entries_from_filesystem(project_folder, files)
+        return PackagedSourceCode(contract_name, contract_version, entries)
 
     @classmethod
-    def _create_entries_from_filesystem(cls, project_folder: Path, contract_folder: Path, contract_name: str) -> List[PackagedSourceCodeEntry]:
-        source_files = get_source_code_files_necessary_for_contract(contract_folder, contract_name)
+    def _create_entries_from_filesystem(cls, project_folder: Path, files: List[Path]) -> List[PackagedSourceCodeEntry]:
         entries: List[PackagedSourceCodeEntry] = []
 
-        for full_path in source_files:
+        for full_path in files:
             content = full_path.read_bytes()
             relative_path = full_path.relative_to(project_folder)
             entries.append(PackagedSourceCodeEntry(relative_path, content))