Package whole source code (bring fix from v5 and v6).

Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.2.1"
+ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.3.0"
 ARG VERSION_RUST="nightly-2022-10-16"
 ARG VERSION_BINARYEN="version_105"
 ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz"

build_with_docker.py

@@ -20,7 +20,7 @@ def main(cli_args: List[str]):
     parser.add_argument("--packaged-src", type=str, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str)
     parser.add_argument("--output", type=str, default=Path(os.getcwd()) / "output")
-    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="include all project files in *.source.json (default: %(default)s)")
+    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="deprecated parameter, not used anymore")
     parser.add_argument("--cargo-target-dir", help="deprecated parameter, not used anymore")
     parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)")
     parser.add_argument("--build-root", type=str, required=False, help="root path (within container) for the build (default: %(default)s)")
@@ -35,7 +35,6 @@ def main(cli_args: List[str]):
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     contract_path = parsed_args.contract
     output_path = Path(parsed_args.output).expanduser().resolve()
-    package_whole_project_src = parsed_args.package_whole_project_src
     no_wasm_opt = parsed_args.no_wasm_opt
     build_root = Path(parsed_args.build_root) if parsed_args.build_root else None
     cargo_verbose = parsed_args.cargo_verbose
@@ -97,9 +96,6 @@ def main(cli_args: List[str]):
     if build_root:
         entrypoint_args.extend(["--build-root", str(build_root)])
 
-    if package_whole_project_src:
-        entrypoint_args.append("--package-whole-project-src")
-
     # Run docker container
     args = docker_general_args + docker_mount_args + docker_env_args + [image] + entrypoint_args
     logger.info(f"Running docker: {args}")

integration_tests/shared.py

@@ -31,7 +31,6 @@ def download_packaged_src(json_url: str, name: str) -> Path:
 
 def run_docker(
     project_path: Optional[Path],
-    package_whole_project_src: bool,
     packaged_src_path: Optional[Path],
     contract_name: Optional[str],
     image: str,
@@ -66,9 +65,6 @@ def run_docker(
     if project_path:
         entrypoint_args.extend(["--project", "project"])
 
-    if package_whole_project_src:
-        entrypoint_args.append("--package-whole-project-src")
-
     if packaged_src_path:
         entrypoint_args.extend(["--packaged-src", "packaged-src.json"])
 

integration_tests/test_previous_builds_are_reproducible.py

@@ -41,7 +41,6 @@ def main(cli_args: List[str]):
 
         run_docker(
             project_path=project_path,
-            package_whole_project_src=False,
             packaged_src_path=packaged_src_path,
             contract_name=build.contract_name,
             image=build.docker_image,

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -17,20 +17,18 @@ def main(cli_args: List[str]):
 
     check_project_folder_and_packaged_src_are_equivalent(
         project_path=project_path,
-        package_whole_project_src=True,
         parent_output_folder=PARENT_OUTPUT_FOLDER,
         contracts=["pair", "farm"],
     )
 
 
 def check_project_folder_and_packaged_src_are_equivalent(
         project_path: Path,
-        package_whole_project_src: bool,
         parent_output_folder: Path,
         contracts: List[str]):
     for contract in contracts:
-        output_using_project = parent_output_folder / "using-project" / contract / ("whole" if package_whole_project_src else "truncated")
-        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract / ("whole" if package_whole_project_src else "truncated")
+        output_using_project = parent_output_folder / "using-project" / contract
+        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract
 
         shutil.rmtree(output_using_project, ignore_errors=True)
         shutil.rmtree(output_using_packaged_src, ignore_errors=True)
@@ -40,18 +38,16 @@ def check_project_folder_and_packaged_src_are_equivalent(
 
         run_docker(
             project_path=project_path,
-            package_whole_project_src=package_whole_project_src,
             packaged_src_path=None,
             contract_name=contract,
             image="sdk-rust-contract-builder:next",
             output_folder=output_using_project
         )
 
-        packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
+        packaged_src_path = next((output_using_project / contract).glob("*.source.json"))
 
         run_docker(
             project_path=None,
-            package_whole_project_src=package_whole_project_src,
             packaged_src_path=packaged_src_path,
             contract_name=contract,
             image="sdk-rust-contract-builder:next",
@@ -64,14 +60,21 @@ def check_project_folder_and_packaged_src_are_equivalent(
 
         assert len(using_project_output_files) == len(using_packaged_src_output_files)
 
-        for index, file in enumerate(using_project_output_files):
-            if not file.is_file() or file.suffix == ".zip":
-                continue
-            using_project_file_content = file.read_bytes()
-            using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
+        for index, file_using_project in enumerate(using_project_output_files):
+            file_using_packaged_src = using_packaged_src_output_files[index]
 
-            if using_project_file_content != using_packaged_src_file_content:
-                raise Exception(f"Files differ ({contract}): {file.name}")
+            if not file_using_project.is_file() or file_using_project.suffix == ".zip":
+                continue
+            file_content_using_project = file_using_project.read_bytes()
+            file_content_using_packaged_src = file_using_packaged_src.read_bytes()
+
+            if file_content_using_project == file_content_using_packaged_src:
+                print(f"Files are identical ({contract}): {file_using_project.name}")
+            else:
+                print(f"Files differ ({contract}):")
+                print(f"  {file_using_project}")
+                print(f"  {file_using_packaged_src}")
+                raise Exception(f"Files differ ({contract}): {file_using_project.name}")
 
 
 if __name__ == "__main__":

multiversx_sdk_rust_contract_builder/build_options.py

@@ -6,21 +6,20 @@
 class BuildOptions:
     def __init__(
         self,
-        package_whole_project_src: bool,
         specific_contract: str,
         cargo_target_dir: Path,
         no_wasm_opt: bool,
         build_root_folder: Path,
     ) -> None:
-        self.package_whole_project_src = package_whole_project_src
         self.specific_contract = specific_contract
         self.cargo_target_dir = cargo_target_dir
         self.no_wasm_opt = no_wasm_opt
         self.build_root_folder = build_root_folder
 
     def to_dict(self) -> Dict[str, Any]:
         return {
-            "packageWholeProjectSrc": self.package_whole_project_src,
+            # "packageWholeProjectSrc" is kept due to compatibility reasons.
+            "packageWholeProjectSrc": True,
             "specificContract": self.specific_contract,
             "cargoTargetDir": str(self.cargo_target_dir),
             "noWasmOpt": self.no_wasm_opt,

multiversx_sdk_rust_contract_builder/build_outcome.py

@@ -70,6 +70,7 @@ def many_from_folders(cls, build_folder: Path, output_folder: Path) -> Dict[str,
             entry.codehash = find_file_in_folder(output_folder, f"{contract_name}.codehash.txt").read_text()
             entry.bytecode_path = BuildArtifact.find_in_output(f"{contract_name}.wasm", output_folder)
             entry.abi_path = BuildArtifact.find_in_output(f"{contract_name}.abi.json", output_folder)
+            # This is the whole project source code. The file *.partial-source.json is not listed here - so that it's advertised as little as possible.
             entry.src_package_path = BuildArtifact.find_in_output("*.source.json", output_folder)
 
             result[contract_name] = entry

multiversx_sdk_rust_contract_builder/builder.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Set
 
-from multiversx_sdk_rust_contract_builder import cargo_toml, source_code
+from multiversx_sdk_rust_contract_builder import source_code
 from multiversx_sdk_rust_contract_builder.build_metadata import BuildMetadata
 from multiversx_sdk_rust_contract_builder.build_options import BuildOptions
 from multiversx_sdk_rust_contract_builder.build_outcome import BuildOutcome
@@ -31,7 +31,6 @@ def build_project(
     project_folder = project_folder.expanduser().resolve()
     parent_output_folder = parent_output_folder.expanduser().resolve()
     cargo_target_dir = options.cargo_target_dir.expanduser().resolve()
-    package_whole_project_src = options.package_whole_project_src
     no_wasm_opt = options.no_wasm_opt
     specific_contract = options.specific_contract
     build_root_folder = options.build_root_folder
@@ -45,9 +44,6 @@ def build_project(
     # We copy the whole project folder to the build path, to ensure that all local dependencies are available.
     project_within_build_folder = copy_project_folder_to_build_folder(project_folder, build_root_folder)
 
-    if not package_whole_project_src:
-        cargo_toml.remove_dev_dependencies_sections_from_all(project_within_build_folder)
-
     for contract_folder in sorted(contracts_folders):
         contract_name, contract_version = get_contract_name_and_version(contract_folder)
         logging.info(f"Contract = {contract_name}, version = {contract_version}")
@@ -75,13 +71,23 @@ def build_project(
         # The bundle (packaged source code) is created after build, so that Cargo.lock files are included (if previously missing).
         create_packaged_source_code(
             parent_project_folder=project_within_build_folder,
-            package_whole_project_src=package_whole_project_src,
+            package_whole_project_src=True,
             contract_folder=contract_build_subfolder,
             output_folder=output_subfolder,
             build_metadata=metadata.to_dict(),
             build_options=options.to_dict(),
+            package_filename=f"{contract_name}-{contract_version}.source.json"
         )
 
+        create_packaged_source_code(
+            parent_project_folder=project_within_build_folder,
+            package_whole_project_src=False,
+            contract_folder=contract_build_subfolder,
+            output_folder=output_subfolder,
+            build_metadata=metadata.to_dict(),
+            build_options=options.to_dict(),
+            package_filename=f"{contract_name}-{contract_version}.partial-source.json"
+        )
         outcome.gather_artifacts(contract_build_subfolder, output_subfolder)
 
     return outcome
@@ -168,7 +174,8 @@ def create_packaged_source_code(
         contract_folder: Path,
         output_folder: Path,
         build_metadata: Dict[str, Any],
-        build_options: Dict[str, Any]
+        build_options: Dict[str, Any],
+        package_filename: str
 ):
     source_code_files = source_code.get_source_code_files(
         project_folder=parent_project_folder,
@@ -185,7 +192,7 @@ def create_packaged_source_code(
     )
 
     package = PackagedSourceCode.from_filesystem(metadata, parent_project_folder, source_code_files)
-    package_path = output_folder / f"{contract_name}-{contract_version}.source.json"
+    package_path = output_folder / package_filename
     package.save_to_file(package_path)
 
     size_of_file = package_path.stat().st_size

multiversx_sdk_rust_contract_builder/cargo_toml.py

@@ -1,12 +1,9 @@
-import logging
 import shutil
 from pathlib import Path
 from typing import Tuple
 
 import toml
 
-from multiversx_sdk_rust_contract_builder.filesystem import get_all_files
-
 
 def get_contract_name_and_version(contract_folder: Path) -> Tuple[str, str]:
     file = contract_folder / "Cargo.toml"
@@ -21,19 +18,3 @@ def promote_cargo_lock_to_contract_folder(build_folder: Path, contract_folder: P
     from_path = build_folder / "wasm" / "Cargo.lock"
     to_path = contract_folder / "wasm" / "Cargo.lock"
     shutil.copy(from_path, to_path)
-
-
-def remove_dev_dependencies_sections_from_all(folder: Path):
-    logging.info(f"remove_dev_dependencies_sections_from_all({folder})")
-
-    all_files = get_all_files(folder, lambda file: file.name == "Cargo.toml")
-    for file in all_files:
-        remove_dev_dependencies_sections(file)
-
-
-def remove_dev_dependencies_sections(file: Path):
-    data = toml.loads(file.read_text())
-
-    if "dev-dependencies" in data:
-        del data["dev-dependencies"]
-        file.write_text(toml.dumps(data))

multiversx_sdk_rust_contract_builder/main.py

@@ -24,7 +24,7 @@ def main(cli_args: List[str]):
 
     parser = ArgumentParser()
     parser.add_argument("--project", type=str, required=False, help="source code folder (project)")
-    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="include all project files in *.source.json (default: %(default)s)")
+    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="deprecated parameter, not used anymore")
     parser.add_argument("--packaged-src", type=str, required=False, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str, required=False, help="contract to build from within the source code folder; should be relative to the project path")
     parser.add_argument("--output", type=str, required=True)
@@ -34,7 +34,6 @@ def main(cli_args: List[str]):
 
     parsed_args = parser.parse_args(cli_args)
     project_path = Path(parsed_args.project).expanduser().resolve() if parsed_args.project else None
-    package_whole_project_src = parsed_args.package_whole_project_src
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     parent_output_folder = Path(parsed_args.output)
     specific_contract = parsed_args.contract
@@ -58,7 +57,6 @@ def main(cli_args: List[str]):
     metadata = BuildMetadata.from_env()
 
     options = BuildOptions(
-        package_whole_project_src=package_whole_project_src,
         specific_contract=specific_contract,
         cargo_target_dir=cargo_target_dir,
         no_wasm_opt=no_wasm_opt,

multiversx_sdk_rust_contract_builder/source_code.py

@@ -61,8 +61,6 @@ def get_source_code_files(
 def _is_source_code_file(path: Path) -> bool:
     if path.suffix == ".rs":
         return True
-    if path.parent.name == "meta" and path.name == "Cargo.lock":
-        return False
     if path.name in ["Cargo.toml", "Cargo.lock", "multicontract.toml", CONTRACT_CONFIG_FILENAME]:
         return True
     return False