Skip to content

Commit

Permalink
Migrating to newest jfr
Browse files Browse the repository at this point in the history
  • Loading branch information
ml-evs committed Jan 4, 2024
1 parent e10d823 commit 208d540
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 29 deletions.
1 change: 1 addition & 0 deletions src/jobflow_remote/testing/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ def add(a, b):
"""Adds two numbers together and writes the answer to a file."""
return a + b


@job
def write_file(n):
with open("results.txt", "w") as f:
Expand Down
53 changes: 27 additions & 26 deletions tests/integration/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
import tempfile
import time
from pathlib import Path
from typing import Dict, Optional

import docker
import pytest
from docker.models.containers import Container


def _get_free_port(upper_bound=50_000):
Expand Down Expand Up @@ -58,9 +58,9 @@ def docker_client():

def build_and_launch_container(
docker_client: docker.client.DockerClient,
dockerfile: Optional[os.PathLike] = None,
image_name: Optional[str] = None,
ports: Optional[Dict[str, int]] = None,
dockerfile: Path | None = None,
image_name: str | None = None,
ports: dict[str, int] | None = None,
):
"""Builds and/or launches a container, returning the container object.
Expand All @@ -76,29 +76,30 @@ def build_and_launch_container(
"""

try:
if dockerfile is not None:
print(f"Building {image_name}")
_, logs = docker_client.images.build(
path=str(Path(__file__).parent.parent.parent.resolve()),
dockerfile=dockerfile,
tag=image_name,
rm=True,
quiet=False,
)

for step in logs:
if step.get("stream"):
print(step["stream"], end="")
if dockerfile is not None:
print(f"Building {image_name}")
_, logs = docker_client.images.build(
path=str(Path(__file__).parent.parent.parent.resolve()),
dockerfile=dockerfile,
tag=image_name,
rm=True,
quiet=False,
)

for step in logs:
if step.get("stream"):
print(step["stream"], end="")

try:
print(f"Launching container for {image_name}...")
container = docker_client.containers.run(
image_name, detach=True, remove=True, tty=True, ports=ports
)
assert isinstance(container, Container)
print("Waiting for container to be ready", end="")
while container.status != "running":
print(".", end="")
time.sleep(10)
time.sleep(1)
container.reload()
print("")
print(f"Container {container.id} launched.")
Expand All @@ -110,8 +111,8 @@ def build_and_launch_container(
print(f"Stopping container {container.id}...")
container.stop()
print("Done!")
except:
pass
except Exception as exc:
print(f"Failed to stop container: {exc}")


@pytest.fixture(scope="session", autouse=True)
Expand All @@ -120,7 +121,7 @@ def slurm_container(docker_client, slurm_ssh_port):
ports = {"22/tcp": slurm_ssh_port}
yield from build_and_launch_container(
docker_client,
"./tests/integration/dockerfiles/Dockerfile.slurm",
Path("./tests/integration/dockerfiles/Dockerfile.slurm"),
"jobflow-slurm:latest",
ports=ports,
)
Expand Down Expand Up @@ -203,7 +204,7 @@ def write_tmp_settings(
work_dir="/home/jobflow/jfr",
user="jobflow",
password="jobflow",
pre_run="micromamba activate jobflow",
pre_run="/home/jobflow/.venv/bin/activate",
resources={"partition": "debug", "ntasks": 1, "time": "00:01:00"},
),
},
Expand All @@ -215,7 +216,7 @@ def write_tmp_settings(
delta_retry=(1, 1, 1),
),
)
project_json = project.json(indent=2)
project_json = project.model_dump_json(indent=2)
with open(tmp_dir / f"{random_project_name}.json", "w") as f:
f.write(project_json)

Expand All @@ -224,10 +225,10 @@ def write_tmp_settings(


@pytest.fixture(scope="session")
def daemon_manager():
def daemon_manager(random_project_name):
from jobflow_remote.jobs.daemon import DaemonManager

yield DaemonManager()
yield DaemonManager.from_project_name(random_project_name)


@pytest.fixture(scope="session")
Expand Down
4 changes: 1 addition & 3 deletions tests/integration/test_slurm.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ def test_submit_flow(worker):
submit_flow(flow, worker=worker)

runner = Runner()
runner.tick_delay = 0.2
runner.run(ticks=100)


Expand All @@ -53,5 +52,4 @@ def test_submit_flow_with_dependencies(worker):
submit_flow(flow, worker=worker)

runner = Runner()
runner.tick_delay = 0.2
runner.run(ticks=10)
runner.run(ticks=100)
2 changes: 2 additions & 0 deletions tests/test_jobflow_remote.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
def test_version():
from jobflow_remote import __version__

assert __version__.startswith("0.0.1")


Expand All @@ -9,4 +10,5 @@ def test_imports():
"""
from jobflow_remote import SETTINGS # noqa

...

0 comments on commit 208d540

Please sign in to comment.