Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

♻️ Greenify pylint (after Pydantic v2 migration) #6747

Merged
Prev Previous commit
Revert "silencing pylint errors"
This reverts commit fd29a60.
giancarloromeo committed Nov 19, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 29c6ca9ee4aed5cfb5af4e386bb8d3440161eadc
Original file line number Diff line number Diff line change
@@ -34,11 +34,11 @@ async def get_or_create_on_demand_cluster(
)
_logger.info("received cluster: %s", returned_cluster)
if returned_cluster.state is not ClusterState.RUNNING:
raise ComputationalBackendOnDemandNotReadyError( # pylint: disable=unexpected-keyword-arg
raise ComputationalBackendOnDemandNotReadyError(
eta=timedelta_as_minute_second(returned_cluster.eta)
)
if not returned_cluster.dask_scheduler_ready:
raise ComputationalBackendOnDemandNotReadyError( # pylint: disable=unexpected-keyword-arg
raise ComputationalBackendOnDemandNotReadyError(
eta=timedelta_as_minute_second(returned_cluster.eta)
)

Original file line number Diff line number Diff line change
@@ -522,9 +522,7 @@ async def get_task_result(self, job_id: str) -> TaskOutputData:
await task_future.result(timeout=_DASK_DEFAULT_TIMEOUT_S),
)
except KeyError as exc:
raise ComputationalBackendTaskNotFoundError( # pylint: disable=unexpected-keyword-arg
job_id=job_id
) from exc
raise ComputationalBackendTaskNotFoundError(job_id=job_id) from exc
except distributed.TimeoutError as exc:
raise ComputationalBackendTaskResultsNotReadyError from exc

Original file line number Diff line number Diff line change
@@ -108,9 +108,7 @@ async def _concurently_safe_acquire_client() -> DaskClient:
try:
dask_client = await _concurently_safe_acquire_client()
except Exception as exc:
raise DaskClientAcquisisitonError( # pylint: disable=unexpected-keyword-arg
cluster=cluster, error=exc
) from exc
raise DaskClientAcquisisitonError(cluster=cluster, error=exc) from exc

try:
yield dask_client
Original file line number Diff line number Diff line change
@@ -110,9 +110,7 @@ async def _compute_user_access_rights(

solved_rights = CLUSTER_NO_RIGHTS.model_dump()
for group_row in filter(lambda ugrp: ugrp[1] != GroupType.PRIMARY, user_groups):
grp_access = cluster.access_rights.get(
group_row.gid, CLUSTER_NO_RIGHTS
).model_dump()
grp_access = cluster.access_rights.get(group_row.gid, CLUSTER_NO_RIGHTS).model_dump()
for operation in ["read", "write", "delete"]:
solved_rights[operation] |= grp_access[operation]
return ClusterAccessRights(**solved_rights)
@@ -161,9 +159,7 @@ async def get_cluster(self, user_id: UserID, cluster_id: ClusterID) -> Cluster:
async with self.db_engine.acquire() as conn:
clusters_list = await _clusters_from_cluster_ids(conn, {cluster_id})
if not clusters_list:
raise ClusterNotFoundError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterNotFoundError(cluster_id=cluster_id)
the_cluster = clusters_list[0]

access_rights = await _compute_user_access_rights(
@@ -175,9 +171,7 @@ async def get_cluster(self, user_id: UserID, cluster_id: ClusterID) -> Cluster:
f"{access_rights=}",
)
if not access_rights.read:
raise ClusterAccessForbiddenError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterAccessForbiddenError(cluster_id=cluster_id)

return the_cluster

@@ -189,9 +183,7 @@ async def update_cluster( # pylint: disable=too-many-branches
conn, {cluster_id}
)
if len(clusters_list) != 1:
raise ClusterNotFoundError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterNotFoundError(cluster_id=cluster_id)
the_cluster = clusters_list[0]

this_user_access_rights = await _compute_user_access_rights(
@@ -204,16 +196,12 @@ async def update_cluster( # pylint: disable=too-many-branches
)

if not this_user_access_rights.write:
raise ClusterAccessForbiddenError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterAccessForbiddenError(cluster_id=cluster_id)

if updated_cluster.owner and updated_cluster.owner != the_cluster.owner:
# if the user wants to change the owner, we need more rights here
if this_user_access_rights != CLUSTER_ADMIN_RIGHTS:
raise ClusterAccessForbiddenError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterAccessForbiddenError(cluster_id=cluster_id)

# ensure the new owner has admin rights, too
if not updated_cluster.access_rights:
@@ -237,9 +225,7 @@ async def update_cluster( # pylint: disable=too-many-branches
]:
# a manager cannot change the owner abilities or create
# managers/admins
raise ClusterAccessForbiddenError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterAccessForbiddenError(cluster_id=cluster_id)

resolved_access_rights.update(updated_cluster.access_rights)
# ensure the user is not trying to mess around owner admin rights
@@ -249,9 +235,7 @@ async def update_cluster( # pylint: disable=too-many-branches
)
!= CLUSTER_ADMIN_RIGHTS
):
raise ClusterAccessForbiddenError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterAccessForbiddenError(cluster_id=cluster_id)

# ok we can update now
try:
@@ -261,16 +245,12 @@ async def update_cluster( # pylint: disable=too-many-branches
.values(to_clusters_db(updated_cluster, only_update=True))
)
except psycopg2.DatabaseError as e:
raise ClusterInvalidOperationError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
) from e
raise ClusterInvalidOperationError(cluster_id=cluster_id) from e
# upsert the rights
if updated_cluster.access_rights:
for grp, rights in resolved_access_rights.items():
insert_stmt = pg_insert(cluster_to_groups).values(
**rights.model_dump(by_alias=True),
gid=grp,
cluster_id=the_cluster.id,
**rights.model_dump(by_alias=True), gid=grp, cluster_id=the_cluster.id
)
on_update_stmt = insert_stmt.on_conflict_do_update(
index_elements=[
@@ -283,18 +263,14 @@ async def update_cluster( # pylint: disable=too-many-branches

clusters_list = await _clusters_from_cluster_ids(conn, {cluster_id})
if not clusters_list:
raise ClusterNotFoundError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterNotFoundError(cluster_id=cluster_id)
return clusters_list[0]

async def delete_cluster(self, user_id: UserID, cluster_id: ClusterID) -> None:
async with self.db_engine.acquire() as conn:
clusters_list = await _clusters_from_cluster_ids(conn, {cluster_id})
if not clusters_list:
raise ClusterNotFoundError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterNotFoundError(cluster_id=cluster_id)
the_cluster = clusters_list[0]

access_rights = await _compute_user_access_rights(
@@ -306,7 +282,5 @@ async def delete_cluster(self, user_id: UserID, cluster_id: ClusterID) -> None:
f"{access_rights=}",
)
if not access_rights.delete:
raise ClusterAccessForbiddenError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
)
raise ClusterAccessForbiddenError(cluster_id=cluster_id)
await conn.execute(sa.delete(clusters).where(clusters.c.id == cluster_id))
Original file line number Diff line number Diff line change
@@ -117,9 +117,7 @@ async def create(
row = await result.first()
return CompRunsAtDB.model_validate(row)
except ForeignKeyViolation as exc:
raise ClusterNotFoundError( # pylint: disable=unexpected-keyword-arg
cluster_id=cluster_id
) from exc
raise ClusterNotFoundError(cluster_id=cluster_id) from exc

async def update(
self, user_id: UserID, project_id: ProjectID, iteration: PositiveInt, **values
Original file line number Diff line number Diff line change
@@ -478,14 +478,14 @@ def check_scheduler_is_still_the_same(
):
_logger.debug("current %s", f"{client.scheduler_info()=}")
if "id" not in client.scheduler_info():
raise ComputationalSchedulerChangedError( # pylint: disable=unexpected-keyword-arg
raise ComputationalSchedulerChangedError(
original_scheduler_id=original_scheduler_id,
current_scheduler_id="No scheduler identifier",
)
current_scheduler_id = client.scheduler_info()["id"]
if current_scheduler_id != original_scheduler_id:
_logger.error("The computational backend changed!")
raise ComputationalSchedulerChangedError( # pylint: disable=unexpected-keyword-arg
raise ComputationalSchedulerChangedError(
original_scheduler_id=original_scheduler_id,
current_scheduler_id=current_scheduler_id,
)
Original file line number Diff line number Diff line change
@@ -158,19 +158,13 @@ async def _connect_with_gateway_and_create_cluster(
raise ConfigurationError(msg) from exc
except ValueError as exc:
# this is when a 404=NotFound,422=MalformedData comes up
raise DaskClientRequestError( # pylint: disable=unexpected-keyword-arg
endpoint=endpoint, error=exc
) from exc
raise DaskClientRequestError(endpoint=endpoint, error=exc) from exc
except dask_gateway.GatewayClusterError as exc:
# this is when a 409=Conflict/Cannot complete request comes up
raise DaskClusterError( # pylint: disable=unexpected-keyword-arg
endpoint=endpoint, error=exc
) from exc
raise DaskClusterError(endpoint=endpoint, error=exc) from exc
except dask_gateway.GatewayServerError as exc:
# this is when a 500 comes up
raise DaskGatewayServerError( # pylint: disable=unexpected-keyword-arg
endpoint=endpoint, error=exc
) from exc
raise DaskGatewayServerError(endpoint=endpoint, error=exc) from exc


def _is_dask_scheduler(authentication: ClusterAuthentication) -> bool:
Original file line number Diff line number Diff line change
@@ -1094,7 +1094,7 @@ async def test_task_progress_triggers(
"backend_error",
[
ComputationalBackendNotConnectedError(msg="faked disconnected backend"),
ComputationalSchedulerChangedError( # pylint: disable=unexpected-keyword-arg
ComputationalSchedulerChangedError(
original_scheduler_id="some_old_scheduler_id",
current_scheduler_id="some_new_scheduler_id",
),
@@ -1181,9 +1181,7 @@ class RebootState:
pytest.param(
RebootState(
dask_task_status=DaskClientTaskState.LOST,
task_result=ComputationalBackendTaskNotFoundError( # pylint: disable=unexpected-keyword-arg
job_id="fake_job_id"
),
task_result=ComputationalBackendTaskNotFoundError(job_id="fake_job_id"),
expected_task_state_group1=RunningState.FAILED,
expected_task_progress_group1=1,
expected_task_state_group2=RunningState.ABORTED,
@@ -1219,7 +1217,7 @@ class RebootState:
pytest.param(
RebootState(
dask_task_status=DaskClientTaskState.PENDING_OR_STARTED,
task_result=ComputationalBackendTaskResultsNotReadyError( # pylint: disable=unexpected-keyword-arg
task_result=ComputationalBackendTaskResultsNotReadyError(
job_id="fake_job_id"
),
expected_task_state_group1=RunningState.STARTED,
@@ -1519,8 +1517,10 @@ async def test_pipeline_with_on_demand_cluster_with_not_ready_backend_waits(
mocked_get_or_create_cluster: mock.Mock,
faker: Faker,
):
mocked_get_or_create_cluster.side_effect = ComputationalBackendOnDemandNotReadyError( # pylint: disable=unexpected-keyword-arg
eta=faker.time_delta(datetime.timedelta(hours=1))
mocked_get_or_create_cluster.side_effect = (
ComputationalBackendOnDemandNotReadyError(
eta=faker.time_delta(datetime.timedelta(hours=1))
)
)
# running the pipeline will trigger a call to the clusters-keeper
assert published_project.project.prj_owner
Original file line number Diff line number Diff line change
@@ -9,11 +9,11 @@ class BaseDynamicSidecarError(Exception):
"""Used as base for all exceptions"""

def __init__(
self, message: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR
self, nessage: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR
) -> None:
self.message: str = message
self.message: str = nessage
self.status_code: int = status_code
super().__init__(message)
super().__init__(nessage)


class VolumeNotFoundError(BaseDynamicSidecarError):
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ async def _execute_command(container_name: str, command: str | Sequence[str]) ->
inspect_result: dict[str, Any] = await exec_instance.inspect()
exit_code: int | None = inspect_result.get("ExitCode", None)
if exit_code != 0:
raise ContainerExecCommandFailedError( # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
raise ContainerExecCommandFailedError(
command=command, exit_code=exit_code, command_result=command_result
)

@@ -78,11 +78,9 @@ async def run_command_in_container(
)
except DockerError as e:
if e.status == status.HTTP_404_NOT_FOUND:
raise ContainerExecContainerNotFoundError( # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
raise ContainerExecContainerNotFoundError(
container_name=container_name
) from e
raise
except asyncio.TimeoutError as e:
raise ContainerExecTimeoutError( # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
timeout=timeout, command=command
) from e
raise ContainerExecTimeoutError(timeout=timeout, command=command) from e