coerce dataarg PR causing regression in utils.prepare_meta #116
Assignees
Labels
Comments
This was referenced Feb 7, 2022
|
@lukasheinrich PR #92 was back in 2019, so I'm not expecting you to remember the specifics, but if we can iterate on this that would be good. |
|
It might be worth noting that PR #92 touches the source code in a small number of places (venv) feickert@ThinkPad-X1:~/Code/GitHub/yadage/yadage$ git show b03a637 -- yadage/ | cat
commit b03a637f2afa53f5967472e5df7d17b229905e0a
Author: Lukas <[email protected]>
Date: Fri Nov 15 03:20:25 2019 +0100
coerce dataarg (#92)
* coerce data arg
diff --git a/yadage/manualcli.py b/yadage/manualcli.py
index 30f5bd2..c8922af 100755
--- a/yadage/manualcli.py
+++ b/yadage/manualcli.py
@@ -87,6 +87,7 @@ def init(
if os.path.exists("input.yml") and not initfiles:
initfiles = ("input.yml",)
initdata = utils.getinit_data(initfiles, parameter)
+ dataarg = utils.coerce_data_arg(dataarg)
dataopts = utils.options_from_eqdelimstring(dataopt)
ys = YadageSteering.create(
@@ -534,6 +535,7 @@ def add(
verbosity,
):
parameter = utils.options_from_eqdelimstring(parameter)
+ dataarg = utils.coerce_data_arg(dataarg)
dataopts = utils.options_from_eqdelimstring(dataopts)
handle_common_options(verbosity)
ys = handle_connection_options(
diff --git a/yadage/state_providers/__init__.py b/yadage/state_providers/__init__.py
index ea0840a..a2c65bd 100644
--- a/yadage/state_providers/__init__.py
+++ b/yadage/state_providers/__init__.py
@@ -80,8 +80,6 @@ def fromenv_provider(dataarg, dataopts):
def state_provider_from_string(dataarg, dataopts=None):
dataopts = dataopts or {}
- if len(dataarg.split(":", 1)) == 1:
- dataarg = "local:" + dataarg
for k in providersetup_handlers.keys():
if dataarg.startswith(k):
return providersetup_handlers[k](dataarg, dataopts)
diff --git a/yadage/steering.py b/yadage/steering.py
index fd3a7e9..043e720 100644
--- a/yadage/steering.py
+++ b/yadage/steering.py
@@ -185,6 +185,7 @@ def main(
enable_plugins(plugins.split(","))
initdata = utils.getinit_data(initfiles, parameter)
+ dataarg = utils.coerce_data_arg(dataarg)
dataopts = utils.options_from_eqdelimstring(dataopt)
backendopts = utils.options_from_eqdelimstring(backendopt)
modelopts = utils.options_from_eqdelimstring(modelopt)
diff --git a/yadage/steering_object.py b/yadage/steering_object.py
index f671c06..7d5fe49 100644
--- a/yadage/steering_object.py
+++ b/yadage/steering_object.py
@@ -45,10 +45,9 @@ class YadageSteering(object):
@classmethod
def create(cls, **kwargs):
- dataarg = kwargs["dataarg"]
dataopts = kwargs.get("dataopts") or {}
- is_local_data = len(dataarg.split(":", 1)) == 1
- if is_local_data:
+ if kwargs["dataarg"].startswith("local:"):
+ dataarg = kwargs['dataarg'].split(':',1)[1]
metadir = kwargs.get("metadir")
metadir = metadir or "{}/_yadage/".format(dataarg)
if dataopts.get("overwrite") and os.path.exists(metadir):
diff --git a/yadage/utils.py b/yadage/utils.py
index 1859174..9c00c48 100644
--- a/yadage/utils.py
+++ b/yadage/utils.py
@@ -247,3 +247,9 @@ def pointerize(data, asref=False, stepid=None):
)
return data.asrefs(callback=callback)
+
+
+def coerce_data_arg(dataarg):
+ if len(dataarg.split(":", 1)) == 1:
+ dataarg = "local:" + dataarg
+ return dataarg |
|
Looking at the following diff applied to $ git diff
diff --git a/yadage/steering_object.py b/yadage/steering_object.py
index 2ece5d0..93ee610 100644
--- a/yadage/steering_object.py
+++ b/yadage/steering_object.py
@@ -7,7 +7,7 @@ import shutil
from .serialize import snapshot
from .wflowstate import load_model_fromstring
from .controllers import setup_controller
-from .utils import setupbackend_fromstring, prepare_meta
+from .utils import setupbackend_fromstring, prepare_meta, coerce_data_arg
from .creators import handlers as creators
log = logging.getLogger(__name__)
@@ -46,21 +46,25 @@ class YadageSteering(object):
@classmethod
def create(cls, **kwargs):
dataopts = kwargs.get("dataopts") or {}
+ # kwargs["dataarg"] = coerce_data_arg(kwargs["dataarg"])
if kwargs["dataarg"].startswith("local:"):
dataarg = kwargs["dataarg"].split(":", 1)[1]
- metadir = kwargs.get("metadir")
- metadir = metadir or "{}/_yadage/".format(dataarg)
- if dataopts.get("overwrite") and os.path.exists(metadir):
- shutil.rmtree(metadir)
- else:
- metadir = kwargs["metadir"]
+
+ metadir = (
+ kwargs["metadir"]
+ if kwargs["metadir"] is not None
+ else f"{kwargs['dataarg']}/_yadage/"
+ )
+ if dataopts.get("overwrite") and os.path.exists(metadir):
+ shutil.rmtree(metadir)
+
accept_metadir = kwargs.pop("accept_metadir", False)
kw = copy.deepcopy(kwargs)
kw["metadir"] = metadir
- prepare_meta(
- metadir, accept_metadir
- ) # meta must be here because data model might store stuff here
+ # meta must be here because data model might store stuff here
+ prepare_meta(metadir, accept_metadir)
ctrl = creators["local"](**kw)
prepare_meta(metadir, accept=True) # Hack in case creator deletes meta
return cls(metadir, ctrl)if the # kwargs["dataarg"] = coerce_data_arg(kwargs["dataarg"])is uncommented so that you'd force then things will run. Though if not (don't force it), then and an error RE: Traceback (most recent call last):
File "/home/feickert/.pyenv/versions/venv/lib/python3.9/site-packages/recastatlas/backends/local.py", line 21, in run_workflow
run_workflow(**spec)
File "/home/feickert/Code/GitHub/yadage/yadage/yadage/steering_api.py", line 19, in run_workflow
with steering_ctx(*args, **kwargs):
File "/home/feickert/.pyenv/versions/3.9.6/lib/python3.9/contextlib.py", line 117, in __enter__
return next(self.gen)
File "/home/feickert/Code/GitHub/yadage/yadage/yadage/steering_api.py", line 89, in steering_ctx
ys = YadageSteering.create(
File "/home/feickert/Code/GitHub/yadage/yadage/yadage/steering_object.py", line 67, in create
prepare_meta(metadir, accept_metadir)
File "/home/feickert/Code/GitHub/yadage/yadage/yadage/creators.py", line 47, in local_workflows
rootprovider = state_provider_from_string(dataarg, dataopts)
File "/home/feickert/Code/GitHub/yadage/yadage/yadage/state_providers/__init__.py", line 86, in state_provider_from_string
raise RuntimeError("unknown data type %s %s" % (dataarg, dataopts))
RuntimeError: unknown data type recast-hello {}
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/feickert/.pyenv/versions/venv/lib/python3.9/site-packages/recastatlas/subcommands/run.py", line 56, in run
run_sync(name, spec, backend=backend)
File "/home/feickert/.pyenv/versions/venv/lib/python3.9/site-packages/recastatlas/backends/__init__.py", line 77, in run_sync
BACKENDS[backend].run_workflow(name, spec)
File "/home/feickert/.pyenv/versions/venv/lib/python3.9/site-packages/recastatlas/backends/local.py", line 23, in run_workflow
raise FailedRunException
recastatlas.exceptions.FailedRunException
Error: Workflow failed |
|
Okay, so @lukasheinrich thinks that this isn't actually a bug but just an accidental (in that this was added 2 years ago before any release was planned) API breaking change that needs to get propagated up through from yadage.utils import coerce_data_arg
...
class LocalBackend:
def run_workflow(self, name, spec):
spec["dataarg"] = coerce_data_arg(spec["dataarg"])
...in with steering_ctx(
dataarg=f"local:{workflow_workspace}",
...) What I'm going to propose is this:
Thoughts and feedback welcome from @lukasheinrich and @mvidalgarcia on this plan. |
This was referenced Feb 17, 2022
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
PR #92 introduced a regression that is hitting
recast-atlasand by proxy is also hittingreana-workflow-engine-yadage(c.f. reanahub/reana-workflow-engine-yadage#220 and reanahub/reana-workflow-engine-yadage#219) when it got into patch release0.20.2.This is showing up in calls to the steering API where
metadir=Noneyadage/yadage/steering_api.py
Lines 89 to 90 in 333fedc
yadage/yadage/steering_object.py
Lines 47 to 63 in 333fedc
where
prepare_metais passing ametadirofNoneyadage/yadage/utils.py
Lines 222 to 229 in 333fedc
Minimal failing example
Example of bug location
There was a very small number of changes to
yadagesrc betweenv0.20.1andv0.20.2(c.f.: v0.20.1...v0.20.2) and if I locally revert PR #92 and install this version in the same working Python virtual environment as above then things passThe text was updated successfully, but these errors were encountered: