From 2b61b90f03c4411cdb4892722db60c665dac91c7 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Thu, 2 May 2024 17:01:53 -0500 Subject: [PATCH 1/6] Adding models for generic workflows/steps --- api/src/shipit_api/common/models.py | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/api/src/shipit_api/common/models.py b/api/src/shipit_api/common/models.py index df336feac..c76bf7b29 100644 --- a/api/src/shipit_api/common/models.py +++ b/api/src/shipit_api/common/models.py @@ -9,6 +9,7 @@ import slugid import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import ENUM import sqlalchemy.orm from backend_common.db import db @@ -264,3 +265,66 @@ def json(self): "completed": self.completed or "", "phases": [p.json for p in self.phases], } + + +class Step(db.Model): + __tablename__ = "shipit_api_workflow_steps" + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.String, nullable=False) + submitted = sa.Column(sa.Boolean, nullable=False, default=False) + task_id = sa.Column(sa.String, nullable=False) + task = sa.Column(sa.JSON, nullable=False) + context = sa.Column(sa.JSON, nullable=False) + created = sa.Column(sa.DateTime, default=datetime.datetime.utcnow) + completed = sa.Column(sa.DateTime) + completed_by = sa.Column(sa.String) + workflow_id = sa.Column(sa.Integer, sa.ForeignKey("shipit_api_workflows.id")) + workflow = sqlalchemy.orm.relationship("Workflow", back_populates="steps") + signoffs = sqlalchemy.orm.relationship("Signoff", order_by=Signoff.id, back_populates="step") + + +class Workflow(db.Model): + __tablename__ = "shipit_api_workflows" + id = sa.Column(sa.Integer, primary_key=True) + attributes = sa.Column(sa.JSON, nullable=False) + name = sa.Column(sa.String(80), nullable=False, unique=True) + status = sa.Column(ENUM("scheduled", "completed", "aborted", name="shipit_api_workflow_status"), nullable=False) + created = sa.Column(sa.DateTime, nullable=False, default=datetime.datetime.utcnow) + completed = sa.Column(sa.DateTime) + steps = sa.orm.relationship("Step", order_by=Step.id, back_populates="workflow") + + def __init__(self, attributes, status): + self.name = self.construct_name(attributes) + self.attributes = attributes + self.status = status + + def construct_name(self, attributes): + raise NotImplementedError("Subclasses must implement this method to construct the workflow's name") + + def step_signoffs(self, step): + raise NotImplementedError("Subclasses must implement this method to get signoffs for a workflow step") + + @property + def allow_step_skipping(self): + # This only affects the frontend, *the API doesn't enforce step skipping.* + return False + + @property + def json(self): + return { + "name": self.name, + "status": self.status, + "attributes": self.attributes, + "created": self.created.isoformat(), + "completed": self.completed.isoformat() if self.completed else "", + "steps": [p.json for p in self.steps], + "allow_step_skipping": self.allow_step_skipping, + } + + +class Merge(Workflow): + def step_signoffs(self, step): + return [] + + def construct_name(self, attributes): + return f"merges-{attributes['version']}" From f95945fca14f530e51cf1b5ee8f387e4ff47bd0c Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Fri, 3 May 2024 15:06:25 -0500 Subject: [PATCH 2/6] More experiments --- api/workflows.yml | 14 ++++++++++++++ docker-compose.yml | 1 + 2 files changed, 15 insertions(+) create mode 100644 api/workflows.yml diff --git a/api/workflows.yml b/api/workflows.yml new file mode 100644 index 000000000..a704361e0 --- /dev/null +++ b/api/workflows.yml @@ -0,0 +1,14 @@ +merges: + authorized-ldap-groups: + - releng + - shipit_relman + steps: + - close-beta # closes beta using the tree status API. + - beta-to-release # merges beta to release, bumps release, tags beta, sends a notification, etc. + - central-to-beta # merges central to beta, bumps beta, tags central, sends a notification, etc. + - open-beta # sets beta the beta tree to "approval required" using the tree status API. + - bump-central # bumps the nightly version and sends a notification. + # - bump-esr-VERSION # not sure what to do about this one yet, but we can probably read HGMO and figure out the ESR bump steps? That data is here https://searchfox.org/mozilla-central/rev/38377227b8f96fda8f418db614e6a8aa67d01c31/taskcluster/config.yml#615-627 + # - update-wiki # I don't know if this is feasible any more :/ see Bug 1414278: https://bugzilla.mozilla.org/show_bug.cgi?id=1414278 can the wiki read dynamic data? + - update-shipit # This part is about updating the nightly version and release dates in shipit. To accomplish this we need to modify the app so updating these doesn't require a deployment. Maybe we can store them in a table and create APIs to manage the data. + - rebuild-product-details # sends a message to the product details pulse topic to pick up the updated shipit data. diff --git a/docker-compose.yml b/docker-compose.yml index 3f3a12c24..43e4e8823 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,7 @@ services: - ./api/src/cli_common:/app/src/cli_common - ./api/src/shipit_api:/app/src/shipit_api - "./api/products.yml:/app/products.yml" + - "./api/products.yml:/app/workflows.yml" environment: - HOST=0.0.0.0 - PORT=8016 From 9657ccd6bc467a1dab4535171e3a324ff9cfbb98 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Fri, 3 May 2024 15:18:39 -0500 Subject: [PATCH 3/6] The orm doesn't like me using the release signoffs on my experimental tables --- api/src/shipit_api/common/models.py | 40 ++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/api/src/shipit_api/common/models.py b/api/src/shipit_api/common/models.py index c76bf7b29..9562f2bb5 100644 --- a/api/src/shipit_api/common/models.py +++ b/api/src/shipit_api/common/models.py @@ -267,6 +267,38 @@ def json(self): } +class Approval(db.Model): + __tablename__ = "shipit_api_workflow_approvals" + id = sa.Column(sa.Integer, primary_key=True) + uid = sa.Column(sa.String, nullable=False, unique=True) + name = sa.Column(sa.String, nullable=False) + description = sa.Column(sa.Text) + permissions = sa.Column(sa.String, nullable=False) + completed = sa.Column(sa.DateTime) + completed_by = sa.Column(sa.String) + signed = sa.Column(sa.Boolean, default=False) + step_id = sa.Column(sa.Integer, sa.ForeignKey("shipit_api_workflow_steps.id")) + step = sqlalchemy.orm.relationship("Step", back_populates="steps") + + def __init__(self, uid, name, description, permissions): + self.uid = uid + self.name = name + self.description = description + self.permissions = permissions + + @property + def json(self): + return dict( + uid=self.uid, + name=self.name, + description=self.description, + permissions=self.permissions, + completed=self.completed or "", + completed_by=self.completed_by or "", + signed=self.signed, + ) + + class Step(db.Model): __tablename__ = "shipit_api_workflow_steps" id = sa.Column(sa.Integer, primary_key=True) @@ -280,7 +312,7 @@ class Step(db.Model): completed_by = sa.Column(sa.String) workflow_id = sa.Column(sa.Integer, sa.ForeignKey("shipit_api_workflows.id")) workflow = sqlalchemy.orm.relationship("Workflow", back_populates="steps") - signoffs = sqlalchemy.orm.relationship("Signoff", order_by=Signoff.id, back_populates="step") + approvals = sqlalchemy.orm.relationship("Approval", order_by=Approval.id, back_populates="step") class Workflow(db.Model): @@ -301,8 +333,8 @@ def __init__(self, attributes, status): def construct_name(self, attributes): raise NotImplementedError("Subclasses must implement this method to construct the workflow's name") - def step_signoffs(self, step): - raise NotImplementedError("Subclasses must implement this method to get signoffs for a workflow step") + def step_approvals(self, step): + raise NotImplementedError("Subclasses must implement this method to get approvals for a workflow step") @property def allow_step_skipping(self): @@ -323,7 +355,7 @@ def json(self): class Merge(Workflow): - def step_signoffs(self, step): + def step_approvals(self, step): return [] def construct_name(self, attributes): From d1af4b771818b5dc3b1999edbcb86f8427fe3c38 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Fri, 3 May 2024 15:23:50 -0500 Subject: [PATCH 4/6] more models tinkering --- api/src/shipit_api/common/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/shipit_api/common/models.py b/api/src/shipit_api/common/models.py index 9562f2bb5..1b031ead6 100644 --- a/api/src/shipit_api/common/models.py +++ b/api/src/shipit_api/common/models.py @@ -312,7 +312,7 @@ class Step(db.Model): completed_by = sa.Column(sa.String) workflow_id = sa.Column(sa.Integer, sa.ForeignKey("shipit_api_workflows.id")) workflow = sqlalchemy.orm.relationship("Workflow", back_populates="steps") - approvals = sqlalchemy.orm.relationship("Approval", order_by=Approval.id, back_populates="step") + approvals = sqlalchemy.orm.relationship("Approval", order_by=Approval.id, back_populates="approvals") class Workflow(db.Model): From 39703ef55e7a9071cf7acf3ed260511a6d59d6c7 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Fri, 3 May 2024 15:29:45 -0500 Subject: [PATCH 5/6] more models tinkering --- api/src/shipit_api/common/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/shipit_api/common/models.py b/api/src/shipit_api/common/models.py index 1b031ead6..30d28c378 100644 --- a/api/src/shipit_api/common/models.py +++ b/api/src/shipit_api/common/models.py @@ -278,7 +278,7 @@ class Approval(db.Model): completed_by = sa.Column(sa.String) signed = sa.Column(sa.Boolean, default=False) step_id = sa.Column(sa.Integer, sa.ForeignKey("shipit_api_workflow_steps.id")) - step = sqlalchemy.orm.relationship("Step", back_populates="steps") + step = sqlalchemy.orm.relationship("Step", back_populates="approvals") def __init__(self, uid, name, description, permissions): self.uid = uid @@ -312,7 +312,7 @@ class Step(db.Model): completed_by = sa.Column(sa.String) workflow_id = sa.Column(sa.Integer, sa.ForeignKey("shipit_api_workflows.id")) workflow = sqlalchemy.orm.relationship("Workflow", back_populates="steps") - approvals = sqlalchemy.orm.relationship("Approval", order_by=Approval.id, back_populates="approvals") + approvals = sqlalchemy.orm.relationship("Approval", order_by=Approval.id, back_populates="step") class Workflow(db.Model): From fa15bdb39688fa07a37fd9ea3ca05945188fb2d9 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Fri, 3 May 2024 17:21:20 -0500 Subject: [PATCH 6/6] appease the linter just cuz --- api/src/shipit_api/common/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/shipit_api/common/models.py b/api/src/shipit_api/common/models.py index 30d28c378..00e03b29e 100644 --- a/api/src/shipit_api/common/models.py +++ b/api/src/shipit_api/common/models.py @@ -9,8 +9,8 @@ import slugid import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import ENUM import sqlalchemy.orm +from sqlalchemy.dialects.postgresql import ENUM from backend_common.db import db from shipit_api.common.config import ALLOW_PHASE_SKIPPING, SIGNOFFS