Normalize optimization.table DB model

ixmp4/core/optimization/table.py

@@ -14,7 +14,6 @@
 from ixmp4.data.abstract import Docs as DocsModel
 from ixmp4.data.abstract import Run
 from ixmp4.data.abstract import Table as TableModel
-from ixmp4.data.abstract.optimization import Column
 
 
 class Table(BaseModelFacade):
@@ -35,7 +34,7 @@ def run_id(self) -> int:
         return self._model.run__id
 
     @property
-    def data(self) -> dict[str, Any]:
+    def data(self) -> dict[str, list[float] | list[int] | list[str]]:
         return self._model.data
 
     def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
@@ -46,12 +45,12 @@ def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
         ).data
 
     @property
-    def constrained_to_indexsets(self) -> list[str]:
-        return [column.indexset.name for column in self._model.columns]
+    def indexsets(self) -> list[str]:
+        return self._model.indexsets
 
     @property
-    def columns(self) -> list[Column]:
-        return self._model.columns
+    def column_names(self) -> list[str] | None:
+        return self._model.column_names
 
     @property
     def created_at(self) -> datetime | None:

ixmp4/data/abstract/optimization/table.py

@@ -13,18 +13,19 @@
 
 from .. import base
 from ..docs import DocsRepository
-from .column import Column
 
 
 class Table(base.BaseModel, Protocol):
     """Table data model."""
 
     name: types.String
     """Unique name of the Table."""
-    data: types.JsonDict
+    data: types.Mapped[dict[str, list[float] | list[int] | list[str]]]
     """Data stored in the Table."""
-    columns: types.Mapped[list[Column]]
-    """Data specifying this Table's Columns."""
+    indexsets: types.Mapped[list[str]]
+    """List of the names of the IndexSets the Table is bound to."""
+    column_names: types.Mapped[list[str] | None]
+    """List of the Table's column names, if distinct from the IndexSet names."""
 
     run__id: types.Integer
     "Foreign unique integer id of a run."
@@ -56,7 +57,7 @@ def create(
         """Creates a Table.
 
         Each column of the Table needs to be constrained to an existing
-        :class:ixmp4.data.abstract.optimization.IndexSet. These are specified by name
+        :class:`ixmp4.data.abstract.optimization.IndexSet`. These are specified by name
         and per default, these will be the column names. They can be overwritten by
         specifying `column_names`, which needs to specify a unique name for each column.
 
@@ -78,7 +79,7 @@ def create(
         ------
         :class:`ixmp4.data.abstract.optimization.Table.NotUnique`:
             If the Table with `name` already exists for the Run with `run_id`.
-        ValueError
+        :class:`ixmp4.core.exceptions.OptimizationItemUsageError`:
             If `column_names` are not unique or not enough names are given.
 
         Returns

ixmp4/data/api/optimization/table.py

@@ -14,7 +14,6 @@
 
 from .. import base
 from ..docs import Docs, DocsRepository
-from .column import Column
 
 
 class Table(base.BaseModel):
@@ -24,8 +23,9 @@ class Table(base.BaseModel):
 
     id: int
     name: str
-    data: dict[str, Any]
-    columns: list["Column"]
+    data: dict[str, list[float] | list[int] | list[str]]
+    indexsets: list[str]
+    column_names: list[str] | None
     run__id: int
 
     created_at: datetime | None

ixmp4/data/db/base.py

@@ -125,6 +125,8 @@ class CreateKwargs(TypedDict, total=False):
     parameters: Mapping[str, Any]
     run_id: int
     unit_name: str | None
+    column_names: list[str] | None
+    constrained_to_indexsets: list[str]
 
 
 class Creator(BaseRepository[ModelType], abstract.Creator):

ixmp4/data/db/optimization/__init__.py

@@ -1,6 +1,6 @@
 from .column import Column, ColumnRepository
 from .equation import Equation, EquationRepository
-from .indexset import IndexSet, IndexSetRepository
+from .indexset import IndexSet, IndexSetData, IndexSetRepository
 from .parameter import Parameter, ParameterRepository
 from .scalar import Scalar, ScalarRepository
 from .table import Table, TableRepository

ixmp4/data/db/optimization/indexset/__init__.py

@@ -1,2 +1,2 @@
-from .model import IndexSet
+from .model import IndexSet, IndexSetData
 from .repository import IndexSetRepository

ixmp4/data/db/optimization/indexset/repository.py

@@ -78,9 +78,7 @@ def add_data(
         indexset = self.get_by_id(id=indexset_id)
         _data = data if isinstance(data, list) else [data]
 
-        bulk_insert_enabled_data: list[dict[str, str]] = [
-            {"value": str(d)} for d in _data
-        ]
+        bulk_insert_enabled_data = [{"value": str(d)} for d in _data]
         try:
             self.session.execute(
                 db.insert(IndexSetData).values(indexset__id=indexset_id),

ixmp4/data/db/optimization/table/model.py

@@ -1,14 +1,26 @@
-from typing import Any, ClassVar
+from typing import ClassVar, Literal, cast
 
-from sqlalchemy.orm import Mapped as Mapped
-from sqlalchemy.orm import validates
+import pandas as pd
 
 from ixmp4 import db
 from ixmp4.core.exceptions import OptimizationDataValidationError
 from ixmp4.data import types
 from ixmp4.data.abstract import optimization as abstract
 
-from .. import Column, base, utils
+from .. import IndexSet, base
+
+
+class TableIndexsetAssociation(base.RootBaseModel):
+    table_prefix = "optimization_"
+
+    table_id: types.TableId
+    table: types.Mapped["Table"] = db.relationship(
+        back_populates="_table_indexset_associations"
+    )
+    indexset_id: types.IndexSetId
+    indexset: types.Mapped[IndexSet] = db.relationship()
+
+    column_name: types.String = db.Column(db.String(255), nullable=True)
 
 
 class Table(base.BaseModel):
@@ -18,20 +30,129 @@ class Table(base.BaseModel):
     DataInvalid: ClassVar = OptimizationDataValidationError
     DeletionPrevented: ClassVar = abstract.Table.DeletionPrevented
 
-    # constrained_to_indexsets: ClassVar[list[str] | None] = None
-
     run__id: types.RunId
-    columns: types.Mapped[list["Column"]] = db.relationship()
-    data: types.JsonDict = db.Column(db.JsonType, nullable=False, default={})
 
-    # TODO: should we pass self to validate_data to raise more specific errors?
+    __table_args__ = (db.UniqueConstraint("name", "run__id"),)
 
-    @validates("data")
-    def validate_data(self, key: Any, data: dict[str, Any]) -> dict[str, Any]:
-        return utils.validate_data(
-            host=self,
-            data=data,
-            columns=self.columns,
-        )
+    _table_indexset_associations: types.Mapped[list[TableIndexsetAssociation]] = (
+        db.relationship(back_populates="table", cascade="all, delete-orphan")
+    )
 
-    __table_args__ = (db.UniqueConstraint("name", "run__id"),)
+    _indexsets: db.AssociationProxy[list[IndexSet]] = db.association_proxy(
+        "_table_indexset_associations", "indexset"
+    )
+    _column_names: db.AssociationProxy[list[str | None]] = db.association_proxy(
+        "_table_indexset_associations", "column_name"
+    )
+
+    @property
+    def indexsets(self) -> list[str]:
+        return [indexset.name for indexset in self._indexsets]
+
+    @property
+    def column_names(self) -> list[str] | None:
+        return cast(list[str], self._column_names) if any(self._column_names) else None
+
+    _data: types.Mapped[list["TableData"]] = db.relationship(
+        back_populates="table", order_by="TableData.id"
+    )
+
+    @property
+    def data(self) -> dict[str, list[float] | list[int] | list[str]]:
+        if self._data == []:
+            return {}
+        else:
+            renames: dict[str, str] = {}
+            type_map: dict[str, str] = {}
+            if self.column_names:
+                for i in range(len(self.column_names)):
+                    renames[f"Column {i}"] = self.column_names[i]
+                    # would only be None if indexset had no data
+                    type_map[self.column_names[i]] = cast(
+                        Literal["float", "int", "str"], self._indexsets[i]._data_type
+                    )
+            else:
+                for i in range(len(self.indexsets)):
+                    renames[f"Column {i}"] = self.indexsets[i]
+                    type_map[self.indexsets[i]] = cast(
+                        Literal["float", "int", "str"], self._indexsets[i]._data_type
+                    )
+            return cast(
+                dict[str, list[float] | list[int] | list[str]],
+                pd.DataFrame.from_records(
+                    [
+                        {
+                            "Column 0": td.value_0,
+                            "Column 1": td.value_1,
+                            "Column 2": td.value_2,
+                            "Column 3": td.value_3,
+                            "Column 4": td.value_4,
+                            "Column 5": td.value_5,
+                            "Column 6": td.value_6,
+                            "Column 7": td.value_7,
+                            "Column 8": td.value_8,
+                            "Column 9": td.value_9,
+                            "Column 10": td.value_10,
+                            "Column 11": td.value_11,
+                            "Column 12": td.value_12,
+                            "Column 13": td.value_13,
+                            "Column 14": td.value_14,
+                        }
+                        for td in self._data
+                    ]
+                )
+                .dropna(axis="columns")
+                .rename(renames, axis="columns")
+                .astype(type_map)
+                .to_dict(orient="list"),
+            )
+
+    @data.setter
+    def data(
+        self, value: dict[str, list[float] | list[int] | list[str]] | pd.DataFrame
+    ) -> None:
+        return None
+
+
+class TableData(base.RootBaseModel):
+    table_prefix = "optimization_"
+
+    table: types.Mapped["Table"] = db.relationship(back_populates="_data")
+    table__id: types.TableId
+
+    value_0: types.String = db.Column(db.String, nullable=False)
+    value_1: types.String = db.Column(db.String, nullable=True)
+    value_2: types.String = db.Column(db.String, nullable=True)
+    value_3: types.String = db.Column(db.String, nullable=True)
+    value_4: types.String = db.Column(db.String, nullable=True)
+    value_5: types.String = db.Column(db.String, nullable=True)
+    value_6: types.String = db.Column(db.String, nullable=True)
+    value_7: types.String = db.Column(db.String, nullable=True)
+    value_8: types.String = db.Column(db.String, nullable=True)
+    value_9: types.String = db.Column(db.String, nullable=True)
+    value_10: types.String = db.Column(db.String, nullable=True)
+    value_11: types.String = db.Column(db.String, nullable=True)
+    value_12: types.String = db.Column(db.String, nullable=True)
+    value_13: types.String = db.Column(db.String, nullable=True)
+    value_14: types.String = db.Column(db.String, nullable=True)
+
+    __table_args__ = (
+        db.UniqueConstraint(
+            "table__id",
+            "value_0",
+            "value_1",
+            "value_2",
+            "value_3",
+            "value_4",
+            "value_5",
+            "value_6",
+            "value_7",
+            "value_8",
+            "value_9",
+            "value_10",
+            "value_11",
+            "value_12",
+            "value_13",
+            "value_14",
+        ),
+    )