altering tables sqlite-pg-mysql

dataloom/loom/__init__.py

@@ -1158,6 +1158,7 @@ def _execute_sql(
         affected_rows: bool = False,
         operation: Optional[str] = None,
         _verbose: int = 1,
+        _is_script: bool = False,
     ) -> Any:
         return self.sql_obj.execute_sql(
             sql=sql,
@@ -1170,6 +1171,7 @@ def _execute_sql(
             fetchmany=fetchmany,
             affected_rows=affected_rows,
             _verbose=_verbose,
+            _is_script=_is_script,
         )
 
     def connect(
@@ -1435,7 +1437,7 @@ def sync(
                         sql = model._alter_sql(
                             dialect=self.dialect, old_columns=old_columns
                         )
-                        self._execute_sql(sql)
+                        self._execute_sql(sql, _is_script=self.dialect == "sqlite")
                     else:
                         for sql in model._create_sql(dialect=self.dialect):
                             if sql is not None:

dataloom/loom/sql.py

@@ -61,6 +61,7 @@ def execute_sql(
         affected_rows: bool = False,
         operation: Optional[str] = None,
         _verbose: int = 1,
+        _is_script: bool = False,
     ) -> Any:
         # do we need to log the executed SQL?
         if self.sql_logger is not None and _verbose > 0:
@@ -128,13 +129,16 @@ def execute_sql(
                     result = cursor.lastrowid
         elif self.dialect == "sqlite":
             cursor = self.conn.cursor()
-            if args is None:
-                cursor.execute(sql)
+            if _is_script:
+                cursor.executescript(sql)
             else:
-                if bulk:
-                    cursor.executemany(sql, args)
+                if args is None:
+                    cursor.execute(sql)
                 else:
-                    cursor.execute(sql, args)
+                    if bulk:
+                        cursor.executemany(sql, args)
+                    else:
+                        cursor.execute(sql, args)
             # options
             if bulk or affected_rows:
                 result = cursor.rowcount

dataloom/statements/__init__.py

@@ -18,7 +18,7 @@
     get_relationships,
     get_create_table_params,
     get_table_fields,
-    get_alter_table_params,
+    AlterTable,
 )
 
 
@@ -150,7 +150,6 @@ def _get_tables_command(self) -> Optional[str]:
     @property
     def _get_create_table_command(self) -> Optional[list[str]]:
         # is the primary key defined in this table?
-        _, pk_name, _, _ = get_table_fields(model=self.model, dialect=self.dialect)
         pks, user_fields, predefined_fields = get_create_table_params(
             dialect=self.dialect,
             model=self.model,
@@ -834,12 +833,19 @@ def _get_alter_table_command(self, old_columns: list[str]) -> str:
         2. check if is new column
         3. check if the column has been removed
         """
-        _, pk_name, _, _ = get_table_fields(model=self.model, dialect=self.dialect)
-        pks, alterations = get_alter_table_params(
+        pks, alterations = AlterTable(
             dialect=self.dialect, model=self.model, old_columns=old_columns
-        )
+        ).get_alter_table_params
 
-        alterations = ", ".join(alterations)
+        alterations = (
+            (
+                ", ".join(alterations)
+                if self.dialect == "mysql"
+                else " ".join(alterations)
+            )
+            if self.dialect != "sqlite"
+            else ""
+        )
         # do we have a single primary key or not?
         if len(pks) == 0:
             raise PkNotDefinedException(
@@ -850,18 +856,43 @@ def _get_alter_table_command(self, old_columns: list[str]) -> str:
                 f"You have defined many field as primary keys which is not allowed. Fields ({', '.join(pks)}) are primary keys."
             )
         if self.dialect == "postgres":
-            sql = PgStatements.ALTER_TABLE_COMMAND.format(
-                table_name=f'"{self.table_name}"', alterations=alterations
-            )
+            sql = PgStatements.ALTER_TABLE_COMMAND.format(alterations=alterations)
         elif self.dialect == "mysql":
             sql = MySqlStatements.ALTER_TABLE_COMMAND.format(
                 table_name=f"`{self.table_name}`", alterations=alterations
             )
 
         elif self.dialect == "sqlite":
+            old_table_name = f"`{self.table_name}`"
+            columns, _, _, _ = get_table_fields(self.model, dialect=self.dialect)
+            new_table_name = f"`{self.table_name}_new`"
+            pks, user_fields, predefined_fields = get_create_table_params(
+                dialect=self.dialect,
+                model=self.model,
+            )
+            if len(pks) == 0:
+                raise PkNotDefinedException(
+                    "Your table does not have a primary key column."
+                )
+            if len(pks) > 1:
+                raise TooManyPkException(
+                    f"You have defined many field as primary keys which is not allowed. Fields ({', '.join(pks)}) are primary keys."
+                )
+            fields = [*user_fields, *predefined_fields]
+            fields_name = ", ".join(f for f in [" ".join(field) for field in fields])
+            create_command = Sqlite3Statements.CREATE_NEW_TABLE_IF_NOT_EXITS.format(
+                table_name=new_table_name, fields_name=fields_name
+            )
+
             sql = Sqlite3Statements.ALTER_TABLE_COMMAND.format(
-                table_name=f"`{self.table_name}`", alterations=alterations
+                create_new_table_command=create_command,
+                new_table_name=new_table_name,
+                old_table_name=old_table_name,
+                new_table_columns=", ".join(
+                    [f"`{col}`" for col in old_columns if col in columns]
+                ),
             )
+
         else:
             raise UnsupportedDialectException(
                 "The dialect passed is not supported the supported dialects are: {'postgres', 'mysql', 'sqlite'}"

dataloom/statements/statements.py

@@ -146,7 +146,25 @@ class Sqlite3Statements:
     # Altering tables
 
     ALTER_TABLE_COMMAND = """
-    ALTER TABLE {table_name} {alterations};
+    -- Begin a transaction
+    BEGIN TRANSACTION;
+    
+    -- Create a new table with the desired schema
+    {create_new_table_command}
+
+    -- Copy data from the old table to the new one
+    INSERT INTO {new_table_name} ({new_table_columns})
+    SELECT {new_table_columns}
+    FROM {old_table_name};
+
+    -- Drop the old table
+    DROP TABLE {old_table_name};
+
+    -- Rename the new table to the original table name
+    ALTER TABLE {new_table_name} RENAME TO {old_table_name};
+    
+    -- Commit the transaction
+    COMMIT;
     """
     # describing table
 
@@ -268,7 +286,7 @@ class PgStatements:
     # Altering tables
 
     ALTER_TABLE_COMMAND = """
-    ALTER TABLE {table_name} {alterations};
+    {alterations}
     """
     # describing table
     DESCRIBE_TABLE_COMMAND = """

dataloom/utils/__init__.py

@@ -2,7 +2,7 @@
 
 from dataloom.utils.logger import console_logger, file_logger
 from dataloom.utils.create_table import get_create_table_params
-from dataloom.utils.alter_table import get_alter_table_params
+from dataloom.utils.alter_table import AlterTable
 from dataloom.utils.aggregations import get_groups
 from dataloom.utils.helpers import is_collection
 from dataloom.utils.tables import (
@@ -163,5 +163,5 @@ def get_formatted_query(
     print_pretty_table,
     is_collection,
     get_groups,
-    get_alter_table_params,
+    AlterTable,
 ]