fixup

polars_db_schema/README.rst

@@ -10,12 +10,12 @@ Polars Db Schema
    !! source digest: sha256:0e6c26c886f0fc18c585b743314981ec58966dd913b8a469530e62e313327de2
    !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png
+.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png
     :target: https://odoo-community.org/page/development-status
-    :alt: Beta
-.. |badge2| image:: https://img.shields.io/badge/licence-GPL--3-blue.png
-    :target: http://www.gnu.org/licenses/gpl-3.0-standalone.html
-    :alt: License: GPL-3
+    :alt: Alpha
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github
     :target: https://github.com/OCA/server-backend/tree/18.0/polars_db_schema
     :alt: OCA/server-backend
@@ -35,6 +35,11 @@ Introspect external database
 Use case: you want discover an external database extracting data from
 only relevant tables and columns
 
+.. IMPORTANT::
+   This is an alpha version, the data model and design can change at any time without warning.
+   Only for development or testing purpose, do not use in production.
+   `More details on development status <https://odoo-community.org/page/development-status>`_
+
 **Table of contents**
 
 .. contents::

polars_db_schema/__manifest__.py

@@ -2,7 +2,9 @@
     "name": "Polars Db Schema",
     "version": "18.0.1.0.0",
     "author": "Akretion, Odoo Community Association (OCA)",
-    "license": "GPL-3",
+    "development_status": "Alpha",
+    "website": "https://github.com/OCA/server-backend",
+    "license": "AGPL-3",
     "depends": [
         "polars_db_process",
     ],
@@ -14,5 +16,8 @@
         "views/db_type.xml",
         "data/db_type.xml",
     ],
+    "demo": [
+        "data/demo.xml",
+    ],
     "installable": True,
 }

polars_db_schema/data/db_type.xml

@@ -23,10 +23,22 @@ ORDER BY row_count desc
     <record id="sqlite_db_type" model="db.type">
         <field name="name">Sqlite</field>
         <field name="code">sqlite</field>
-        <field name="excluded_types" />
+        <field name="excluded_types">BLOB</field>
         <field name="row_count_query">
 -- This query order tables by rows count
 SELECT tbl, stat FROM sqlite_stat1
         </field>
     </record>
+
+    <record id="postgres_db_type" model="db.type">
+        <field name="name">PostgreSQL</field>
+        <field name="code">postgresql</field>
+        <field name="excluded_types">bytea</field>
+        <field name="row_count_query">
+-- This query order tables by rows count
+SELECT relname AS name, n_live_tup AS row_count
+  FROM pg_stat_user_tables
+ORDER BY row_count DESC;
+        </field>
+    </record>
 </odoo>

polars_db_schema/data/db_config.xml → polars_db_schema/data/demo.xml

@@ -1,5 +1,5 @@
 <odoo>
-    <record id="sqlite_chinook" model="database.config">
+    <record id="sqlite_chinook" model="db.config">
         <field name="name">Chinook</field>
         <field
             name="string_connexion"

polars_db_schema/models/__init__.py

@@ -1,3 +1,4 @@
 from . import db_config
 from . import db_type
 from . import db_table
+from . import db_source

polars_db_schema/models/db_config.py

@@ -1,7 +1,8 @@
 import connectorx as cx
 import polars as pl
 
-from odoo import exceptions, fields, models
+from odoo.modules.module import get_module_path
+from odoo import _, exceptions, fields, models
 
 
 class DbConfig(models.Model):
@@ -11,35 +12,35 @@ class DbConfig(models.Model):
     db_table_ids = fields.One2many(comodel_name="db.table", inverse_name="db_config_id")
     row_count_query = fields.Text(related="db_type_id.row_count_query")
 
-    def get_metadata(self):
+    def get_db_metadata(self):
         self.ensure_one()
         connexion = self._get_connexion()
         if self.row_count_query:
-            self.read_sql(connexion, "SELECT 1")
-            df = self.read_sql(connexion, self.row_count_query)
+            self._read_sql(connexion, "SELECT 1")
+            df = self._read_sql(connexion, self.row_count_query)
             if self.db_type_id.code == "sqlite":
                 # https://docs.pola.rs/user-guide/expressions/user-defined-functions/#processing-individual-values-with-map_elements
-                df = df.with_columns(
-                    pl.col("stat").map_elements(sqlite, return_dtype=pl.Int32)
+                df = (
+                    df.with_columns(
+                        pl.col("stat").map_elements(sqlite, return_dtype=pl.Int32)
+                    )
+                    # rename columns
+                    .rename({"tbl": "name", "stat": "row_count"})
+                    # stat columns store extra info leading to duplicate lines,
+                    # then make it unique
+                    .unique(maintain_order=True)
                 )
-                df = df.rename({"tbl": "name", "stat": "row_count"})
-                df = df.unique(maintain_order=True)
-                # stat columns store extra info leading to duplicate lines,
-                # then make it unique
             df = df.filter(pl.col("row_count") > 0).with_columns(
+                # add m2o foreign key
                 db_config_id=pl.lit(self.id)
             )
+            df = self._filter_df(df)
             self.env["db.table"].search([("db_config_id", "=", self.id)]).unlink()
             self.env["db.table"].create(df.to_dicts())
 
-    def read_sql(self, connexion, query):
-        try:
-            return cx.read_sql(connexion, query, return_type="polars")
-        except RuntimeError as err:
-            raise exceptions.ValidationError(err) from err
-        except Exception as err:
-            raise exceptions.ValidationError(err) from err
-
+    def _filter_df(self, df):
+        "You may want ignore some tables: inherit me"
+        return df
 
 def sqlite(value):
     "Extract row_count info from 'stat' column"

polars_db_schema/models/db_source.py

@@ -0,0 +1,19 @@
+from odoo import models
+from odoo.modules.module import get_module_path
+from pathlib import Path
+
+
+class DfSource(models.Model):
+    _inherit = "df.source"
+
+    # def _get_test_file_paths(self):
+    #     res = super()._get_test_file_paths()
+    #     res.update(
+    #         {
+    #             "polars_db_schema": {
+    #                 "relative_path": "tests/files",
+    #                 "xmlid": "migr.contact",
+    #             }
+    #         }
+    #     )
+    #     return res

polars_db_schema/models/db_table.py

@@ -4,7 +4,7 @@
 import connectorx as cx
 import polars as pl
 
-from odoo import fields, models
+from odoo import _, exceptions, fields, models
 
 
 class DbTable(models.Model):
@@ -27,28 +27,6 @@ class DbTable(models.Model):
         "because they're probably unused by the application",
     )
 
-    def get_spreadsheet(self):
-        self.ensure_one()
-        if not self.sql:
-            self.get_metadata_info()
-        df = cx.read_sql(
-            self.db_config_id._get_connexion(), self.sql, return_type="polars"
-        )
-        excel_stream = io.BytesIO()
-        df.write_excel(
-            workbook=excel_stream,
-            position="B1",
-            table_style="Table Style Light 16",
-            dtype_formats={pl.Date: "mm/dd/yyyy"},
-            float_precision=6,
-            header_format={"bold": True, "font_color": "#702963"},
-            freeze_panes="A2",
-            autofit=True,
-        )
-        excel_stream.seek(0)
-        self.filename = f"{self.name}.xlsx"
-        self.xlsx = base64.encodebytes(excel_stream.read())
-
     def get_metadata_info(self):
         self.ensure_one()
         query = f"SELECT * FROM {self.name}"
@@ -72,4 +50,42 @@ def get_metadata_info(self):
                 # column has the same value whatever row
                 uniques[col] = res.to_series()[0]
         self.uniques = f"{uniques}"
-        self.sql = f"SELECT {', '. join(new_cols)}\nFROM {self.name};\n"
+        if new_cols:
+            self.sql = f"SELECT {', '. join(new_cols)}\nFROM {self.name};\n"
+
+    # WARNING Thread <Thread(odoo.service.http.request.129007460812352,
+    # started 129007460812352)> virtual real time limit (151/120s) reached.
+    # Dumping stacktrace of limit exceeding threads before reloading
+
+    def get_spreadsheet(self):
+        self.ensure_one()
+        if not self.sql:
+            self.get_metadata_info()
+        if not self.sql:
+            raise exceptions.ValidationError(
+                _(
+                    "There is no column with varaiable data in this table: "
+                    "check Uniques Values column"
+                )
+            )
+        df = cx.read_sql(
+            self.db_config_id._get_connexion(), self.sql, return_type="polars"
+        )
+        excel_stream = io.BytesIO()
+        vals = {"workbook": excel_stream}
+        vals.update(self.get_spreadsheet_settings())
+        df.write_excel(**vals)
+        excel_stream.seek(0)
+        self.filename = f"{self.name}.xlsx"
+        self.xlsx = base64.encodebytes(excel_stream.read())
+
+    def get_spreadsheet_settings(self):
+        return {
+            "position": "A1",
+            "table_style": "Table Style Light 16",
+            "dtype_formats": {pl.Date: "dd/mm/yyyy"},
+            "float_precision": 6,
+            "header_format": {"bold": True, "font_color": "#702963"},
+            "freeze_panes": "A2",
+            "autofit": True,
+        }

polars_db_schema/models/db_type.py

@@ -11,5 +11,5 @@ class DbType(models.Model):
         required=True, help="SQL code to find how many records contains each table"
     )
     excluded_types = fields.Text(
-        help="Database column type to ignore for introspection"
+        help="Column types to ignore for better introspection (set 1 data by line)"
     )

polars_db_schema/static/description/index.html

@@ -369,11 +369,17 @@ <h1 class="title">Polars Db Schema</h1>
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 !! source digest: sha256:0e6c26c886f0fc18c585b743314981ec58966dd913b8a469530e62e313327de2
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->
-<p><a class="reference external image-reference" href="https://odoo-community.org/page/development-status"><img alt="Beta" src="https://img.shields.io/badge/maturity-Beta-yellow.png" /></a> <a class="reference external image-reference" href="http://www.gnu.org/licenses/gpl-3.0-standalone.html"><img alt="License: GPL-3" src="https://img.shields.io/badge/licence-GPL--3-blue.png" /></a> <a class="reference external image-reference" href="https://github.com/OCA/server-backend/tree/18.0/polars_db_schema"><img alt="OCA/server-backend" src="https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github" /></a> <a class="reference external image-reference" href="https://translation.odoo-community.org/projects/server-backend-18-0/server-backend-18-0-polars_db_schema"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external image-reference" href="https://runboat.odoo-community.org/builds?repo=OCA/server-backend&amp;target_branch=18.0"><img alt="Try me on Runboat" src="https://img.shields.io/badge/runboat-Try%20me-875A7B.png" /></a></p>
+<p><a class="reference external image-reference" href="https://odoo-community.org/page/development-status"><img alt="Alpha" src="https://img.shields.io/badge/maturity-Alpha-red.png" /></a> <a class="reference external image-reference" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/licence-AGPL--3-blue.png" /></a> <a class="reference external image-reference" href="https://github.com/OCA/server-backend/tree/18.0/polars_db_schema"><img alt="OCA/server-backend" src="https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github" /></a> <a class="reference external image-reference" href="https://translation.odoo-community.org/projects/server-backend-18-0/server-backend-18-0-polars_db_schema"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external image-reference" href="https://runboat.odoo-community.org/builds?repo=OCA/server-backend&amp;target_branch=18.0"><img alt="Try me on Runboat" src="https://img.shields.io/badge/runboat-Try%20me-875A7B.png" /></a></p>
 <p>Introspect external database</p>
 <img alt="https://raw.githubusercontent.com/OCA/server-backend/18.0/polars_db_schema/static/description/figure1.png" src="https://raw.githubusercontent.com/OCA/server-backend/18.0/polars_db_schema/static/description/figure1.png" />
 <p>Use case: you want discover an external database extracting data from
 only relevant tables and columns</p>
+<div class="admonition important">
+<p class="first admonition-title">Important</p>
+<p class="last">This is an alpha version, the data model and design can change at any time without warning.
+Only for development or testing purpose, do not use in production.
+<a class="reference external" href="https://odoo-community.org/page/development-status">More details on development status</a></p>
+</div>
 <p><strong>Table of contents</strong></p>
 <div class="contents local topic" id="contents">
 <ul class="simple">

polars_db_schema/views/db_config.xml

@@ -6,7 +6,7 @@
             <xpath expr="//header" position="inside">
                 <button
                     type="object"
-                    name="get_metadata"
+                    name="get_db_metadata"
                     string="Metadata"
                     invisible="not row_count_query"
                 />

polars_db_schema/views/db_type.xml

@@ -3,8 +3,8 @@
         <field name="model">db.type</field>
         <field name="arch" type="xml">
             <form>
+                <header />
                 <sheet>
-                    <header />
                     <group>
                         <group>
                             <field name="name" readonly="1" />
@@ -28,6 +28,8 @@
         <field name="arch" type="xml">
             <list>
                 <field name="name" />
+                <field name="code" />
+                <field name="excluded_types" />
             </list>
         </field>
     </record>
@@ -43,6 +45,6 @@
         id="db_type_menu"
         action="db_type_action"
         parent="polars_process.polars_menu"
-        sequence="9"
+        sequence="8"
     />
 </odoo>