fix: fix db error (#125)

* fix: fix OutOfRangeError

Some product quantity triggered an OutOfRangeError in PostgreSQL
We now set the quantity to None if it's above a threshold

* fix: set unique_scans_n default to 0

alembic/versions/20240109_1330_1bf4306fff2f_set_unique_scans_n_default_to_0.py

@@ -0,0 +1,43 @@
+"""set unique_scans_n default to 0
+
+Revision ID: 1bf4306fff2f
+Revises: 3f8d293e669e
+Create Date: 2024-01-09 13:30:47.327891
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "1bf4306fff2f"
+down_revision: Union[str, None] = "3f8d293e669e"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("UPDATE products SET unique_scans_n = 0 WHERE unique_scans_n IS NULL;")
+    op.alter_column(
+        "products",
+        "unique_scans_n",
+        existing_type=sa.INTEGER(),
+        nullable=False,
+        server_default="0",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "products",
+        "unique_scans_n",
+        existing_type=sa.INTEGER(),
+        nullable=True,
+        server_default=None,
+    )
+    # ### end Alembic commands ###

app/models.py

@@ -44,7 +44,7 @@ class Product(Base):
     product_quantity = Column(Integer)
     brands = Column(String)
     image_url = Column(String)
-    unique_scans_n = Column(Integer, nullable=True)
+    unique_scans_n = Column(Integer, nullable=False, server_default=0)
 
     prices: Mapped[list["Price"]] = relationship(back_populates="product")
 

app/tasks.py

@@ -142,6 +142,12 @@ def import_product_db(db: Session, batch_size: int = 1000):
             item = {"code": product_code, "source": Flavor.off}
             for key in OFF_FIELDS:
                 item[key] = product[key] if key in product else None
+
+            if product.get("product_quantity", 0) >= 100_000:
+                # If the product quantity is too high, it's probably an
+                # error, and causes an OutOfRangeError in the database
+                product["product_quantity"] = None
+
             item["image_url"] = generate_main_image_url(
                 product_code, images, product["lang"]
             )

app/utils.py

@@ -54,19 +54,22 @@ def openfoodfacts_product_search(code: str):
 
 
 def fetch_product_openfoodfacts_details(product: ProductBase):
-    product_openfoodfacts_details = dict()
+    product = {}
     try:
         response = openfoodfacts_product_search(code=product.code)
         if response["status"]:
-            product_openfoodfacts_details["source"] = Flavor.off
+            product["source"] = Flavor.off
             for off_field in OFF_FIELDS:
                 if off_field in response["product"]:
-                    product_openfoodfacts_details[off_field] = response["product"][
-                        off_field
-                    ]
-        return product_openfoodfacts_details
+                    product[off_field] = response["product"][off_field]
+            if product.get("product_quantity", 0) >= 100_000:
+                # If the product quantity is too high, it's probably an
+                # error, and cause an OutOfRangeError in the database
+                product["product_quantity"] = None
+
+        return product
     except Exception:
-        logger.exception("Error returned from OpenFoodFacts")
+        logger.exception("Error returned from Open Food Facts")
         return