Merge pull request #191 from brazil-data-cube/b-1.0

Merge v1.0.1 into master

CHANGES.rst

@@ -20,6 +20,14 @@
 Changes
 =======
 
+Version 1.0.1 (2022-12-06)
+--------------------------
+
+- Remove integration with lccs-db `#187 <https://github.com/brazil-data-cube/bdc-catalog/issues/187>`_.
+- Improve performance on items retrieval
+- Fix migration 0.8 - 1.0 related bands metadata ``null`` entry `#189 <https://github.com/brazil-data-cube/bdc-catalog/issues/1189>`_.
+
+
 Version 1.0.0 (2022-09-22)
 --------------------------
 

MANIFEST.in

@@ -38,3 +38,4 @@ recursive-include docs *.rst
 recursive-include docs Makefile
 recursive-include examples *.py
 recursive-include tests *.py
+recursive-include examples *.json

USAGE.rst

@@ -41,7 +41,6 @@ all definitions for ``BDC-Catalog``::
         bdc-db db init
         bdc-db db create-namespaces
         bdc-db db create-extension-postgis
-        lccs-db db create-extension-hstore
         bdc-db db create-schema
 
 
@@ -62,9 +61,6 @@ Creating database definition
         SQLALCHEMY_DATABASE_URI="postgresql://postgres:postgres@localhost:5432/bdcdb" \
         bdc-db db create-extension-postgis
 
-        SQLALCHEMY_DATABASE_URI="postgresql://postgres:postgres@localhost:5432/bdcdb" \
-        lccs-db db create-extension-hstore
-
 
 **2.** After that, run ``BDC-DB`` command to prepare the Brazil Data Cube data model::
 

bdc_catalog/alembic/5067fb4381c0_add_roles_system.py

@@ -44,10 +44,6 @@ def upgrade():
         schema='bdc'
     )
 
-    op.drop_constraint('collections_classification_system_id_class_systems_fkey', 'collections', schema='bdc', type_='foreignkey')
-    op.create_foreign_key(op.f('collections_classification_system_id_classification_systems_fkey'), 'collections', 'classification_systems', ['classification_system_id'], ['id'], source_schema='bdc', referent_schema='lccs', onupdate='CASCADE', ondelete='CASCADE')
-    op.drop_column('collections', 'is_public', schema='bdc')
-
     op.create_index(op.f('idx_bdc_quicklook_collection_id'), 'quicklook', ['collection_id'], unique=False, schema='bdc')
     # ### end Alembic commands ###
 
@@ -56,9 +52,6 @@ def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index(op.f('idx_bdc_quicklook_collection_id'), table_name='quicklook', schema='bdc')
 
-    op.drop_constraint(op.f('collections_classification_system_id_classification_systems_fkey'), 'collections', schema='bdc', type_='foreignkey')
-    op.create_foreign_key('collections_classification_system_id_class_systems_fkey', 'collections', 'classification_systems', ['classification_system_id'], ['id'], source_schema='bdc', referent_schema='lccs')
-
     op.drop_table('collections_roles', schema='bdc')
     op.drop_index(op.f('idx_bdc_roles_name'), table_name='roles', schema='bdc')
     op.drop_table('roles', schema='bdc')

bdc_catalog/alembic/c68b17b1860b_v0_8_0.py

@@ -12,21 +12,14 @@
 revision = 'c68b17b1860b'
 down_revision = '566a05da999d'
 branch_labels = ()
-depends_on = 'e8b12ba52665' # LCCS-DB reference id
 
 
+# Do not remove this migration.
+# This part is important to keep migration tree
+
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('collections', sa.Column('classification_system_id', sa.Integer(), nullable=True), schema='bdc')
-    op.create_foreign_key(op.f('collections_classification_system_id_class_systems_fkey'), 'collections', 'classification_systems', ['classification_system_id'], ['id'], source_schema='bdc', referent_schema='lccs', onupdate='CASCADE', ondelete='CASCADE')
-    op.create_index(op.f('idx_bdc_collections_classification_system_id'), 'collections', ['classification_system_id'],
-                    unique=False, schema='bdc')
-    # ### end Alembic commands ###
+    pass
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(op.f('idx_bdc_collections_classification_system_id'), table_name='collections', schema='bdc')
-    op.drop_constraint(op.f('collections_classification_system_id_classification_systems_fkey'), 'collections', schema='bdc', type_='foreignkey')
-    op.drop_column('collections', 'classification_system_id', schema='bdc')
-    # ### end Alembic commands ###
+    pass

bdc_catalog/alembic/d01f09b5dd8b_v1_0_0.py

@@ -17,7 +17,6 @@
 revision = 'd01f09b5dd8b'
 down_revision = 'c68b17b1860b'
 branch_labels = ()
-depends_on = '561ebe6266ad'  # LCCS-DB stable 0.8.1
 
 
 def upgrade():
@@ -56,7 +55,7 @@ def upgrade():
                                       comment='The value to sum in scale mult'))
 
         sql = """UPDATE bdc.bands 
-        SET metadata = coalesce(metadata::jsonb,'{}'::jsonb) || ('{\"eo\":{\"resolution_x\":'||resolution_x||',\"resolution_y\":'||resolution_y||',\"center_wavelength\":'||coalesce(center_wavelength, 0)||',\"full_width_half_max\": '||coalesce(full_width_half_max, 0)||'}}')::jsonb"""
+        SET metadata = CASE WHEN metadata::TEXT = 'null' THEN '{}'::jsonb ELSE coalesce(metadata::jsonb,'{}'::jsonb) || ('{\"eo\":{\"resolution_x\":'||resolution_x||',\"resolution_y\":'||resolution_y||',\"center_wavelength\":'||coalesce(center_wavelength, 0)||',\"full_width_half_max\": '||coalesce(full_width_half_max, 0)||'}}')::jsonb END"""
         bands_op.execute(sql)
         bands_op.drop_column('center_wavelength')
         bands_op.drop_column('full_width_half_max')

bdc_catalog/alembic/f3112636be24_remove_lccs_db.py

@@ -0,0 +1,79 @@
+"""remove lccs-db
+
+Revision ID: f3112636be24
+Revises: 561ebe6266ad
+Create Date: 2022-11-01 17:05:17.726004
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.engine.reflection import Inspector
+
+# revision identifiers, used by Alembic.
+revision = 'f3112636be24'
+down_revision = '5067fb4381c0'
+branch_labels = ()
+depends_on = None
+
+
+def _has_column(table_name: str, column: str, schema: str = None) -> bool:
+    bind = op.get_bind()
+    inspector = Inspector.from_engine(bind=bind)
+
+    found_column = False
+    for col in inspector.get_columns(table_name=table_name, schema=schema):
+        if column in col['name']:
+            found_column = True
+            break
+
+    return found_column
+
+
+def upgrade():
+    with op.batch_alter_table('collections', schema='bdc') as batch_op:
+        if _has_column(table_name='collections', schema='bdc', column='classification_system_id'):
+            batch_op.drop_column('classification_system_id')
+        if not _has_column(table_name='collections', schema='bdc', column='is_public'):
+            batch_op.add_column(sa.Column('is_public', sa.Boolean(), nullable=False, server_default='true'))
+            batch_op.create_index(op.f('idx_bdc_collections_is_public'), ['is_public'], unique=False)
+
+    op.create_index('idx_bdc_items_start_date_desc_id', 'items', [sa.text('start_date DESC'), 'id'], unique=False,
+                    schema='bdc')
+    op.create_index('idx_bdc_items_start_date_desc_id_is_available', 'items',
+                    [sa.text('start_date DESC'), 'id', 'is_available'], unique=False,
+                    schema='bdc')
+    op.drop_table('collections_roles', schema='bdc')
+    op.drop_table('roles', schema='bdc')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    op.create_table(
+        'roles',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=64), nullable=False),
+        sa.Column('description', sa.Text(), nullable=True),
+        sa.Column('created', sa.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
+        sa.Column('updated', sa.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
+        sa.PrimaryKeyConstraint('id', name=op.f('roles_pkey')),
+        sa.UniqueConstraint('name', name=op.f('roles_name_key')),
+        schema='bdc'
+    )
+    op.create_index(op.f('idx_bdc_roles_name'), 'roles', ['name'], unique=False, schema='bdc')
+    op.create_table(
+        'collections_roles',
+        sa.Column('collection_id', sa.Integer(), nullable=False),
+        sa.Column('role_id', sa.Integer(), nullable=False),
+        sa.Column('created', sa.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
+        sa.Column('updated', sa.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
+        sa.ForeignKeyConstraint(['collection_id'], ['bdc.collections.id'],
+                                name=op.f('collections_roles_collection_id_collections_fkey'), onupdate='CASCADE',
+                                ondelete='CASCADE'),
+        sa.ForeignKeyConstraint(['role_id'], ['bdc.roles.id'], name=op.f('collections_roles_role_id_roles_fkey'),
+                                onupdate='CASCADE', ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('collection_id', 'role_id', name=op.f('collections_roles_pkey')),
+        schema='bdc'
+    )
+    op.drop_index('idx_bdc_items_start_date_desc_id', table_name='items', schema='bdc')
+    op.drop_index('idx_bdc_items_start_date_desc_id_is_available', table_name='items', schema='bdc')

bdc_catalog/cli.py

@@ -72,14 +72,16 @@ def load_data(ifile: str, from_dir: str, verbose: bool):
                           'Make sure to install with "pip install shapely" or '
                           '"pip install -e .[geo]".')
 
+    if not ifile and not from_dir:
+        raise click.MissingParameter("Missing --ifile or --from-dir parameter.")
+
     entries = []
+
     if ifile:
         entries.append(Path(ifile))
     elif from_dir:
         for entry in Path(from_dir).glob('*.json'):
             entries.append(entry)
-    else:
-        raise click.MissingParameter("Missing --ifile or --from-dir parameter.")
 
     for entry in entries:
         with entry.open() as fd:

bdc_catalog/ext.py

@@ -83,7 +83,7 @@ def __init__(self, app=None):
         if app:
             self.init_app(app)
 
-    def init_app(self, app: Flask):
+    def init_app(self, app: Flask, **kwargs):
         """Initialize Flask application instance.
 
         Args:

bdc_catalog/models/__init__.py

@@ -20,8 +20,7 @@
 
 from .band import Band, BandSRC
 from .base_sql import db
-from .collection import (Collection, CollectionRole, CollectionsProviders,
-                         CollectionSRC)
+from .collection import Collection, CollectionsProviders, CollectionSRC
 from .composite_function import CompositeFunction
 from .grid_ref_sys import GridRefSys
 from .item import Item, ItemsProcessors, SpatialRefSys
@@ -30,7 +29,6 @@
 from .provider import Provider
 from .quicklook import Quicklook
 from .resolution_unit import ResolutionUnit
-from .role import Role
 from .tile import Tile
 from .timeline import Timeline
 
@@ -39,7 +37,6 @@
     'Band',
     'BandSRC',
     'Collection',
-    'CollectionRole',
     'CollectionSRC',
     'CollectionsProviders',
     'CompositeFunction',
@@ -51,7 +48,6 @@
     'Provider',
     'Quicklook',
     'ResolutionUnit',
-    'Role',
     'SpatialRefSys',
     'Tile',
     'Timeline',

bdc_catalog/models/collection.py

@@ -22,7 +22,6 @@
 
 from bdc_db.sqltypes import JSONB
 from geoalchemy2 import Geometry
-from lccs_db.models import LucClassificationSystem
 from sqlalchemy import (ARRAY, TIMESTAMP, Boolean, Column, Enum, ForeignKey,
                         Index, Integer, PrimaryKeyConstraint, String, Text,
                         UniqueConstraint)
@@ -33,7 +32,6 @@
 from ..config import BDC_CATALOG_SCHEMA
 from .base_sql import BaseModel, db
 from .provider import Provider
-from .role import Role
 
 name_collection_type = 'collection_type'
 options_collection_type = ('cube', 'collection', 'classification', 'mosaic')
@@ -57,7 +55,6 @@ class Collection(BaseModel):
         ForeignKey(f'{BDC_CATALOG_SCHEMA}.composite_functions.id', onupdate='CASCADE', ondelete='CASCADE'),
         comment='Function schema identifier. Used for data cubes.')
     grid_ref_sys_id = Column(ForeignKey(f'{BDC_CATALOG_SCHEMA}.grid_ref_sys.id', onupdate='CASCADE', ondelete='CASCADE'))
-    classification_system_id = Column(ForeignKey(LucClassificationSystem.id, onupdate='CASCADE', ondelete='CASCADE'))
     collection_type = Column(enum_collection_type, nullable=False)
     metadata_ = Column('metadata', JSONB('bdc-catalog/collection-metadata.json'),
                        comment='Follow the JSONSchema @jsonschemas/collection-metadata.json')
@@ -69,6 +66,7 @@ class Collection(BaseModel):
     item_assets = Column('item_assets', JSONB('bdc-catalog/collection-item-assets.json'),
                          comment='Contains the STAC Extension Item Assets.')
     is_available = Column(Boolean(), nullable=False, default=False, server_default='False')
+    is_public = Column(Boolean(), nullable=False, default=True, server_default='true')
     category = Column(enum_collection_category, nullable=False)
     start_date = Column(TIMESTAMP(timezone=True))
     end_date = Column(TIMESTAMP(timezone=True))
@@ -82,17 +80,15 @@ class Collection(BaseModel):
     bands = relationship('Band', back_populates='collection')
     quicklook = relationship('Quicklook')
     timeline = relationship('Timeline')
-    # Joined Eager Loading. Default is Left Outer Join to lead object that does not refer to a related row.
-    classification_system = relationship('LucClassificationSystem', lazy='joined')
 
     __table_args__ = (
         UniqueConstraint('name', 'version'),
         Index(None, grid_ref_sys_id),
         Index(None, name),
         Index(None, spatial_extent, postgresql_using='gist'),
-        Index(None, classification_system_id),
         Index(None, category),
         Index(None, is_available),
+        Index(None, is_public),
         Index(None, start_date, end_date),
         dict(schema=BDC_CATALOG_SCHEMA),
     )
@@ -143,6 +139,44 @@ def identifier(self):
         """
         return func.concat(self.name, '-', self.version)
 
+    @classmethod
+    def get_collection_sources(cls, collection: Union['Collection', str, int]) -> List['Collection']:
+        """Trace data cube collection origin.
+
+        It traces all the collection origin from the given collection using
+        :class:`bdc_catalog.models.CollectionSRC`
+
+        Raises:
+            ValueError: When collection is related itself (cyclic relationship).
+        """
+        out = []
+        dupes = []
+        ref = collection
+        if not isinstance(collection, Collection):
+            ref = Collection.get_by_id(collection)
+
+        while ref is not None:
+            source: CollectionSRC = (
+                CollectionSRC.query()
+                .filter(CollectionSRC.collection_id == ref.id)
+                .first()
+            )
+            if source is None:
+                break
+
+            ref: Collection = Collection.query().get(source.collection_src_id)
+            if ref.id in dupes:
+                raise ValueError(f'Collection {ref.identifier} has self reference')
+
+            dupes.append(ref.id)
+            out.append(ref)
+        return out
+
+    @property
+    def sources(self) -> List['Collection']:
+        """Retrieve the list of referred collections marked as origin."""
+        return Collection.get_collection_sources(self)
+
 
 class CollectionSRC(BaseModel):
     """Model for collection provenance/lineage."""
@@ -223,25 +257,3 @@ def to_dict(self) -> dict:
         """
         return dict(name=self.provider.name, description=self.provider.description,
                     url=self.provider.url, roles=self.roles)
-
-
-class CollectionRole(BaseModel):
-    """Model to represent the link between Collection and Role."""
-
-    __tablename__ = 'collections_roles'
-
-    collection_id = db.Column('collection_id', db.Integer(),
-                              db.ForeignKey(Collection.id, onupdate='CASCADE', ondelete='CASCADE'),
-                              nullable=False)
-
-    role_id = db.Column('role_id', db.Integer(),
-                        db.ForeignKey(Role.id, onupdate='CASCADE', ondelete='CASCADE'),
-                        nullable=False)
-
-    collection = relationship('Collection', lazy='joined', foreign_keys=[collection_id])
-    role = relationship('Role', lazy='joined')
-
-    __table_args__ = (
-        PrimaryKeyConstraint(collection_id, role_id),
-        dict(schema=BDC_CATALOG_SCHEMA),
-    )