From 35621dc61bef016a68307628218945c8071f2e5d Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 25 Aug 2021 09:34:13 +0200 Subject: [PATCH] Add example index with `varchar_pattern_ops` --- .../versions/1de112340b17_django_parity_2.py | 15 +++ .../versions/2b40c8131fe0_django_parity_1.py | 91 +++++++++---------- aiida/backends/sqlalchemy/models/authinfo.py | 4 +- aiida/backends/sqlalchemy/models/comment.py | 8 +- aiida/backends/sqlalchemy/models/group.py | 6 +- aiida/backends/sqlalchemy/models/log.py | 6 +- aiida/backends/sqlalchemy/models/node.py | 14 ++- aiida/backends/sqlalchemy/models/user.py | 6 +- .../aiida_sqlalchemy/test_migrations.py | 8 +- 9 files changed, 95 insertions(+), 63 deletions(-) diff --git a/aiida/backends/sqlalchemy/migrations/versions/1de112340b17_django_parity_2.py b/aiida/backends/sqlalchemy/migrations/versions/1de112340b17_django_parity_2.py index 67ca1ca0f0..6c80120928 100644 --- a/aiida/backends/sqlalchemy/migrations/versions/1de112340b17_django_parity_2.py +++ b/aiida/backends/sqlalchemy/migrations/versions/1de112340b17_django_parity_2.py @@ -79,6 +79,14 @@ def upgrade(): op.alter_column('db_dbuser', 'last_name', existing_type=sa.String(254), nullable=False) op.alter_column('db_dbuser', 'institution', existing_type=sa.String(254), nullable=False) + op.create_index( + 'db_dbnode_node_type_like', + 'db_dbnode', ['node_type'], + unique=False, + postgresql_using='btree', + postgresql_ops={'data': 'varchar_pattern_ops'} + ) + def downgrade(): op.alter_column('db_dbuser', 'institution', existing_type=sa.String(254), nullable=True) @@ -130,3 +138,10 @@ def downgrade(): op.alter_column('db_dbauthinfo', 'enabled', existing_type=sa.BOOLEAN(), nullable=True) op.alter_column('db_dbauthinfo', 'auth_params', existing_type=JSONB, nullable=True) op.alter_column('db_dbauthinfo', 'metadata', existing_type=JSONB, nullable=True) + + op.drop_index( + 'db_dbnode_node_type_like', + table_name='db_dbnode', + postgresql_using='btree', + postgresql_ops={'data': 'varchar_pattern_ops'} + ) diff --git a/aiida/backends/sqlalchemy/migrations/versions/2b40c8131fe0_django_parity_1.py b/aiida/backends/sqlalchemy/migrations/versions/2b40c8131fe0_django_parity_1.py index 4cbc056942..4d2addb629 100644 --- a/aiida/backends/sqlalchemy/migrations/versions/2b40c8131fe0_django_parity_1.py +++ b/aiida/backends/sqlalchemy/migrations/versions/2b40c8131fe0_django_parity_1.py @@ -33,7 +33,7 @@ def upgrade(): db_dbauthinfo = sa.sql.table( - "db_dbauthinfo", + 'db_dbauthinfo', # sa.Column('aiidauser_id', sa.Integer), # sa.Column('dbcomputer_id', sa.Integer), sa.Column('enabled', sa.Boolean), @@ -41,12 +41,12 @@ def upgrade(): sa.Column('metadata', JSONB), ) - op.execute(db_dbauthinfo.update().where(db_dbauthinfo.c.enabled.is_(None)).values(enabled = True)) - op.execute(db_dbauthinfo.update().where(db_dbauthinfo.c.auth_params == JSONB.NULL).values(auth_params = {})) - op.execute(db_dbauthinfo.update().where(db_dbauthinfo.c.metadata == JSONB.NULL).values(metadata = {})) + op.execute(db_dbauthinfo.update().where(db_dbauthinfo.c.enabled.is_(None)).values(enabled=True)) + op.execute(db_dbauthinfo.update().where(db_dbauthinfo.c.auth_params == JSONB.NULL).values(auth_params={})) + op.execute(db_dbauthinfo.update().where(db_dbauthinfo.c.metadata == JSONB.NULL).values(metadata={})) db_dbcomment = sa.sql.table( - "db_dbcomment", + 'db_dbcomment', # sa.Column('dbnode_id', sa.Integer), # sa.Column('user_id', sa.Integer), sa.Column('content', sa.Text), @@ -55,13 +55,13 @@ def upgrade(): sa.Column('uuid', UUID(as_uuid=True)), ) - op.execute(db_dbcomment.update().where(db_dbcomment.c.content.is_(None)).values(content = "")) - op.execute(db_dbcomment.update().where(db_dbcomment.c.mtime.is_(None)).values(mtime = timezone.now())) - op.execute(db_dbcomment.update().where(db_dbcomment.c.ctime.is_(None)).values(ctime = timezone.now())) - op.execute(db_dbcomment.update().where(db_dbcomment.c.uuid.is_(None)).values(uuid = get_new_uuid())) + op.execute(db_dbcomment.update().where(db_dbcomment.c.content.is_(None)).values(content='')) + op.execute(db_dbcomment.update().where(db_dbcomment.c.mtime.is_(None)).values(mtime=timezone.now())) + op.execute(db_dbcomment.update().where(db_dbcomment.c.ctime.is_(None)).values(ctime=timezone.now())) + op.execute(db_dbcomment.update().where(db_dbcomment.c.uuid.is_(None)).values(uuid=get_new_uuid())) db_dbcomputer = sa.sql.table( - "db_dbcomputer", + 'db_dbcomputer', sa.Column('description', sa.Text), sa.Column('hostname', sa.String(255)), sa.Column('metadata', JSONB), @@ -70,16 +70,15 @@ def upgrade(): sa.Column('uuid', UUID(as_uuid=True)), ) - op.execute(db_dbcomputer.update().where(db_dbcomputer.c.description.is_(None)).values(description = "")) - op.execute(db_dbcomputer.update().where(db_dbcomputer.c.hostname.is_(None)).values(hostname = "")) - op.execute(db_dbcomputer.update().where(db_dbcomputer.c.metadata == JSONB.NULL).values(metadata = {})) - op.execute(db_dbcomputer.update().where(db_dbcomputer.c.scheduler_type.is_(None)).values(scheduler_type = "")) - op.execute(db_dbcomputer.update().where(db_dbcomputer.c.transport_type.is_(None)).values(transport_type = "")) - op.execute(db_dbcomputer.update().where(db_dbcomputer.c.uuid.is_(None)).values(uuid = get_new_uuid())) - + op.execute(db_dbcomputer.update().where(db_dbcomputer.c.description.is_(None)).values(description='')) + op.execute(db_dbcomputer.update().where(db_dbcomputer.c.hostname.is_(None)).values(hostname='')) + op.execute(db_dbcomputer.update().where(db_dbcomputer.c.metadata == JSONB.NULL).values(metadata={})) + op.execute(db_dbcomputer.update().where(db_dbcomputer.c.scheduler_type.is_(None)).values(scheduler_type='')) + op.execute(db_dbcomputer.update().where(db_dbcomputer.c.transport_type.is_(None)).values(transport_type='')) + op.execute(db_dbcomputer.update().where(db_dbcomputer.c.uuid.is_(None)).values(uuid=get_new_uuid())) db_dbgroup = sa.sql.table( - "db_dbgroup", + 'db_dbgroup', # sa.Column('user_id', sa.Integer), sa.Column('description', sa.Text), sa.Column('label', sa.String(255)), @@ -88,11 +87,11 @@ def upgrade(): sa.Column('uuid', UUID(as_uuid=True)), ) - op.execute(db_dbgroup.update().where(db_dbgroup.c.description.is_(None)).values(description = "")) - op.execute(db_dbgroup.update().where(db_dbgroup.c.label.is_(None)).values(label = "")) - op.execute(db_dbgroup.update().where(db_dbgroup.c.time.is_(None)).values(time = timezone.now())) - op.execute(db_dbgroup.update().where(db_dbgroup.c.type_string.is_(None)).values(type_string = "")) - op.execute(db_dbgroup.update().where(db_dbgroup.c.uuid.is_(None)).values(uuid = get_new_uuid())) + op.execute(db_dbgroup.update().where(db_dbgroup.c.description.is_(None)).values(description='')) + op.execute(db_dbgroup.update().where(db_dbgroup.c.label.is_(None)).values(label='')) + op.execute(db_dbgroup.update().where(db_dbgroup.c.time.is_(None)).values(time=timezone.now())) + op.execute(db_dbgroup.update().where(db_dbgroup.c.type_string.is_(None)).values(type_string='')) + op.execute(db_dbgroup.update().where(db_dbgroup.c.uuid.is_(None)).values(uuid=get_new_uuid())) # db_dbgroup_dbnode = sa.sql.table( # "db_dbgroup_dbnode", @@ -101,16 +100,16 @@ def upgrade(): # ) db_dblink = sa.sql.table( - "db_dblink", + 'db_dblink', # sa.Column('input_id', sa.Integer), # sa.Column('output_id', sa.Integer), sa.Column('type', sa.String(255)), ) - op.execute(db_dblink.update().where(db_dblink.c.type.is_(None)).values(type = "")) + op.execute(db_dblink.update().where(db_dblink.c.type.is_(None)).values(type='')) db_dblog = sa.sql.table( - "db_dblog", + 'db_dblog', sa.Column('levelname', sa.String(255)), sa.Column('loggername', sa.String(255)), sa.Column('message', sa.Text), @@ -119,15 +118,15 @@ def upgrade(): sa.Column('uuid', UUID(as_uuid=True)), ) - op.execute(db_dblog.update().where(db_dblog.c.levelname.is_(None)).values(levelname = "")) - op.execute(db_dblog.update().where(db_dblog.c.loggername.is_(None)).values(loggername = "")) - op.execute(db_dblog.update().where(db_dblog.c.message.is_(None)).values(message = "")) - op.execute(db_dblog.update().where(db_dblog.c.metadata == JSONB.NULL).values(metadata = {})) - op.execute(db_dblog.update().where(db_dblog.c.time.is_(None)).values(time = timezone.now())) - op.execute(db_dblog.update().where(db_dblog.c.uuid.is_(None)).values(uuid = get_new_uuid())) + op.execute(db_dblog.update().where(db_dblog.c.levelname.is_(None)).values(levelname='')) + op.execute(db_dblog.update().where(db_dblog.c.loggername.is_(None)).values(loggername='')) + op.execute(db_dblog.update().where(db_dblog.c.message.is_(None)).values(message='')) + op.execute(db_dblog.update().where(db_dblog.c.metadata == JSONB.NULL).values(metadata={})) + op.execute(db_dblog.update().where(db_dblog.c.time.is_(None)).values(time=timezone.now())) + op.execute(db_dblog.update().where(db_dblog.c.uuid.is_(None)).values(uuid=get_new_uuid())) db_dbnode = sa.sql.table( - "db_dbnode", + 'db_dbnode', sa.Column('ctime', sa.DateTime(timezone=True)), sa.Column('description', sa.Text), sa.Column('label', sa.String(255)), @@ -136,32 +135,32 @@ def upgrade(): sa.Column('uuid', UUID(as_uuid=True)), ) - op.execute(db_dbnode.update().where(db_dbnode.c.ctime.is_(None)).values(ctime = timezone.now())) - op.execute(db_dbnode.update().where(db_dbnode.c.description.is_(None)).values(description = "")) - op.execute(db_dbnode.update().where(db_dbnode.c.label.is_(None)).values(label = "")) - op.execute(db_dbnode.update().where(db_dbnode.c.mtime.is_(None)).values(mtime = timezone.now())) - op.execute(db_dbnode.update().where(db_dbnode.c.node_type.is_(None)).values(node_type = "")) - op.execute(db_dbnode.update().where(db_dbnode.c.uuid.is_(None)).values(uuid = get_new_uuid())) + op.execute(db_dbnode.update().where(db_dbnode.c.ctime.is_(None)).values(ctime=timezone.now())) + op.execute(db_dbnode.update().where(db_dbnode.c.description.is_(None)).values(description='')) + op.execute(db_dbnode.update().where(db_dbnode.c.label.is_(None)).values(label='')) + op.execute(db_dbnode.update().where(db_dbnode.c.mtime.is_(None)).values(mtime=timezone.now())) + op.execute(db_dbnode.update().where(db_dbnode.c.node_type.is_(None)).values(node_type='')) + op.execute(db_dbnode.update().where(db_dbnode.c.uuid.is_(None)).values(uuid=get_new_uuid())) db_dbsetting = sa.sql.table( - "db_dbsetting", + 'db_dbsetting', sa.Column('time', sa.DateTime(timezone=True)), ) - op.execute(db_dbsetting.update().where(db_dbsetting.c.time.is_(None)).values(time = timezone.now())) + op.execute(db_dbsetting.update().where(db_dbsetting.c.time.is_(None)).values(time=timezone.now())) db_dbuser = sa.sql.table( - "db_dbuser", + 'db_dbuser', sa.Column('email', sa.String(254)), sa.Column('first_name', sa.String(254)), sa.Column('last_name', sa.String(254)), sa.Column('institution', sa.String(254)), ) - op.execute(db_dbuser.update().where(db_dbuser.c.email.is_(None)).values(email = "")) - op.execute(db_dbuser.update().where(db_dbuser.c.first_name.is_(None)).values(first_name = "")) - op.execute(db_dbuser.update().where(db_dbuser.c.last_name.is_(None)).values(last_name = "")) - op.execute(db_dbuser.update().where(db_dbuser.c.institution.is_(None)).values(institution = "")) + op.execute(db_dbuser.update().where(db_dbuser.c.email.is_(None)).values(email='')) + op.execute(db_dbuser.update().where(db_dbuser.c.first_name.is_(None)).values(first_name='')) + op.execute(db_dbuser.update().where(db_dbuser.c.last_name.is_(None)).values(last_name='')) + op.execute(db_dbuser.update().where(db_dbuser.c.institution.is_(None)).values(institution='')) def downgrade(): diff --git a/aiida/backends/sqlalchemy/models/authinfo.py b/aiida/backends/sqlalchemy/models/authinfo.py index b08cbefbce..0ab1c0e5b1 100644 --- a/aiida/backends/sqlalchemy/models/authinfo.py +++ b/aiida/backends/sqlalchemy/models/authinfo.py @@ -29,7 +29,9 @@ class DbAuthInfo(Base): Integer, ForeignKey('db_dbuser.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False ) dbcomputer_id = Column( - Integer, ForeignKey('db_dbcomputer.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False + Integer, + ForeignKey('db_dbcomputer.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), + nullable=False ) aiidauser = relationship('DbUser', backref='authinfos') diff --git a/aiida/backends/sqlalchemy/models/comment.py b/aiida/backends/sqlalchemy/models/comment.py index 701a0edf0e..c14fd7c3a0 100644 --- a/aiida/backends/sqlalchemy/models/comment.py +++ b/aiida/backends/sqlalchemy/models/comment.py @@ -28,12 +28,16 @@ class DbComment(Base): id = Column(Integer, primary_key=True) # pylint: disable=invalid-name uuid = Column(UUID(as_uuid=True), default=get_new_uuid, unique=True, nullable=False) - dbnode_id = Column(Integer, ForeignKey('db_dbnode.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False) + dbnode_id = Column( + Integer, ForeignKey('db_dbnode.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False + ) ctime = Column(DateTime(timezone=True), default=timezone.now, nullable=False) mtime = Column(DateTime(timezone=True), default=timezone.now, onupdate=timezone.now, nullable=False) - user_id = Column(Integer, ForeignKey('db_dbuser.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False) + user_id = Column( + Integer, ForeignKey('db_dbuser.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False + ) content = Column(Text, default='', nullable=False) dbnode = relationship('DbNode', backref='dbcomments') diff --git a/aiida/backends/sqlalchemy/models/group.py b/aiida/backends/sqlalchemy/models/group.py index 55a257f74f..f38b01d5f8 100644 --- a/aiida/backends/sqlalchemy/models/group.py +++ b/aiida/backends/sqlalchemy/models/group.py @@ -45,11 +45,13 @@ class DbGroup(Base): type_string = Column(String(255), default='', index=True, nullable=False) time = Column(DateTime(timezone=True), default=timezone.now, nullable=False) - description = Column(Text, default="", nullable=False) + description = Column(Text, default='', nullable=False) extras = Column(JSONB, default=dict, nullable=False) - user_id = Column(Integer, ForeignKey('db_dbuser.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False) + user_id = Column( + Integer, ForeignKey('db_dbuser.id', ondelete='CASCADE', deferrable=True, initially='DEFERRED'), nullable=False + ) user = relationship('DbUser', backref=backref('dbgroups', cascade='merge')) dbnodes = relationship('DbNode', secondary=table_groups_nodes, backref='dbgroups', lazy='dynamic') diff --git a/aiida/backends/sqlalchemy/models/log.py b/aiida/backends/sqlalchemy/models/log.py index 2f81ccf406..60c66d8d1e 100644 --- a/aiida/backends/sqlalchemy/models/log.py +++ b/aiida/backends/sqlalchemy/models/log.py @@ -28,12 +28,12 @@ class DbLog(Base): id = Column(Integer, primary_key=True) # pylint: disable=invalid-name uuid = Column(UUID(as_uuid=True), default=get_new_uuid, unique=True, nullable=False) time = Column(DateTime(timezone=True), default=timezone.now, nullable=False) - loggername = Column(String(255), default="", index=True, nullable=False) - levelname = Column(String(255), default="", index=True, nullable=False) + loggername = Column(String(255), default='', index=True, nullable=False) + levelname = Column(String(255), default='', index=True, nullable=False) dbnode_id = Column( Integer, ForeignKey('db_dbnode.id', deferrable=True, initially='DEFERRED', ondelete='CASCADE'), nullable=False ) - message = Column(Text(), default="", nullable=False) + message = Column(Text(), default='', nullable=False) _metadata = Column('metadata', JSONB, default=dict, nullable=False) dbnode = relationship('DbNode', backref=backref('dblogs', passive_deletes='all', cascade='merge')) diff --git a/aiida/backends/sqlalchemy/models/node.py b/aiida/backends/sqlalchemy/models/node.py index 96271c141d..d5ec9108f6 100644 --- a/aiida/backends/sqlalchemy/models/node.py +++ b/aiida/backends/sqlalchemy/models/node.py @@ -10,9 +10,10 @@ # pylint: disable=import-error,no-name-in-module """Module to manage nodes for the SQLA backend.""" -from sqlalchemy import ForeignKey +from sqlalchemy import ForeignKey, text from sqlalchemy.orm import relationship, backref from sqlalchemy.schema import Column +from sqlalchemy.sql.schema import Index from sqlalchemy.types import Integer, String, DateTime, Text # Specific to PGSQL. If needed to be agnostic # http://docs.sqlalchemy.org/en/rel_0_9/core/custom_types.html?highlight=guid#backend-agnostic-guid-type @@ -31,7 +32,7 @@ class DbNode(Base): id = Column(Integer, primary_key=True) # pylint: disable=invalid-name uuid = Column(UUID(as_uuid=True), default=get_new_uuid, unique=True, nullable=False) - node_type = Column(String(255), default="", index=True, nullable=False) + node_type = Column(String(255), default='', index=True, nullable=False) process_type = Column(String(255), index=True) label = Column( String(255), index=True, nullable=False, default='' @@ -81,6 +82,15 @@ class DbNode(Base): passive_deletes=True ) + __tableargs__ = ( + Index( + 'db_dbnode_node_type_like', + node_type, + postgresql_using='btree', + postgresql_ops={'data': 'varchar_pattern_ops'} + ), + ) + def __init__(self, *args, **kwargs): """Add three additional attributes to the base class: mtime, attributes and extras.""" super().__init__(*args, **kwargs) diff --git a/aiida/backends/sqlalchemy/models/user.py b/aiida/backends/sqlalchemy/models/user.py index 6e651b8527..46ac827cc6 100644 --- a/aiida/backends/sqlalchemy/models/user.py +++ b/aiida/backends/sqlalchemy/models/user.py @@ -22,9 +22,9 @@ class DbUser(Base): id = Column(Integer, primary_key=True) # pylint: disable=invalid-name email = Column(String(254), unique=True, index=True, nullable=False) - first_name = Column(String(254), default="", nullable=False) - last_name = Column(String(254), default="", nullable=False) - institution = Column(String(254), default="", nullable=False) + first_name = Column(String(254), default='', nullable=False) + last_name = Column(String(254), default='', nullable=False) + institution = Column(String(254), default='', nullable=False) def __init__(self, email, first_name='', last_name='', institution='', **kwargs): """Set additional class attributes with respect to the base class.""" diff --git a/tests/backends/aiida_sqlalchemy/test_migrations.py b/tests/backends/aiida_sqlalchemy/test_migrations.py index af9e0bd826..4a67ffc60a 100644 --- a/tests/backends/aiida_sqlalchemy/test_migrations.py +++ b/tests/backends/aiida_sqlalchemy/test_migrations.py @@ -2076,7 +2076,7 @@ def setUpBeforeMigration(self): session.add(comment) session.commit() - computer = DbComputer(label="label", description=None, hostname=None, metadata=None, scheduler_type=None, transport_type=None, uuid=None) + computer = DbComputer(label='label', description=None, hostname=None, metadata=None, scheduler_type=None, transport_type=None, uuid=None) session.add(computer) session.commit() @@ -2084,7 +2084,7 @@ def setUpBeforeMigration(self): session.add(group) session.commit() - link = DbLink(label="label", type=None) + link = DbLink(label='label', type=None) session.add(link) session.commit() @@ -2096,7 +2096,7 @@ def setUpBeforeMigration(self): session.add(log) session.commit() - setting = DbSetting(key="key", description="", time=None) + setting = DbSetting(key='key', description='', time=None) session.add(setting) session.commit() self.setting_id = setting.id @@ -2113,7 +2113,7 @@ def test_migration(self): DbSetting = self.get_current_table('db_dbsetting') DbUser = self.get_current_table('db_dbuser') with self.get_session() as session: - + authinfo = session.query(DbAuthInfo).one() assert authinfo.auth_params is not None assert authinfo.enabled is not None