From e613320bbcf015443c12dad7575927b48e93b316 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Sun, 28 May 2023 19:22:53 +0545 Subject: [PATCH 01/11] alembic setup --- src/backend/alembic.ini | 47 ++++++++++++++++ src/backend/migrations/README | 1 + src/backend/migrations/env.py | 79 +++++++++++++++++++++++++++ src/backend/migrations/script.py.mako | 24 ++++++++ 4 files changed, 151 insertions(+) create mode 100644 src/backend/alembic.ini create mode 100755 src/backend/migrations/README create mode 100755 src/backend/migrations/env.py create mode 100755 src/backend/migrations/script.py.mako diff --git a/src/backend/alembic.ini b/src/backend/alembic.ini new file mode 100644 index 0000000000..9d4ce51a76 --- /dev/null +++ b/src/backend/alembic.ini @@ -0,0 +1,47 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migrations + +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + + +sqlalchemy.url = postgresql+psycopg2://fmtm:fmtm@fmtm_db:5432/fmtm + +[post_write_hooks] + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/src/backend/migrations/README b/src/backend/migrations/README new file mode 100755 index 0000000000..98e4f9c44e --- /dev/null +++ b/src/backend/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/src/backend/migrations/env.py b/src/backend/migrations/env.py new file mode 100755 index 0000000000..c62888cfba --- /dev/null +++ b/src/backend/migrations/env.py @@ -0,0 +1,79 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from db.database import FmtmMetadata +target_metadata = FmtmMetadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/src/backend/migrations/script.py.mako b/src/backend/migrations/script.py.mako new file mode 100755 index 0000000000..55df2863d2 --- /dev/null +++ b/src/backend/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} From 7a7882cfeba7ebf9d7b61d35e825c10a87689d57 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Thu, 1 Jun 2023 09:27:36 +0545 Subject: [PATCH 02/11] alembic.ini file moved to app directory --- src/backend/alembic.ini | 47 --------------- src/backend/app/alembic.ini | 110 ++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 47 deletions(-) delete mode 100644 src/backend/alembic.ini create mode 100644 src/backend/app/alembic.ini diff --git a/src/backend/alembic.ini b/src/backend/alembic.ini deleted file mode 100644 index 9d4ce51a76..0000000000 --- a/src/backend/alembic.ini +++ /dev/null @@ -1,47 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = migrations - -version_path_separator = os # Use os.pathsep. Default configuration used for new projects. - - -sqlalchemy.url = postgresql+psycopg2://fmtm:fmtm@fmtm_db:5432/fmtm - -[post_write_hooks] - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/src/backend/app/alembic.ini b/src/backend/app/alembic.ini new file mode 100644 index 0000000000..a2f396fcde --- /dev/null +++ b/src/backend/app/alembic.ini @@ -0,0 +1,110 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql+psycopg2://fmtm:fmtm@fmtm-db:5432/testingfmtm + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S From bd40e16e91a299bd668abc801b376fdcd862c2f1 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Thu, 1 Jun 2023 09:29:48 +0545 Subject: [PATCH 03/11] Base.metadata.create_all(bind=engine) removed since alembic is used now for migration --- src/backend/app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/app/main.py b/src/backend/app/main.py index 57c9fb9885..56ac25a9f5 100644 --- a/src/backend/app/main.py +++ b/src/backend/app/main.py @@ -120,7 +120,7 @@ async def startup_event(): """Commands to run on server startup.""" logger.debug("Starting up FastAPI server.") logger.debug("Connecting to DB with SQLAlchemy") - Base.metadata.create_all(bind=engine) + # Base.metadata.create_all(bind=engine) # Read in XLSForms read_xlsforms(next(get_db()), xlsforms_path) From c96af13e49b5e2cce195142b1e9957ae3874d3b8 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Thu, 1 Jun 2023 09:30:47 +0545 Subject: [PATCH 04/11] env.py for alembic is moved inside app directory --- src/backend/{ => app}/migrations/env.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) rename src/backend/{ => app}/migrations/env.py (66%) mode change 100755 => 100644 diff --git a/src/backend/migrations/env.py b/src/backend/app/migrations/env.py old mode 100755 new mode 100644 similarity index 66% rename from src/backend/migrations/env.py rename to src/backend/app/migrations/env.py index c62888cfba..76f0c6da17 --- a/src/backend/migrations/env.py +++ b/src/backend/app/migrations/env.py @@ -5,26 +5,13 @@ from alembic import context -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. config = context.config -# Interpret the config file for Python logging. -# This line sets up loggers basically. if config.config_file_name is not None: fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -from db.database import FmtmMetadata -target_metadata = FmtmMetadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. +from db.db_models import Base +target_metadata = Base.metadata def run_migrations_offline() -> None: @@ -59,15 +46,13 @@ def run_migrations_online() -> None: """ connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), + config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool, ) with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) + context.configure(connection=connection, target_metadata=target_metadata) with context.begin_transaction(): context.run_migrations() From b0bfb9ba06bd63ac8c7d98a638c1441d9954947f Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Thu, 1 Jun 2023 09:31:49 +0545 Subject: [PATCH 05/11] script.py.mako is moved inside app directory --- src/backend/{ => app}/migrations/script.py.mako | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/backend/{ => app}/migrations/script.py.mako (100%) mode change 100755 => 100644 diff --git a/src/backend/migrations/script.py.mako b/src/backend/app/migrations/script.py.mako old mode 100755 new mode 100644 similarity index 100% rename from src/backend/migrations/script.py.mako rename to src/backend/app/migrations/script.py.mako From ac24b56f6e347e1da46a8e31562572185b051e9d Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Thu, 1 Jun 2023 09:32:06 +0545 Subject: [PATCH 06/11] readme for alembic moved in app directory --- src/backend/{ => app}/migrations/README | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/backend/{ => app}/migrations/README (100%) mode change 100755 => 100644 diff --git a/src/backend/migrations/README b/src/backend/app/migrations/README old mode 100755 new mode 100644 similarity index 100% rename from src/backend/migrations/README rename to src/backend/app/migrations/README From c9906ecee3891f7ca10baf0bb829a049d5dc2f4c Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Sun, 4 Jun 2023 11:22:34 +0545 Subject: [PATCH 07/11] updated alembic.ini file to exclude postgis exxtension --- src/backend/app/alembic.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/app/alembic.ini b/src/backend/app/alembic.ini index a2f396fcde..ed8b40c909 100644 --- a/src/backend/app/alembic.ini +++ b/src/backend/app/alembic.ini @@ -74,6 +74,10 @@ sqlalchemy.url = postgresql+psycopg2://fmtm:fmtm@fmtm-db:5432/testingfmtm # black.entrypoint = black # black.options = -l 79 REVISION_SCRIPT_FILENAME +# Custom param that enables us to specify tables to ignore when determining migrations +[alembic:exclude] +tables = spatial_ref_sys + # Logging configuration [loggers] keys = root,sqlalchemy,alembic From e81bd03eba3c62bb68cde3c5e8825f37557b9bf9 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Sun, 4 Jun 2023 11:23:54 +0545 Subject: [PATCH 08/11] updated env.py in migrations to exclude postgis extension --- src/backend/app/migrations/env.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/backend/app/migrations/env.py b/src/backend/app/migrations/env.py index 76f0c6da17..90d1e4240a 100644 --- a/src/backend/app/migrations/env.py +++ b/src/backend/app/migrations/env.py @@ -14,6 +14,20 @@ target_metadata = Base.metadata +exclude_tables = config.get_section("alembic:exclude").get("tables", "").split(",") + +def include_object(object, name, type_, reflected, compare_to): + """ + Custom helper function that enables us to ignore our excluded tables in the autogen sweep + """ + if type_ == "table" and name in exclude_tables: + return False + else: + return alembic_helpers.include_object( + object, name, type_, reflected, compare_to + ) + + def run_migrations_offline() -> None: """Run migrations in 'offline' mode. @@ -29,6 +43,7 @@ def run_migrations_offline() -> None: url = config.get_main_option("sqlalchemy.url") context.configure( url=url, + include_object=include_object, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, @@ -52,7 +67,9 @@ def run_migrations_online() -> None: ) with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) + context.configure(connection=connection, + include_object=include_object, + target_metadata=target_metadata) with context.begin_transaction(): context.run_migrations() From d125256f48ef6e9b515576aed3da68ff2a55b8ab Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Sun, 4 Jun 2023 11:25:30 +0545 Subject: [PATCH 09/11] sqlalchemy url obtained from settings for alembic migrations --- src/backend/app/alembic.ini | 5 +++-- src/backend/app/migrations/env.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/app/alembic.ini b/src/backend/app/alembic.ini index ed8b40c909..7166e16918 100644 --- a/src/backend/app/alembic.ini +++ b/src/backend/app/alembic.ini @@ -60,8 +60,7 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = postgresql+psycopg2://fmtm:fmtm@fmtm-db:5432/testingfmtm - +sqlalchemy.url = [post_write_hooks] # post_write_hooks defines scripts or Python functions that are run @@ -112,3 +111,5 @@ formatter = generic [formatter_generic] format = %(levelname)-5.5s [%(name)s] %(message)s datefmt = %H:%M:%S + + diff --git a/src/backend/app/migrations/env.py b/src/backend/app/migrations/env.py index 90d1e4240a..c3d1cf071b 100644 --- a/src/backend/app/migrations/env.py +++ b/src/backend/app/migrations/env.py @@ -2,10 +2,13 @@ from sqlalchemy import engine_from_config from sqlalchemy import pool +from config import settings +from geoalchemy2 import alembic_helpers from alembic import context config = context.config +config.set_main_option("sqlalchemy.url", settings.SQLALCHEMY_URL) if config.config_file_name is not None: fileConfig(config.config_file_name) From 330cbadd70779108f4fb1abcbaf1634e2a2ef8ed Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Sun, 4 Jun 2023 11:26:04 +0545 Subject: [PATCH 10/11] sqlalchemy url obtained from env in config settings --- src/backend/app/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/app/config.py b/src/backend/app/config.py index 634bb3387c..e5e3c25206 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -113,6 +113,7 @@ def assemble_db_connection(cls, v: str, values: dict[str, Any]) -> Any: OSM_LOGIN_REDIRECT_URI: AnyUrl OSM_SECRET_KEY: str OAUTHLIB_INSECURE_TRANSPORT: Optional[str] = 1 + SQLALCHEMY_URL: Optional[str] class Config: """Pydantic settings config.""" From 9f5d771136aecb74fcae4c696efa396c43041eea Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Sun, 4 Jun 2023 11:28:27 +0545 Subject: [PATCH 11/11] first migration file --- .../ffa958c0e643_create_initial_tables.py | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 src/backend/app/migrations/versions/ffa958c0e643_create_initial_tables.py diff --git a/src/backend/app/migrations/versions/ffa958c0e643_create_initial_tables.py b/src/backend/app/migrations/versions/ffa958c0e643_create_initial_tables.py new file mode 100644 index 0000000000..af1f15abf5 --- /dev/null +++ b/src/backend/app/migrations/versions/ffa958c0e643_create_initial_tables.py @@ -0,0 +1,361 @@ +"""create_initial_tables + +Revision ID: ffa958c0e643 +Revises: +Create Date: 2023-06-04 05:10:26.929986 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +import geoalchemy2 + +# revision identifiers, used by Alembic. +revision = 'ffa958c0e643' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute('CREATE EXTENSION IF NOT EXISTS postgis;') + + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('background_tasks', + sa.Column('id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('status', sa.Enum('PENDING', 'FAILED', 'RECEIVED', 'SUCCESS', name='backgroundtaskstatus'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('licenses', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('plain_text', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('mapping_issue_categories', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('archived', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('organisations', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=512), nullable=False), + sa.Column('slug', sa.String(length=255), nullable=False), + sa.Column('logo', sa.String(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('url', sa.String(), nullable=True), + sa.Column('type', sa.Enum('FREE', 'DISCOUNTED', 'FULL_FEE', name='organisationtype'), nullable=False), + sa.Column('subscription_tier', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name'), + sa.UniqueConstraint('slug') + ) + op.create_table('qr_code', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('filename', sa.String(), nullable=True), + sa.Column('image', sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('users', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('username', sa.String(), nullable=True), + sa.Column('role', sa.Enum('MAPPER', 'ADMIN', 'VALIDATOR', 'FIELD_ADMIN', 'ORGANIZATION_ADMIN', 'READ_ONLY', name='userrole'), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('city', sa.String(), nullable=True), + sa.Column('country', sa.String(), nullable=True), + sa.Column('email_address', sa.String(), nullable=True), + sa.Column('is_email_verified', sa.Boolean(), nullable=True), + sa.Column('is_expert', sa.Boolean(), nullable=True), + sa.Column('mapping_level', sa.Enum('BEGINNER', 'INTERMEDIATE', 'ADVANCED', name='mappinglevel'), nullable=False), + sa.Column('tasks_mapped', sa.Integer(), nullable=False), + sa.Column('tasks_validated', sa.Integer(), nullable=False), + sa.Column('tasks_invalidated', sa.Integer(), nullable=False), + sa.Column('projects_mapped', sa.ARRAY(sa.Integer()), nullable=True), + sa.Column('date_registered', sa.DateTime(), nullable=True), + sa.Column('last_validation_date', sa.DateTime(), nullable=True), + sa.Column('password', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('username') + ) + op.create_index(op.f('ix_users_id'), 'users', ['id'], unique=False) + op.create_table('xlsforms', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('title', sa.String(), nullable=True), + sa.Column('category', sa.String(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('xml', sa.String(), nullable=True), + sa.Column('xls', sa.LargeBinary(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('title') + ) + op.create_table('organisation_managers', + sa.Column('organisation_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['organisation_id'], ['organisations.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.UniqueConstraint('organisation_id', 'user_id', name='organisation_user_key') + ) + op.create_table('projects', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('odkid', sa.Integer(), nullable=True), + sa.Column('author_id', sa.BigInteger(), nullable=False), + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('task_creation_mode', sa.Enum('GRID', 'ROADS', 'UPLOAD', name='taskcreationmode'), nullable=False), + sa.Column('project_name_prefix', sa.String(), nullable=True), + sa.Column('task_type_prefix', sa.String(), nullable=True), + sa.Column('location_str', sa.String(), nullable=True), + sa.Column('outline', geoalchemy2.types.Geometry(geometry_type='POLYGON', srid=4326, from_text='ST_GeomFromEWKT', name='geometry'), nullable=True), + sa.Column('last_updated', sa.DateTime(), nullable=True), + sa.Column('status', sa.Enum('ARCHIVED', 'PUBLISHED', 'DRAFT', name='projectstatus'), nullable=False), + sa.Column('total_tasks', sa.Integer(), nullable=True), + sa.Column('odk_central_src', sa.String(), nullable=True), + sa.Column('xform_title', sa.String(), nullable=True), + sa.Column('private', sa.Boolean(), nullable=True), + sa.Column('mapper_level', sa.Enum('BEGINNER', 'INTERMEDIATE', 'ADVANCED', name='mappinglevel'), nullable=False), + sa.Column('priority', sa.Enum('URGENT', 'HIGH', 'MEDIUM', 'LOW', name='projectpriority'), nullable=True), + sa.Column('featured', sa.Boolean(), nullable=True), + sa.Column('mapping_permission', sa.Enum('ANY', 'LEVEL', 'TEAMS', 'TEAMS_LEVEL', name='mappingpermission'), nullable=True), + sa.Column('validation_permission', sa.Enum('ANY', 'LEVEL', 'TEAMS', 'TEAMS_LEVEL', name='validationpermission'), nullable=True), + sa.Column('organisation_id', sa.Integer(), nullable=True), + sa.Column('due_date', sa.DateTime(), nullable=True), + sa.Column('changeset_comment', sa.String(), nullable=True), + sa.Column('osmcha_filter_id', sa.String(), nullable=True), + sa.Column('imagery', sa.String(), nullable=True), + sa.Column('osm_preset', sa.String(), nullable=True), + sa.Column('odk_preset', sa.String(), nullable=True), + sa.Column('josm_preset', sa.String(), nullable=True), + sa.Column('id_presets', sa.ARRAY(sa.String()), nullable=True), + sa.Column('extra_id_params', sa.String(), nullable=True), + sa.Column('license_id', sa.Integer(), nullable=True), + sa.Column('centroid', geoalchemy2.types.Geometry(geometry_type='POINT', srid=4326, from_text='ST_GeomFromEWKT', name='geometry'), nullable=True), + sa.Column('odk_central_url', sa.String(), nullable=True), + sa.Column('odk_central_user', sa.String(), nullable=True), + sa.Column('odk_central_password', sa.String(), nullable=True), + sa.Column('extract_completed_count', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['users.id'], name='fk_users'), + sa.ForeignKeyConstraint(['license_id'], ['licenses.id'], name='fk_licenses'), + sa.ForeignKeyConstraint(['organisation_id'], ['organisations.id'], name='fk_organisations'), + sa.ForeignKeyConstraint(['xform_title'], ['xlsforms.title'], name='fk_xform'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('idx_geometry', 'projects', ['outline'], unique=False, postgresql_using='gist') + # op.create_index('idx_projects_centroid', 'projects', ['centroid'], unique=False, postgresql_using='gist') + # op.create_index('idx_projects_outline', 'projects', ['outline'], unique=False, postgresql_using='gist') + op.create_index(op.f('ix_projects_mapper_level'), 'projects', ['mapper_level'], unique=False) + op.create_index(op.f('ix_projects_organisation_id'), 'projects', ['organisation_id'], unique=False) + op.create_table('teams', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('organisation_id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=512), nullable=False), + sa.Column('logo', sa.String(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('invite_only', sa.Boolean(), nullable=False), + sa.Column('visibility', sa.Enum('PUBLIC', 'PRIVATE', name='teamvisibility'), nullable=False), + sa.ForeignKeyConstraint(['organisation_id'], ['organisations.id'], name='fk_organisations'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_licenses', + sa.Column('user', sa.BigInteger(), nullable=True), + sa.Column('license', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['license'], ['licenses.id'], ), + sa.ForeignKeyConstraint(['user'], ['users.id'], ) + ) + op.create_table('project_allowed_users', + sa.Column('project_id', sa.Integer(), nullable=True), + sa.Column('user_id', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ) + ) + op.create_table('project_chat', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('time_stamp', sa.DateTime(), nullable=False), + sa.Column('message', sa.String(), nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_project_chat_project_id'), 'project_chat', ['project_id'], unique=False) + op.create_table('project_info', + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('project_id_str', sa.String(), nullable=True), + sa.Column('name', sa.String(length=512), nullable=True), + sa.Column('short_description', sa.String(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('text_searchable', postgresql.TSVECTOR(), nullable=True), + sa.Column('per_task_instructions', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.PrimaryKeyConstraint('project_id') + ) + op.create_index('textsearch_idx', 'project_info', ['text_searchable'], unique=False) + op.create_table('project_teams', + sa.Column('team_id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('role', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['team_id'], ['teams.id'], ), + sa.PrimaryKeyConstraint('team_id', 'project_id') + ) + op.create_table('tasks', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('project_task_index', sa.Integer(), nullable=True), + sa.Column('project_task_name', sa.String(), nullable=True), + sa.Column('outline', geoalchemy2.types.Geometry(geometry_type='POLYGON', srid=4326, from_text='ST_GeomFromEWKT', name='geometry'), nullable=True), + sa.Column('geometry_geojson', sa.String(), nullable=True), + sa.Column('initial_feature_count', sa.Integer(), nullable=True), + sa.Column('task_status', sa.Enum('READY', 'LOCKED_FOR_MAPPING', 'MAPPED', 'LOCKED_FOR_VALIDATION', 'VALIDATED', 'INVALIDATED', 'BAD', 'SPLIT', 'ARCHIVED', name='taskstatus'), nullable=True), + sa.Column('locked_by', sa.BigInteger(), nullable=True), + sa.Column('mapped_by', sa.BigInteger(), nullable=True), + sa.Column('validated_by', sa.BigInteger(), nullable=True), + sa.Column('qr_code_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['locked_by'], ['users.id'], name='fk_users_locked'), + sa.ForeignKeyConstraint(['mapped_by'], ['users.id'], name='fk_users_mapper'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['qr_code_id'], ['qr_code.id'], ), + sa.ForeignKeyConstraint(['validated_by'], ['users.id'], name='fk_users_validator'), + sa.PrimaryKeyConstraint('id', 'project_id') + ) + op.create_index(op.f('ix_tasks_locked_by'), 'tasks', ['locked_by'], unique=False) + op.create_index(op.f('ix_tasks_mapped_by'), 'tasks', ['mapped_by'], unique=False) + op.create_index(op.f('ix_tasks_project_id'), 'tasks', ['project_id'], unique=False) + op.create_index(op.f('ix_tasks_qr_code_id'), 'tasks', ['qr_code_id'], unique=False) + op.create_index(op.f('ix_tasks_validated_by'), 'tasks', ['validated_by'], unique=False) + op.create_table('user_roles', + sa.Column('user_id', sa.BigInteger(), nullable=False), + sa.Column('organization_id', sa.Integer(), nullable=True), + sa.Column('project_id', sa.Integer(), nullable=True), + sa.Column('role', sa.Enum('MAPPER', 'ADMIN', 'VALIDATOR', 'FIELD_ADMIN', 'ORGANIZATION_ADMIN', 'READ_ONLY', name='userrole'), nullable=False), + sa.ForeignKeyConstraint(['organization_id'], ['organisations.id'], ), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('user_id') + ) + op.create_table('features', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=True), + sa.Column('category_title', sa.String(), nullable=True), + sa.Column('task_id', sa.Integer(), nullable=False), + sa.Column('properties', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('geometry', geoalchemy2.types.Geometry(srid=4326, from_text='ST_GeomFromEWKT', name='geometry'), nullable=True), + sa.ForeignKeyConstraint(['category_title'], ['xlsforms.title'], name='fk_xform'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['task_id', 'project_id'], ['tasks.id', 'tasks.project_id'], name='fk_tasks'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('idx_features_composite', 'features', ['task_id', 'project_id'], unique=False) + op.create_table('task_history', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=True), + sa.Column('task_id', sa.Integer(), nullable=False), + sa.Column('action', sa.Enum('RELEASED_FOR_MAPPING', 'LOCKED_FOR_MAPPING', 'MARKED_MAPPED', 'LOCKED_FOR_VALIDATION', 'VALIDATED', 'MARKED_INVALID', 'MARKED_BAD', 'SPLIT_NEEDED', 'RECREATED', 'COMMENT', name='taskaction'), nullable=False), + sa.Column('action_text', sa.String(), nullable=True), + sa.Column('action_date', sa.DateTime(), nullable=False), + sa.Column('user_id', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['task_id', 'project_id'], ['tasks.id', 'tasks.project_id'], name='fk_tasks'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='fk_users'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('idx_task_history_composite', 'task_history', ['task_id', 'project_id'], unique=False) + op.create_index('idx_task_history_project_id_user_id', 'task_history', ['user_id', 'project_id'], unique=False) + op.create_index(op.f('ix_task_history_project_id'), 'task_history', ['project_id'], unique=False) + op.create_index(op.f('ix_task_history_user_id'), 'task_history', ['user_id'], unique=False) + op.create_table('task_invalidation_history', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.Integer(), nullable=False), + sa.Column('task_id', sa.Integer(), nullable=False), + sa.Column('is_closed', sa.Boolean(), nullable=True), + sa.Column('mapper_id', sa.BigInteger(), nullable=True), + sa.Column('mapped_date', sa.DateTime(), nullable=True), + sa.Column('invalidator_id', sa.BigInteger(), nullable=True), + sa.Column('invalidated_date', sa.DateTime(), nullable=True), + sa.Column('invalidation_history_id', sa.Integer(), nullable=True), + sa.Column('validator_id', sa.BigInteger(), nullable=True), + sa.Column('validated_date', sa.DateTime(), nullable=True), + sa.Column('updated_date', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['invalidation_history_id'], ['task_history.id'], name='fk_invalidation_history'), + sa.ForeignKeyConstraint(['invalidator_id'], ['users.id'], name='fk_invalidators'), + sa.ForeignKeyConstraint(['mapper_id'], ['users.id'], name='fk_mappers'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ), + sa.ForeignKeyConstraint(['task_id', 'project_id'], ['tasks.id', 'tasks.project_id'], name='fk_tasks'), + sa.ForeignKeyConstraint(['validator_id'], ['users.id'], name='fk_validators'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('idx_task_validation_history_composite', 'task_invalidation_history', ['task_id', 'project_id'], unique=False) + op.create_index('idx_task_validation_mapper_status_composite', 'task_invalidation_history', ['mapper_id', 'is_closed'], unique=False) + op.create_index('idx_task_validation_validator_status_composite', 'task_invalidation_history', ['invalidator_id', 'is_closed'], unique=False) + op.create_table('task_mapping_issues', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('task_history_id', sa.Integer(), nullable=False), + sa.Column('issue', sa.String(), nullable=False), + sa.Column('mapping_issue_category_id', sa.Integer(), nullable=False), + sa.Column('count', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['mapping_issue_category_id'], ['mapping_issue_categories.id'], name='fk_issue_category'), + sa.ForeignKeyConstraint(['task_history_id'], ['task_history.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_task_mapping_issues_task_history_id'), 'task_mapping_issues', ['task_history_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + + # ### commands auto generated by Alembic - please adjust! ### + + op.execute('DROP EXTENSION IF EXISTS postgis;') + + op.drop_index(op.f('ix_task_mapping_issues_task_history_id'), table_name='task_mapping_issues') + op.drop_table('task_mapping_issues') + op.drop_index('idx_task_validation_validator_status_composite', table_name='task_invalidation_history') + op.drop_index('idx_task_validation_mapper_status_composite', table_name='task_invalidation_history') + op.drop_index('idx_task_validation_history_composite', table_name='task_invalidation_history') + op.drop_table('task_invalidation_history') + op.drop_index(op.f('ix_task_history_user_id'), table_name='task_history') + op.drop_index(op.f('ix_task_history_project_id'), table_name='task_history') + op.drop_index('idx_task_history_project_id_user_id', table_name='task_history') + op.drop_index('idx_task_history_composite', table_name='task_history') + op.drop_table('task_history') + op.drop_index('idx_features_geometry', table_name='features', postgresql_using='gist') + op.drop_index('idx_features_composite', table_name='features') + op.drop_table('features') + op.drop_table('user_roles') + op.drop_index(op.f('ix_tasks_validated_by'), table_name='tasks') + op.drop_index(op.f('ix_tasks_qr_code_id'), table_name='tasks') + op.drop_index(op.f('ix_tasks_project_id'), table_name='tasks') + op.drop_index(op.f('ix_tasks_mapped_by'), table_name='tasks') + op.drop_index(op.f('ix_tasks_locked_by'), table_name='tasks') + op.drop_index('idx_tasks_outline', table_name='tasks', postgresql_using='gist') + op.drop_index('idx_geometry', table_name='tasks', postgresql_using='gist') + op.drop_table('tasks') + op.drop_table('project_teams') + op.drop_index('textsearch_idx', table_name='project_info') + op.drop_table('project_info') + op.drop_index(op.f('ix_project_chat_project_id'), table_name='project_chat') + op.drop_table('project_chat') + op.drop_table('project_allowed_users') + op.drop_table('user_licenses') + op.drop_table('teams') + op.drop_index(op.f('ix_projects_organisation_id'), table_name='projects') + op.drop_index(op.f('ix_projects_mapper_level'), table_name='projects') + op.drop_index('idx_projects_outline', table_name='projects', postgresql_using='gist') + op.drop_index('idx_projects_centroid', table_name='projects', postgresql_using='gist') + op.drop_index('idx_geometry', table_name='projects', postgresql_using='gist') + op.drop_table('projects') + op.drop_table('organisation_managers') + op.drop_table('xlsforms') + op.drop_index(op.f('ix_users_id'), table_name='users') + op.drop_table('users') + op.drop_table('qr_code') + op.drop_table('organisations') + op.drop_table('mapping_issue_categories') + op.drop_table('licenses') + op.drop_table('background_tasks') + # ### end Alembic commands ###