From bc6abbd2ca33c3ea316428a09fd9fb6bc707dd53 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Sun, 24 Mar 2024 13:23:04 +0100 Subject: [PATCH 01/46] feat: Implement Rosemary CLI --- .gitignore | 4 ++- Dockerfile.dev | 10 +++++-- README.md | 50 ++++++++++++++++++++++++++--------- requirements.txt | 3 ++- rosemary/__init__.py | 0 rosemary/cli.py | 19 +++++++++++++ rosemary/commands/__init__.py | 0 rosemary/commands/env.py | 15 +++++++++++ rosemary/commands/info.py | 31 ++++++++++++++++++++++ rosemary/commands/update.py | 29 ++++++++++++++++++++ setup.py | 21 +++++++++++++++ 11 files changed, 166 insertions(+), 16 deletions(-) create mode 100644 rosemary/__init__.py create mode 100644 rosemary/cli.py create mode 100644 rosemary/commands/__init__.py create mode 100644 rosemary/commands/env.py create mode 100644 rosemary/commands/info.py create mode 100644 rosemary/commands/update.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 556db4840..8522fcf08 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ __pycache__/ .idea uploads/ app.log -.DS_Store \ No newline at end of file +.DS_Store +rosemary.egg-info/ +build/ \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev index 0c9327e84..2e0426cad 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,14 +1,17 @@ # Use an official Python runtime as a parent image FROM python:3.11-alpine +# Set this environment variable to suppress the "Running as root" warning from pip +ENV PIP_ROOT_USER_ACTION=ignore + # Install the MySQL client to be able to use it in the standby script. RUN apk add --no-cache mysql-client # Set the working directory in the container to /app WORKDIR /app -# Copy the contents of the local app/ directory to the /app directory in the container -COPY app/ ./app +# Copy the entire project into the container +COPY . . # Copy requirements.txt at the /app working directory COPY requirements.txt . @@ -19,6 +22,9 @@ COPY --chmod=+x scripts/wait-for-db.sh ./scripts/ # Install any needed packages specified in requirements.txt RUN pip install --no-cache-dir -r requirements.txt +# Install rosemary CLI tool +RUN pip install --no-cache-dir . + # Update pip RUN pip install --no-cache-dir --upgrade pip diff --git a/README.md b/README.md index e9264a44a..b78815a60 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,44 @@ To run unit test, please enter inside `web` container: pytest app/tests/units.py ``` +## Using Rosemary CLI + +`Rosemary` is a CLI tool developed to facilitate project management and development tasks. + +To use the Rosemary CLI, you need to be inside the `web_app_container` Docker container. This ensures that Rosemary operates in the correct environment and has access to all necessary files and settings. + +First, make sure your Docker environment is running. Then, access the `web_app_container` using the following command: + +``` +docker exec -it web_app_container /bin/sh +``` + +In the terminal, you should see the prefix `/app #`. You are now ready to use Rosemary's commands. + +### Update Project Dependencies + +To update all project dependencies, run: + +``` +rosemary update +``` + +Note: it is the responsibility of the developer to check that the update of the dependencies has not broken any +functionality and each dependency maintains backwards compatibility. Use the script with care! + +### Viewing Environment Variables + +To view the current `.env` file settings, use: + +``` +rosemary env +``` +### Available Commands + +- `rosemary update`: Updates all project dependencies and the `requirements.txt` file. +- `rosemary info`: Displays information about the Rosemary CLI, including version and author. +- `rosemary env`: Displays the current environment variables from the `.env` file. + ## Deploy in production (Docker Compose) ``` @@ -107,15 +145,3 @@ To renew a certificate that is less than 60 days from expiry, execute: cd scripts chmod +x ssl_renew.sh && ./ssl_renew.sh ``` - -## Update dependencies - -To update all project dependencies automatically, run: - -``` -cd scripts -chmod +x update_dependencies.sh && ./update_dependencies.sh -``` - -Note: it is the responsibility of the developer to check that the update of the dependencies has not broken any functionality and each dependency maintains backwards compatibility. Use the script with care! - diff --git a/requirements.txt b/requirements.txt index 09d69c4b7..a8754311f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,8 @@ PyMySQL==1.1.0 pytest==8.1.1 python-dotenv==1.0.1 requests==2.31.0 -SQLAlchemy==2.0.28 +rosemary @ file:///app +SQLAlchemy==2.0.29 typing_extensions==4.10.0 Unidecode==1.3.8 urllib3==2.2.1 diff --git a/rosemary/__init__.py b/rosemary/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/rosemary/cli.py b/rosemary/cli.py new file mode 100644 index 000000000..b803b50f7 --- /dev/null +++ b/rosemary/cli.py @@ -0,0 +1,19 @@ +# rosemary/cli.py + +import click +from rosemary.commands.update import update +from rosemary.commands.info import info +from rosemary.commands.env import env + + +@click.group() +def cli(): + pass + + +cli.add_command(update) +cli.add_command(info) +cli.add_command(env) + +if __name__ == '__main__': + cli() diff --git a/rosemary/commands/__init__.py b/rosemary/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/rosemary/commands/env.py b/rosemary/commands/env.py new file mode 100644 index 000000000..b6e4d62a6 --- /dev/null +++ b/rosemary/commands/env.py @@ -0,0 +1,15 @@ +# rosemary/commands/env.py + +import click +from dotenv import dotenv_values + + +@click.command() +def env(): + """Displays the current .env file values.""" + # Load the .env file + env_values = dotenv_values(".env") + + # Display keys and values + for key, value in env_values.items(): + click.echo(f"{key}={value}") diff --git a/rosemary/commands/info.py b/rosemary/commands/info.py new file mode 100644 index 000000000..7ff7f8843 --- /dev/null +++ b/rosemary/commands/info.py @@ -0,0 +1,31 @@ +import click +import pkg_resources + + +def get_metadata_value(metadata_lines, key): + default_value = f"{key}: Unknown" + line = next((line for line in metadata_lines if line.startswith(key)), default_value) + return line.split(':', 1)[1].strip() if line != default_value else default_value.split(':', 1)[1].strip() + + +@click.command() +def info(): + """Displays information about the Rosemary CLI.""" + distribution = pkg_resources.get_distribution("rosemary") + + try: + metadata = distribution.get_metadata_lines('METADATA') + author = get_metadata_value(metadata, 'Author') + author_email = get_metadata_value(metadata, 'Author-email') + description = get_metadata_value(metadata, 'Summary') + except FileNotFoundError: + author, author_email, description = "Unknown", "Unknown", "Not available" + + name = distribution.project_name + version = distribution.version + + click.echo(f"Name: {name}") + click.echo(f"Version: {version}") + click.echo(f"Author: {author}") + click.echo(f"Author-email: {author_email}") + click.echo(f"Description: {description}") diff --git a/rosemary/commands/update.py b/rosemary/commands/update.py new file mode 100644 index 000000000..9ca4af556 --- /dev/null +++ b/rosemary/commands/update.py @@ -0,0 +1,29 @@ +# rosemary/commands/update.py + +import click +import subprocess +import os + + +@click.command() +def update(): + """This command updates pip, all packages, and updates requirements.txt.""" + try: + # Update pip + subprocess.check_call(['pip', 'install', '--upgrade', 'pip']) + + # Get the list of installed packages and update them + packages = subprocess.check_output(['pip', 'freeze']).decode('utf-8').split('\n') + for package in packages: + package_name = package.split('==')[0] + if package_name: # Check if the package name is not empty + subprocess.check_call(['pip', 'install', '--upgrade', package_name]) + + # Update requirements.txt + requirements_path = os.path.join(os.getcwd(), 'requirements.txt') + with open(requirements_path, 'w') as f: + subprocess.check_call(['pip', 'freeze'], stdout=f) + + click.echo('Update completed!') + except subprocess.CalledProcessError as e: + click.echo(f'Error during the update: {e}') diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..45cddb146 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +# setup.py + +from setuptools import setup, find_packages + +setup( + name='rosemary', + version='0.1.0', + packages=find_packages(), + include_package_data=True, + install_requires=[ + 'click', + 'python-dotenv', + ], + author='David Romero', + author_email='drorganvidez@us.es', + description="Rosemary is a CLI to be able to work on UVLHub development more easily.", + entry_points=''' + [console_scripts] + rosemary=rosemary.cli:cli + ''', +) From a922740df16120e6a052100aebeb994ddd05bd9c Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Sun, 24 Mar 2024 16:16:06 +0100 Subject: [PATCH 02/46] feat: Implement 'make:module' command in Rosemary --- README.md | 28 ++++++++- app/blueprints/zenodo/__init__.py | 3 + app/blueprints/zenodo/forms.py | 6 ++ app/blueprints/zenodo/models.py | 5 ++ app/blueprints/zenodo/repositories.py | 7 +++ app/blueprints/zenodo/routes.py | 7 +++ app/blueprints/zenodo/services.py | 7 +++ .../zenodo/templates/zenodo/index.html | 7 +++ rosemary/cli.py | 4 +- rosemary/commands/make_module.py | 57 +++++++++++++++++++ rosemary/templates/blueprint_forms.py.j2 | 6 ++ rosemary/templates/blueprint_init.py.j2 | 3 + rosemary/templates/blueprint_models.py.j2 | 5 ++ .../templates/blueprint_repositories.py.j2 | 7 +++ rosemary/templates/blueprint_routes.py.j2 | 7 +++ rosemary/templates/blueprint_services.py.j2 | 7 +++ .../blueprint_templates_index.html.j2 | 7 +++ 17 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 app/blueprints/zenodo/__init__.py create mode 100644 app/blueprints/zenodo/forms.py create mode 100644 app/blueprints/zenodo/models.py create mode 100644 app/blueprints/zenodo/repositories.py create mode 100644 app/blueprints/zenodo/routes.py create mode 100644 app/blueprints/zenodo/services.py create mode 100644 app/blueprints/zenodo/templates/zenodo/index.html create mode 100644 rosemary/commands/make_module.py create mode 100644 rosemary/templates/blueprint_forms.py.j2 create mode 100644 rosemary/templates/blueprint_init.py.j2 create mode 100644 rosemary/templates/blueprint_models.py.j2 create mode 100644 rosemary/templates/blueprint_repositories.py.j2 create mode 100644 rosemary/templates/blueprint_routes.py.j2 create mode 100644 rosemary/templates/blueprint_services.py.j2 create mode 100644 rosemary/templates/blueprint_templates_index.html.j2 diff --git a/README.md b/README.md index b78815a60..f39987be5 100644 --- a/README.md +++ b/README.md @@ -89,17 +89,39 @@ rosemary update Note: it is the responsibility of the developer to check that the update of the dependencies has not broken any functionality and each dependency maintains backwards compatibility. Use the script with care! -### Viewing Environment Variables +### Extending the Project with New Modules -To view the current `.env` file settings, use: +To quickly generate a new module within the project, including necessary boilerplate files +like `__init__.py`, `routes.py`, `models.py`, `repositories.py`, `services.py`, `forms.py`, +and a basic `index.html` template, you can use the `rosemary` CLI tool's `make:module` +command. This command will create a new blueprint structure ready for development. + +To create a new module, run the following command from the root of the project: + +``` +rosemary make:module +``` + +Replace `` with the desired name of your module. For example, to create a +module named "zenodo", you would run: ``` -rosemary env +rosemary make:module zenodo ``` + + +This command creates a new directory under `app/blueprints/` with the name of your module and sets up the initial files and directories needed to get started, including a dedicated `templates` directory for your module's templates. + +**Note:** If the module already exists, `rosemary` will simply notify you and not overwrite any existing files. + +This feature is designed to streamline the development process, making it easy to add new features to the project. + + ### Available Commands - `rosemary update`: Updates all project dependencies and the `requirements.txt` file. - `rosemary info`: Displays information about the Rosemary CLI, including version and author. +- `rosemary make:module `: Generates a new module with the specified name. - `rosemary env`: Displays the current environment variables from the `.env` file. ## Deploy in production (Docker Compose) diff --git a/app/blueprints/zenodo/__init__.py b/app/blueprints/zenodo/__init__.py new file mode 100644 index 000000000..513b92826 --- /dev/null +++ b/app/blueprints/zenodo/__init__.py @@ -0,0 +1,3 @@ +from flask import Blueprint + +zenodo_bp = Blueprint('zenodo', __name__, template_folder='templates') diff --git a/app/blueprints/zenodo/forms.py b/app/blueprints/zenodo/forms.py new file mode 100644 index 000000000..aa61d03e7 --- /dev/null +++ b/app/blueprints/zenodo/forms.py @@ -0,0 +1,6 @@ +from flask_wtf import FlaskForm +from wtforms import SubmitField + + +class ZenodoForm(FlaskForm): + submit = SubmitField('Save zenodo') diff --git a/app/blueprints/zenodo/models.py b/app/blueprints/zenodo/models.py new file mode 100644 index 000000000..be29e74b3 --- /dev/null +++ b/app/blueprints/zenodo/models.py @@ -0,0 +1,5 @@ +from app import db + + +class Zenodo(db.Model): + id = db.Column(db.Integer, primary_key=True) diff --git a/app/blueprints/zenodo/repositories.py b/app/blueprints/zenodo/repositories.py new file mode 100644 index 000000000..8ee608d98 --- /dev/null +++ b/app/blueprints/zenodo/repositories.py @@ -0,0 +1,7 @@ +from app.blueprints.zenodo.models import Zenodo +from app.repositories.BaseRepository import BaseRepository + + +class ZenodoRepository(BaseRepository): + def __init__(self): + super().__init__(Zenodo) diff --git a/app/blueprints/zenodo/routes.py b/app/blueprints/zenodo/routes.py new file mode 100644 index 000000000..e3a4ba8e8 --- /dev/null +++ b/app/blueprints/zenodo/routes.py @@ -0,0 +1,7 @@ +from flask import render_template +from app.blueprints.zenodo import zenodo_bp + + +@zenodo_bp.route('/zenodo', methods=['GET']) +def index(): + return render_template('zenodo/index.html') diff --git a/app/blueprints/zenodo/services.py b/app/blueprints/zenodo/services.py new file mode 100644 index 000000000..3fcb2e5b2 --- /dev/null +++ b/app/blueprints/zenodo/services.py @@ -0,0 +1,7 @@ +from app.blueprints.zenodo.repositories import ZenodoRepository +from app.services.BaseService import BaseService + + +class Zenodo(BaseService): + def __init__(self): + super().__init__(ZenodoRepository()) diff --git a/app/blueprints/zenodo/templates/zenodo/index.html b/app/blueprints/zenodo/templates/zenodo/index.html new file mode 100644 index 000000000..be0e42400 --- /dev/null +++ b/app/blueprints/zenodo/templates/zenodo/index.html @@ -0,0 +1,7 @@ +{% extends "base_template.html" %} + +{% block title %}View zenodo{% endblock %} + +{% block content %} + +{% endblock %} diff --git a/rosemary/cli.py b/rosemary/cli.py index b803b50f7..5f9a0bb58 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -3,7 +3,7 @@ import click from rosemary.commands.update import update from rosemary.commands.info import info -from rosemary.commands.env import env +from rosemary.commands.make_module import make_module @click.group() @@ -13,7 +13,7 @@ def cli(): cli.add_command(update) cli.add_command(info) -cli.add_command(env) +cli.add_command(make_module) if __name__ == '__main__': cli() diff --git a/rosemary/commands/make_module.py b/rosemary/commands/make_module.py new file mode 100644 index 000000000..863a70da9 --- /dev/null +++ b/rosemary/commands/make_module.py @@ -0,0 +1,57 @@ +import click +from jinja2 import Environment, FileSystemLoader, select_autoescape +import os + + +def pascalcase(s): + """Converts string to PascalCase.""" + return ''.join(word.capitalize() for word in s.split('_')) + + +def setup_jinja_env(): + """Configures and returns a Jinja environment.""" + env = Environment( + loader=FileSystemLoader(searchpath="./rosemary/templates"), + autoescape=select_autoescape(['html', 'xml', 'j2']) + ) + env.filters['pascalcase'] = pascalcase + return env + + +def render_and_write_file(env, template_name, filename, context): + """Renders a template and writes it to a specified file.""" + template = env.get_template(template_name) + content = template.render(context) + "\n" + with open(filename, 'w') as f: + f.write(content) + + +@click.command('make:module', help="Creates a new module with a given name.") +@click.argument('name') +def make_module(name): + blueprint_path = f'app/blueprints/{name}' + + if os.path.exists(blueprint_path): + click.echo(f"The module '{name}' already exists.") + return + + env = setup_jinja_env() + + files_and_templates = { + '__init__.py': 'blueprint_init.py.j2', + 'routes.py': 'blueprint_routes.py.j2', + 'models.py': 'blueprint_models.py.j2', + 'repositories.py': 'blueprint_repositories.py.j2', + 'services.py': 'blueprint_services.py.j2', + 'forms.py': 'blueprint_forms.py.j2', + os.path.join('templates', name, 'index.html'): 'blueprint_templates_index.html.j2' + } + + # Create necessary directories + os.makedirs(os.path.join(blueprint_path, 'templates', name), exist_ok=True) + + # Render and write files + for filename, template_name in files_and_templates.items(): + render_and_write_file(env, template_name, os.path.join(blueprint_path, filename), {'blueprint_name': name}) + + click.echo(f"Module '{name}' created successfully.") diff --git a/rosemary/templates/blueprint_forms.py.j2 b/rosemary/templates/blueprint_forms.py.j2 new file mode 100644 index 000000000..3f774d8dd --- /dev/null +++ b/rosemary/templates/blueprint_forms.py.j2 @@ -0,0 +1,6 @@ +from flask_wtf import FlaskForm +from wtforms import SubmitField + + +class {{ blueprint_name | pascalcase }}Form(FlaskForm): + submit = SubmitField('Save {{ blueprint_name }}') diff --git a/rosemary/templates/blueprint_init.py.j2 b/rosemary/templates/blueprint_init.py.j2 new file mode 100644 index 000000000..0ac9ee4a1 --- /dev/null +++ b/rosemary/templates/blueprint_init.py.j2 @@ -0,0 +1,3 @@ +from flask import Blueprint + +{{ blueprint_name }}_bp = Blueprint('{{ blueprint_name }}', __name__, template_folder='templates') diff --git a/rosemary/templates/blueprint_models.py.j2 b/rosemary/templates/blueprint_models.py.j2 new file mode 100644 index 000000000..8ca7322d3 --- /dev/null +++ b/rosemary/templates/blueprint_models.py.j2 @@ -0,0 +1,5 @@ +from app import db + + +class {{ blueprint_name | pascalcase }}(db.Model): + id = db.Column(db.Integer, primary_key=True) \ No newline at end of file diff --git a/rosemary/templates/blueprint_repositories.py.j2 b/rosemary/templates/blueprint_repositories.py.j2 new file mode 100644 index 000000000..1f627c368 --- /dev/null +++ b/rosemary/templates/blueprint_repositories.py.j2 @@ -0,0 +1,7 @@ +from app.blueprints.{{ blueprint_name }}.models import {{ blueprint_name | pascalcase }} +from app.repositories.BaseRepository import BaseRepository + + +class {{ blueprint_name | pascalcase }}Repository(BaseRepository): + def __init__(self): + super().__init__({{ blueprint_name | pascalcase }}) diff --git a/rosemary/templates/blueprint_routes.py.j2 b/rosemary/templates/blueprint_routes.py.j2 new file mode 100644 index 000000000..9884ba9d3 --- /dev/null +++ b/rosemary/templates/blueprint_routes.py.j2 @@ -0,0 +1,7 @@ +from flask import render_template +from app.blueprints.{{ blueprint_name }} import {{ blueprint_name }}_bp + + +@{{ blueprint_name }}_bp.route('/{{ blueprint_name }}', methods=['GET']) +def index(): + return render_template('{{ blueprint_name }}/index.html') \ No newline at end of file diff --git a/rosemary/templates/blueprint_services.py.j2 b/rosemary/templates/blueprint_services.py.j2 new file mode 100644 index 000000000..3b28efa59 --- /dev/null +++ b/rosemary/templates/blueprint_services.py.j2 @@ -0,0 +1,7 @@ +from app.blueprints.{{ blueprint_name }}.repositories import {{ blueprint_name | pascalcase }}Repository +from app.services.BaseService import BaseService + + +class {{ blueprint_name | pascalcase }}(BaseService): + def __init__(self): + super().__init__({{ blueprint_name | pascalcase }}Repository()) \ No newline at end of file diff --git a/rosemary/templates/blueprint_templates_index.html.j2 b/rosemary/templates/blueprint_templates_index.html.j2 new file mode 100644 index 000000000..0c057de33 --- /dev/null +++ b/rosemary/templates/blueprint_templates_index.html.j2 @@ -0,0 +1,7 @@ +{% raw %}{%{% endraw %} extends "base_template.html" {% raw %}%}{% endraw %} + +{% raw %}{%{% endraw %} block title {% raw %}%}{% endraw %}View {{ blueprint_name }}{% raw %}{%{% endraw %} endblock {% raw %}%}{% endraw %} + +{% raw %}{%{% endraw %} block content {% raw %}%}{% endraw %} + +{% raw %}{%{% endraw %} endblock {% raw %}%}{% endraw %} From 809fdc60ab1465841960f696f199347f1bab0121 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Sun, 24 Mar 2024 16:19:05 +0100 Subject: [PATCH 03/46] fix: Fix bug in Rosemary --- rosemary/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rosemary/cli.py b/rosemary/cli.py index 5f9a0bb58..a0c28fe3e 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -4,6 +4,7 @@ from rosemary.commands.update import update from rosemary.commands.info import info from rosemary.commands.make_module import make_module +from rosemary.commands.env import env @click.group() @@ -14,6 +15,7 @@ def cli(): cli.add_command(update) cli.add_command(info) cli.add_command(make_module) +cli.add_command(env) if __name__ == '__main__': cli() From 4850bc9cf080e103e40b6a4e9c89da15c732029f Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 02:00:20 +0100 Subject: [PATCH 04/46] feat: Implement first test in a module --- .github/workflows/tests.yml | 2 +- Dockerfile.dev | 9 +- Dockerfile.mariadb | 5 ++ app/__init__.py | 52 ++++++++--- app/blueprints/__init__.py | 0 app/blueprints/auth/models.py | 5 ++ app/blueprints/auth/routes.py | 22 ++--- app/blueprints/auth/services.py | 14 +++ app/blueprints/profile/tests/__init__.py | 0 app/blueprints/profile/tests/test_unit.py | 102 ++++++++++++++++++++++ app/blueprints/public/routes.py | 9 ++ docker-compose.dev.yml | 16 +++- scripts/init-db.sh | 11 +++ scripts/wait-for-db.sh | 3 +- 14 files changed, 216 insertions(+), 34 deletions(-) create mode 100644 Dockerfile.mariadb create mode 100644 app/blueprints/__init__.py create mode 100644 app/blueprints/auth/services.py create mode 100644 app/blueprints/profile/tests/__init__.py create mode 100644 app/blueprints/profile/tests/test_unit.py create mode 100644 scripts/init-db.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 685e99bc4..f0520083c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,4 +43,4 @@ jobs: export MYSQL_DATABASE=fmlibdb export MYSQL_USER=fmlibuser export MYSQL_PASSWORD=fmlibpass - pytest app/tests/units.py + pytest app/blueprints/ diff --git a/Dockerfile.dev b/Dockerfile.dev index 2e0426cad..c2166395e 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -4,8 +4,8 @@ FROM python:3.11-alpine # Set this environment variable to suppress the "Running as root" warning from pip ENV PIP_ROOT_USER_ACTION=ignore -# Install the MySQL client to be able to use it in the standby script. -RUN apk add --no-cache mysql-client +# Install the MariaDB client to be able to use it in the standby script. +RUN apk add --no-cache mariadb-client # Set the working directory in the container to /app WORKDIR /app @@ -19,6 +19,9 @@ COPY requirements.txt . # Copy the wait-for-db.sh script and set execution permissions COPY --chmod=+x scripts/wait-for-db.sh ./scripts/ +# Copy the init-db.sh script and set execution permissions +COPY --chmod=+x scripts/init-db.sh ./scripts/ + # Install any needed packages specified in requirements.txt RUN pip install --no-cache-dir -r requirements.txt @@ -32,4 +35,4 @@ RUN pip install --no-cache-dir --upgrade pip EXPOSE 5000 # Sets the CMD command to correctly execute the wait-for-db.sh script -CMD sh ./scripts/wait-for-db.sh && flask db upgrade && flask run --host=0.0.0.0 --port=5000 --reload --debug +CMD sh ./scripts/wait-for-db.sh && sh ./scripts/init-db.sh && flask db upgrade && flask run --host=0.0.0.0 --port=5000 --reload --debug diff --git a/Dockerfile.mariadb b/Dockerfile.mariadb new file mode 100644 index 000000000..03a58892a --- /dev/null +++ b/Dockerfile.mariadb @@ -0,0 +1,5 @@ +FROM mariadb:latest + +RUN apt-get update && \ + apt-get install -y mariadb-client && \ + rm -rf /var/lib/apt/lists/* diff --git a/app/__init__.py b/app/__init__.py index 0dc231826..6dd06d712 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -17,27 +17,52 @@ migrate = Migrate() -def create_app(config_name=None): - app = Flask(__name__) - app.secret_key = secrets.token_bytes() - - # Database configuration - app.config['SQLALCHEMY_DATABASE_URI'] = ( +class Config: + SECRET_KEY = os.getenv('SECRET_KEY', secrets.token_bytes()) + SQLALCHEMY_DATABASE_URI = ( f"mysql+pymysql://{os.getenv('MARIADB_USER', 'default_user')}:" f"{os.getenv('MARIADB_PASSWORD', 'default_password')}@" f"{os.getenv('MARIADB_HOSTNAME', 'localhost')}:" f"{os.getenv('MARIADB_PORT', '3306')}/" f"{os.getenv('MARIADB_DATABASE', 'default_db')}" ) + SQLALCHEMY_TRACK_MODIFICATIONS = False + TIMEZONE = 'Europe/Madrid' + TEMPLATES_AUTO_RELOAD = True + + +class DevelopmentConfig(Config): + DEBUG = True + + +class TestingConfig(Config): + TESTING = True + SECRET_KEY = os.getenv('SECRET_KEY', 'secret_test_key') + SQLALCHEMY_DATABASE_URI = ( + f"mysql+pymysql://{os.getenv('MARIADB_USER', 'default_user')}:" + f"{os.getenv('MARIADB_PASSWORD', 'default_password')}@" + f"{os.getenv('MARIADB_HOSTNAME', 'localhost')}:" + f"{os.getenv('MARIADB_PORT', '3306')}/" + f"{os.getenv('MARIADB_TEST_DATABASE', 'default_db')}" + ) + UPLOAD_FOLDER = 'uploads' + WTF_CSRF_ENABLED = False - # Timezone - app.config['TIMEZONE'] = 'Europe/Madrid' - # Templates configuration - app.config['TEMPLATES_AUTO_RELOAD'] = True +class ProductionConfig(Config): + pass + + +def create_app(config_name='development'): + app = Flask(__name__) - # Uploads feature models configuration - app.config['UPLOAD_FOLDER'] = os.path.join(app.root_path, 'uploads') + # Load configuration + if config_name == 'testing': + app.config.from_object(TestingConfig) + elif config_name == 'production': + app.config.from_object(ProductionConfig) + else: + app.config.from_object(DevelopmentConfig) # Initialize SQLAlchemy and Migrate with the app db.init_app(app) @@ -45,7 +70,8 @@ def create_app(config_name=None): # Register blueprints register_blueprints(app) - print_registered_blueprints(app) + if config_name == 'development': + print_registered_blueprints(app) from flask_login import LoginManager login_manager = LoginManager() diff --git a/app/blueprints/__init__.py b/app/blueprints/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/blueprints/auth/models.py b/app/blueprints/auth/models.py index dc1ad6645..b70f14b3d 100644 --- a/app/blueprints/auth/models.py +++ b/app/blueprints/auth/models.py @@ -16,6 +16,11 @@ class User(db.Model, UserMixin): data_sets = db.relationship('DataSet', backref='user', lazy=True) profile = db.relationship('UserProfile', backref='user', uselist=False) + def __init__(self, **kwargs): + super(User, self).__init__(**kwargs) + if 'password' in kwargs: + self.set_password(kwargs['password']) + def __repr__(self): return f'' diff --git a/app/blueprints/auth/routes.py b/app/blueprints/auth/routes.py index 212621922..26ddc57b0 100644 --- a/app/blueprints/auth/routes.py +++ b/app/blueprints/auth/routes.py @@ -1,8 +1,9 @@ -from flask import (render_template, redirect, url_for) +from flask import (render_template, redirect, url_for, flash, request) from flask_login import current_user, login_user, logout_user from app.blueprints.auth import auth_bp from app.blueprints.auth.forms import SignupForm, LoginForm +from app.blueprints.auth.services import AuthenticationService from app.blueprints.profile.models import UserProfile @@ -45,16 +46,15 @@ def login(): if current_user.is_authenticated: return redirect(url_for('public.index')) form = LoginForm() - if form.validate_on_submit(): - from app.blueprints.auth.models import User - user = User.get_by_email(form.email.data) - - if user is not None and user.check_password(form.password.data): - login_user(user, remember=form.remember_me.data) - return redirect(url_for('public.index')) - else: - error = 'Invalid credentials' - return render_template("auth/login_form.html", form=form, error=error) + if request.method == 'POST': + if form.validate_on_submit(): + email = form.email.data + password = form.password.data + if AuthenticationService.login(email, password): + return redirect(url_for('public.index')) + else: + error = 'Invalid credentials' + return render_template("auth/login_form.html", form=form, error=error) return render_template('auth/login_form.html', form=form) diff --git a/app/blueprints/auth/services.py b/app/blueprints/auth/services.py new file mode 100644 index 000000000..0032bbc6b --- /dev/null +++ b/app/blueprints/auth/services.py @@ -0,0 +1,14 @@ +from flask_login import login_user + +from app.blueprints.auth.models import User + + +class AuthenticationService: + + @staticmethod + def login(email, password, remember=True): + user = User.get_by_email(email) + if user is not None and user.check_password(password): + login_user(user, remember=remember) + return True + return False diff --git a/app/blueprints/profile/tests/__init__.py b/app/blueprints/profile/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/blueprints/profile/tests/test_unit.py b/app/blueprints/profile/tests/test_unit.py new file mode 100644 index 000000000..d14ddcc6a --- /dev/null +++ b/app/blueprints/profile/tests/test_unit.py @@ -0,0 +1,102 @@ +import pytest +from flask import url_for +from app import create_app, db +from app.blueprints.auth.models import User + + +@pytest.fixture(scope='module') +def test_client(): + flask_app = create_app('testing') + + with flask_app.test_client() as testing_client: + with flask_app.app_context(): + db.create_all() + + user_test = User(email='test@example.com') + user_test.set_password('test1234') + db.session.add(user_test) + db.session.commit() + + yield testing_client + + db.session.remove() + db.drop_all() + + +def login(test_client, email, password): + """ + Authenticates the user with the credentials provided. + + Args: + test_client: Flask test client. + email (str): User's email address. + password (str): User's password. + + Returns: + response: POST login request response. + """ + response = test_client.post('/login', data=dict( + email=email, + password=password + ), follow_redirects=True) + return response + + +def logout(test_client): + """ + Logs out the user. + + Args: + test_client: Flask test client. + + Returns: + response: Response to GET request to log out. + """ + return test_client.get('/logout', follow_redirects=True) + + +def test_login_success(test_client): + response = test_client.post('/login', data=dict( + email='test@example.com', + password='test1234' + ), follow_redirects=True) + + assert response.request.path != url_for('auth.login'), "Login was unsuccessful" + + test_client.get('/logout', follow_redirects=True) + + +def test_login_unsuccessful_bad_email(test_client): + response = test_client.post('/login', data=dict( + email='bademail@example.com', + password='test1234' + ), follow_redirects=True) + + assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + + test_client.get('/logout', follow_redirects=True) + + +def test_login_unsuccessful_bad_password(test_client): + response = test_client.post('/login', data=dict( + email='test@example.com', + password='basspassword' + ), follow_redirects=True) + + assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + + test_client.get('/logout', follow_redirects=True) + + +def test_edit_profile_page_get(test_client): + """ + Tests access to the profile editing page via a GET request. + """ + login_response = login(test_client, 'test@example.com', 'test1234') + assert login_response.status_code == 200, "Login was unsuccessful." + + response = test_client.get('/profile/edit') + assert response.status_code == 200, "The profile editing page could not be accessed." + assert b"Edit profile" in response.data, "The expected content is not present on the page" + + logout(test_client) diff --git a/app/blueprints/public/routes.py b/app/blueprints/public/routes.py index 5832855b3..40c3a7109 100644 --- a/app/blueprints/public/routes.py +++ b/app/blueprints/public/routes.py @@ -1,4 +1,7 @@ import logging + +from flask_login import login_required + import app from flask import render_template @@ -23,3 +26,9 @@ def index(): datasets=latest_datasets, datasets_counter=datasets_counter, feature_models_counter=feature_models_counter) + + +@public_bp.route('/secret') +@login_required +def secret(): + return "Esto es secreto!" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 44aae76c1..fb7a3e68c 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -8,22 +8,28 @@ services: context: . dockerfile: Dockerfile.dev volumes: - - .:/app + - type: bind + source: . + target: /app expose: - "5000" environment: FLASK_ENV: development MARIADB_HOSTNAME: ${MARIADB_HOSTNAME} + MARIADB_DATABASE: ${MARIADB_DATABASE} + MARIADB_TEST_DATABASE: ${MARIADB_TEST_DATABASE} MARIADB_PORT: ${MARIADB_PORT} MARIADB_USER: ${MARIADB_USER} MARIADB_PASSWORD: ${MARIADB_PASSWORD} + MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD} depends_on: - db db: container_name: mariadb_container - image: mariadb:latest - command: --default-authentication-plugin=mysql_native_password + build: + context: ./ + dockerfile: Dockerfile.mariadb restart: always environment: MARIADB_DATABASE: ${MARIADB_DATABASE} @@ -39,7 +45,9 @@ services: container_name: nginx_web_server image: nginx:latest volumes: - - ./nginx/nginx.dev.conf:/etc/nginx/nginx.conf + - type: bind + source: ./nginx/nginx.dev.conf + target: /etc/nginx/nginx.conf ports: - "80:80" depends_on: diff --git a/scripts/init-db.sh b/scripts/init-db.sh new file mode 100644 index 000000000..c9b115c21 --- /dev/null +++ b/scripts/init-db.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "(testing) Hostname: $MARIADB_HOSTNAME, Port: $MARIADB_PORT, User: $MARIADB_USER, Test DB: $MARIADB_TEST_DATABASE" + +echo "MariaDB is up - creating test database if it doesn't exist" + +# Create the test database if it does not exist +mariadb -h "$MARIADB_HOSTNAME" -P "$MARIADB_PORT" -u root -p"$MARIADB_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS \`${MARIADB_TEST_DATABASE}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; GRANT ALL PRIVILEGES ON \`${MARIADB_TEST_DATABASE}\`.* TO '$MARIADB_USER'@'%'; FLUSH PRIVILEGES;" + + +echo "Test database created and privileges granted" diff --git a/scripts/wait-for-db.sh b/scripts/wait-for-db.sh index 6b478e655..c4b9e4e82 100644 --- a/scripts/wait-for-db.sh +++ b/scripts/wait-for-db.sh @@ -2,11 +2,10 @@ echo "Hostname: $MARIADB_HOSTNAME, Port: $MARIADB_PORT, User: $MARIADB_USER" -until mysql -h "$MARIADB_HOSTNAME" -P "$MARIADB_PORT" -u"$MARIADB_USER" -p"$MARIADB_PASSWORD" -e 'SELECT 1' &> /dev/null +until mariadb -h "$MARIADB_HOSTNAME" -P "$MARIADB_PORT" -u"$MARIADB_USER" -p"$MARIADB_PASSWORD" -e 'SELECT 1' &> /dev/null do echo "MariaDB is unavailable - sleeping" sleep 1 done echo "MariaDB is up - executing command" -exec "$@" From 3ccdec49739709275fb0a12c94f64df862fbc377 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 16:24:58 +0100 Subject: [PATCH 05/46] feat: Implements rosemary test command --- .github/workflows/tests.yml | 37 ++++++++++++++++++++--------------- Dockerfile.prod | 23 ++++++++++++---------- README.md | 30 ++++++++++++++++------------ app/__init__.py | 4 ++++ app/blueprints/auth/routes.py | 2 +- rosemary/cli.py | 17 +++++++++++++--- rosemary/commands/test.py | 24 +++++++++++++++++++++++ 7 files changed, 94 insertions(+), 43 deletions(-) create mode 100644 rosemary/commands/test.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f0520083c..32fd81183 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,26 +2,29 @@ name: Flask CI on: push: - branches: [ main ] + branches: [ main, develop ] pull_request: - branches: [ main ] + branches: [ main, develop ] jobs: build: - runs-on: ubuntu-latest services: - mysql: - image: mysql:5.7 + mariadb: + image: mariadb:latest env: - MYSQL_ROOT_PASSWORD: fmlibrootpass - MYSQL_DATABASE: fmlibdb - MYSQL_USER: fmlibuser - MYSQL_PASSWORD: fmlibpass + MARIADB_DATABASE: uvlhubdb_test + MARIADB_USER: uvlhub + MARIADB_PASSWORD: uvlhub_password + MARIADB_ROOT_PASSWORD: uvlhub_root_password ports: - 3306:3306 - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + options: >- + --health-cmd="mysqladmin ping -h localhost -u root --password=vRwgXWu0ns" + --health-interval=10s + --health-timeout=5s + --health-retries=5 steps: - uses: actions/checkout@v2 @@ -37,10 +40,12 @@ jobs: pip install -r requirements.txt - name: Run Tests + env: + FLASK_ENV: testing + MARIADB_HOSTNAME: 127.0.0.1 + MARIADB_PORT: 3306 + MARIADB_DATABASE: uvlhubdb_test + MARIADB_USER: uvlhub_user + MARIADB_PASSWORD: uvlhub_password run: | - export MYSQL_HOSTNAME=127.0.0.1 - export MYSQL_PORT=3306 - export MYSQL_DATABASE=fmlibdb - export MYSQL_USER=fmlibuser - export MYSQL_PASSWORD=fmlibpass - pytest app/blueprints/ + rosemary test diff --git a/Dockerfile.prod b/Dockerfile.prod index 5bd264051..caffdad3e 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,8 +1,9 @@ -# Use an official Python runtime as a parent image +# Use an official Python runtime as a parent image, Alpine version for a lighter footprint FROM python:3.11-alpine -# Install the MySQL client to be able to use it in the standby script. -RUN apk add --no-cache mysql-client +# Install MySQL client and temporary build dependencies +RUN apk add --no-cache mysql-client \ + && apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev libffi-dev openssl-dev # Set the working directory in the container to /app WORKDIR /app @@ -10,17 +11,16 @@ WORKDIR /app # Copy the contents of the local app/ directory to the /app directory in the container COPY app/ ./app -# Copy requirements.txt at the /app working directory +# Copy requirements.txt into the working directory /app COPY requirements.txt . # Copy the wait-for-db.sh script and set execution permissions COPY --chmod=+x scripts/wait-for-db.sh ./scripts/ -# Install any needed packages specified in requirements.txt -RUN pip install --no-cache-dir -r requirements.txt - -# Update pip -RUN pip install --no-cache-dir --upgrade pip +# Install any needed packages specified in requirements.txt and upgrade pip +RUN pip install --no-cache-dir -r requirements.txt \ + && pip install --no-cache-dir --upgrade pip \ + && apk del .build-deps # Copy the migration scripts to the /app directory in the container COPY migrations/ ./migrations @@ -28,5 +28,8 @@ COPY migrations/ ./migrations # Expose port 5000 EXPOSE 5000 +# Set environment variables for production +ENV FLASK_ENV=production + # Run the database migrations and then start the application with Gunicorn -CMD sh ./scripts/wait-for-db.sh && flask db upgrade && gunicorn --bind 0.0.0.0:5000 app:app --log-level debug --timeout 3600 +CMD sh ./scripts/wait-for-db.sh && flask db upgrade && gunicorn --bind 0.0.0.0:5000 app:app --log-level info --timeout 3600 diff --git a/README.md b/README.md index f39987be5..e280cff17 100644 --- a/README.md +++ b/README.md @@ -56,14 +56,6 @@ flask db migrate flask db upgrade ``` -### Tests - -To run unit test, please enter inside `web` container: - -``` -pytest app/tests/units.py -``` - ## Using Rosemary CLI `Rosemary` is a CLI tool developed to facilitate project management and development tasks. @@ -116,13 +108,25 @@ This command creates a new directory under `app/blueprints/` with the name of yo This feature is designed to streamline the development process, making it easy to add new features to the project. +### Testing All Modules -### Available Commands +To run tests across all modules in the project, you can use the following command: -- `rosemary update`: Updates all project dependencies and the `requirements.txt` file. -- `rosemary info`: Displays information about the Rosemary CLI, including version and author. -- `rosemary make:module `: Generates a new module with the specified name. -- `rosemary env`: Displays the current environment variables from the `.env` file. +``` +rosemary test +``` + +This command will execute all tests found within the app/blueprints directory, covering all the modules of the project. + +### Testing a Specific Module + +If you're focusing on a particular module and want to run tests only for that module, you can specify the module +name as an argument to the rosemary test command. For example, to run tests only for the zenodo module, you would +use: + +``` +rosemary test zenodo +``` ## Deploy in production (Docker Compose) diff --git a/app/__init__.py b/app/__init__.py index 6dd06d712..e7fd9aab6 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -56,6 +56,10 @@ class ProductionConfig(Config): def create_app(config_name='development'): app = Flask(__name__) + # If config_name is not provided, use the environment variable FLASK_ENV + if config_name is None: + config_name = os.getenv('FLASK_ENV', 'development') + # Load configuration if config_name == 'testing': app.config.from_object(TestingConfig) diff --git a/app/blueprints/auth/routes.py b/app/blueprints/auth/routes.py index 26ddc57b0..b8bfae92f 100644 --- a/app/blueprints/auth/routes.py +++ b/app/blueprints/auth/routes.py @@ -1,4 +1,4 @@ -from flask import (render_template, redirect, url_for, flash, request) +from flask import (render_template, redirect, url_for, request) from flask_login import current_user, login_user, logout_user from app.blueprints.auth import auth_bp diff --git a/rosemary/cli.py b/rosemary/cli.py index a0c28fe3e..13cecfd24 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,14 +1,24 @@ -# rosemary/cli.py - import click from rosemary.commands.update import update from rosemary.commands.info import info from rosemary.commands.make_module import make_module from rosemary.commands.env import env +from rosemary.commands.test import test + + +class RosemaryCLI(click.Group): + def get_command(self, ctx, cmd_name): + rv = super().get_command(ctx, cmd_name) + if rv is not None: + return rv + click.echo(f"No such command '{cmd_name}'.") + click.echo("Try 'rosemary --help' for a list of available commands.") + return None -@click.group() +@click.group(cls=RosemaryCLI) def cli(): + """A CLI tool to help with project management.""" pass @@ -16,6 +26,7 @@ def cli(): cli.add_command(info) cli.add_command(make_module) cli.add_command(env) +cli.add_command(test) if __name__ == '__main__': cli() diff --git a/rosemary/commands/test.py b/rosemary/commands/test.py new file mode 100644 index 000000000..465f392c6 --- /dev/null +++ b/rosemary/commands/test.py @@ -0,0 +1,24 @@ +import click +import subprocess +import os + + +@click.command('test', help="Runs pytest on the blueprints directory or a specific module.") +@click.argument('module_name', required=False) +def test(module_name): + base_path = 'app/blueprints' + test_path = base_path + + if module_name: + test_path = os.path.join(base_path, module_name) + if not os.path.exists(test_path): + click.echo(f"Module '{module_name}' does not exist.") + return + click.echo(f"Running tests for the '{module_name}' module...") + else: + click.echo("Running tests for all modules...") + + try: + subprocess.run(['pytest', '-v', test_path], check=True) + except subprocess.CalledProcessError as e: + click.echo(f"Error running tests: {e}") From 9413d95598bc676e272e7dbbbab2a7ea349b0543 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 16:29:05 +0100 Subject: [PATCH 06/46] fix: Fix test CI --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 32fd81183..73d2028a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,13 +15,13 @@ jobs: image: mariadb:latest env: MARIADB_DATABASE: uvlhubdb_test - MARIADB_USER: uvlhub + MARIADB_USER: uvlhub_user MARIADB_PASSWORD: uvlhub_password MARIADB_ROOT_PASSWORD: uvlhub_root_password ports: - 3306:3306 options: >- - --health-cmd="mysqladmin ping -h localhost -u root --password=vRwgXWu0ns" + --health-cmd="mysqladmin ping -h localhost -u root --password=uvlhub_root_password" --health-interval=10s --health-timeout=5s --health-retries=5 From 0a9f3f8700ab4bf47dd6de9cff1f0f8cc5236813 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 16:42:01 +0100 Subject: [PATCH 07/46] fix: Fix test CI --- .github/workflows/tests.yml | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 73d2028a4..39bd8915b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,29 +10,25 @@ jobs: build: runs-on: ubuntu-latest - services: - mariadb: - image: mariadb:latest - env: - MARIADB_DATABASE: uvlhubdb_test - MARIADB_USER: uvlhub_user - MARIADB_PASSWORD: uvlhub_password - MARIADB_ROOT_PASSWORD: uvlhub_root_password - ports: - - 3306:3306 - options: >- - --health-cmd="mysqladmin ping -h localhost -u root --password=uvlhub_root_password" - --health-interval=10s - --health-timeout=5s - --health-retries=5 - steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + + - name: Set up MariaDB + uses: getong/mariadb-action@v1.1 + with: + host port: 3600 + container port: 3600 + character set server: 'utf8' + collation server: 'utf8_general_ci' + mariadb version: 'latest' + mysql database: 'uvlhubdb_test' + mysql root password: 'uvlhub_root_password' + mysql user: 'uvlhub_user' + mysql password: 'uvlhub_password' - - name: Set up Python 3.9 - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: '3.11' - name: Install dependencies run: | From c43e8c95c4b394edb34389f0d12bd25eb22b87db Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:05:27 +0100 Subject: [PATCH 08/46] fix: Fix test CI --- .github/workflows/tests.yml | 4 +--- scripts/check_env_and_install.sh | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 scripts/check_env_and_install.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39bd8915b..31678f876 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,9 +31,7 @@ jobs: python-version: '3.11' - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + run: bash ./scripts/check_env_and_install.sh - name: Run Tests env: diff --git a/scripts/check_env_and_install.sh b/scripts/check_env_and_install.sh new file mode 100644 index 000000000..0c227a6af --- /dev/null +++ b/scripts/check_env_and_install.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ -n "$GITHUB_ACTIONS" ]; then + sed -i 's|file:///app|file://.'$GITHUB_WORKSPACE'/|' requirements.txt +fi + +python -m pip install --upgrade pip +pip install -r requirements.txt From 87d7fcdd06609f1e0f2e2910b9cb33e006e48c43 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:07:57 +0100 Subject: [PATCH 09/46] fix: Fix test CI --- scripts/check_env_and_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_env_and_install.sh b/scripts/check_env_and_install.sh index 0c227a6af..e4f9f8e25 100644 --- a/scripts/check_env_and_install.sh +++ b/scripts/check_env_and_install.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -n "$GITHUB_ACTIONS" ]; then - sed -i 's|file:///app|file://.'$GITHUB_WORKSPACE'/|' requirements.txt + sed -i 's|file:///app|file:///|' requirements.txt fi python -m pip install --upgrade pip From e134f89280c5b5ae6230fb63e78f4f72950c0f60 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:10:58 +0100 Subject: [PATCH 10/46] fix: Fix test CI --- .github/workflows/tests.yml | 10 ++++++++-- scripts/check_env_and_install.sh | 8 -------- 2 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 scripts/check_env_and_install.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 31678f876..ffd99eb5b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,8 +30,14 @@ jobs: with: python-version: '3.11' + - name: Prepare environment + run: | + sed -i '/rosemary @ file:\/\/\/app/d' requirements.txt + - name: Install dependencies - run: bash ./scripts/check_env_and_install.sh + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - name: Run Tests env: @@ -42,4 +48,4 @@ jobs: MARIADB_USER: uvlhub_user MARIADB_PASSWORD: uvlhub_password run: | - rosemary test + pytest app/blueprints/ diff --git a/scripts/check_env_and_install.sh b/scripts/check_env_and_install.sh deleted file mode 100644 index e4f9f8e25..000000000 --- a/scripts/check_env_and_install.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if [ -n "$GITHUB_ACTIONS" ]; then - sed -i 's|file:///app|file:///|' requirements.txt -fi - -python -m pip install --upgrade pip -pip install -r requirements.txt From 121b566a1b7ae9e40b36b209d69e52b80c951478 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:18:37 +0100 Subject: [PATCH 11/46] fix: Fix test CI --- .github/workflows/tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ffd99eb5b..58c3ad802 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,22 +10,22 @@ jobs: build: runs-on: ubuntu-latest + services: + mariadb: + image: mariadb:latest + env: + MYSQL_ALLOW_EMPTY_PASSWORD: false + MYSQL_USER: uvlhub_user + MYSQL_PASSWORD: uvlhub_password + MYSQL_ROOT_PASSWORD: uvlhub_root_password + MYSQL_DATABASE: uvlhubdb_test + ports: + - 3305:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: - uses: actions/checkout@v4 - - name: Set up MariaDB - uses: getong/mariadb-action@v1.1 - with: - host port: 3600 - container port: 3600 - character set server: 'utf8' - collation server: 'utf8_general_ci' - mariadb version: 'latest' - mysql database: 'uvlhubdb_test' - mysql root password: 'uvlhub_root_password' - mysql user: 'uvlhub_user' - mysql password: 'uvlhub_password' - - uses: actions/setup-python@v5 with: python-version: '3.11' From c4010a3f72e12f7d2d58eca6e22c1db7e04e8abb Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:24:59 +0100 Subject: [PATCH 12/46] fix: Fix test CI --- .github/workflows/tests.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 58c3ad802..75d83776e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,8 +20,12 @@ jobs: MYSQL_ROOT_PASSWORD: uvlhub_root_password MYSQL_DATABASE: uvlhubdb_test ports: - - 3305:3306 - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping -h localhost" + --health-interval=10s + --health-timeout=10s + --health-retries=6 steps: - uses: actions/checkout@v4 @@ -42,7 +46,7 @@ jobs: - name: Run Tests env: FLASK_ENV: testing - MARIADB_HOSTNAME: 127.0.0.1 + MARIADB_HOSTNAME: mariadb MARIADB_PORT: 3306 MARIADB_DATABASE: uvlhubdb_test MARIADB_USER: uvlhub_user From fe87ccfee6b9a5f98c7283cbfa95fb43f5220978 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:34:02 +0100 Subject: [PATCH 13/46] fix: Fix test CI --- .github/workflows/tests.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 75d83776e..4bdb44199 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,25 +7,20 @@ on: branches: [ main, develop ] jobs: - build: + pytest: runs-on: ubuntu-latest services: mariadb: image: mariadb:latest env: - MYSQL_ALLOW_EMPTY_PASSWORD: false MYSQL_USER: uvlhub_user MYSQL_PASSWORD: uvlhub_password MYSQL_ROOT_PASSWORD: uvlhub_root_password MYSQL_DATABASE: uvlhubdb_test ports: - 3306:3306 - options: >- - --health-cmd="mysqladmin ping -h localhost" - --health-interval=10s - --health-timeout=10s - --health-retries=6 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - uses: actions/checkout@v4 @@ -46,7 +41,7 @@ jobs: - name: Run Tests env: FLASK_ENV: testing - MARIADB_HOSTNAME: mariadb + MARIADB_HOSTNAME: 127.0.0.1 MARIADB_PORT: 3306 MARIADB_DATABASE: uvlhubdb_test MARIADB_USER: uvlhub_user From 3721020c9fe2a9dc7d9a084cab37d941668317ba Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:36:20 +0100 Subject: [PATCH 14/46] fix: Fix test CI --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4bdb44199..ce3323324 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest services: - mariadb: - image: mariadb:latest + mysql: + image: mysql:5.7 env: - MYSQL_USER: uvlhub_user - MYSQL_PASSWORD: uvlhub_password MYSQL_ROOT_PASSWORD: uvlhub_root_password MYSQL_DATABASE: uvlhubdb_test + MYSQL_USER: uvlhub_user + MYSQL_PASSWORD: uvl_password ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 From 8196dc4bb49102d243db1c28254d0c82108bf554 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:38:15 +0100 Subject: [PATCH 15/46] fix: Fix test CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce3323324..2f679a83a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: MYSQL_ROOT_PASSWORD: uvlhub_root_password MYSQL_DATABASE: uvlhubdb_test MYSQL_USER: uvlhub_user - MYSQL_PASSWORD: uvl_password + MYSQL_PASSWORD: uvlhub_password ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 From 044ff5adfa4f36c08fcb213f553fa553a6813af4 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 17:40:43 +0100 Subject: [PATCH 16/46] fix: Fix test CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2f679a83a..d65d71ee5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,7 +43,7 @@ jobs: FLASK_ENV: testing MARIADB_HOSTNAME: 127.0.0.1 MARIADB_PORT: 3306 - MARIADB_DATABASE: uvlhubdb_test + MARIADB_TEST_DATABASE: uvlhubdb_test MARIADB_USER: uvlhub_user MARIADB_PASSWORD: uvlhub_password run: | From e5b57ad8e0ee85c43d81a4f6a1545ed518bc99fd Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 18:19:05 +0100 Subject: [PATCH 17/46] feat: Implement rosemary linter --- app/tests/__init__.py | 0 app/tests/routes.py | 27 --------------------------- app/tests/units.py | 31 ------------------------------- rosemary/cli.py | 5 ++++- rosemary/commands/linter.py | 20 ++++++++++++++++++++ rosemary/commands/make_module.py | 4 ++-- rosemary/commands/test.py | 4 ++-- rosemary/commands/update.py | 4 ++-- 8 files changed, 30 insertions(+), 65 deletions(-) delete mode 100644 app/tests/__init__.py delete mode 100644 app/tests/routes.py delete mode 100644 app/tests/units.py create mode 100644 rosemary/commands/linter.py diff --git a/app/tests/__init__.py b/app/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/app/tests/routes.py b/app/tests/routes.py deleted file mode 100644 index 997a6b9fb..000000000 --- a/app/tests/routes.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from flask import Blueprint, jsonify -from sqlalchemy import text -from app import db - -test_routes = Blueprint('test_routes', __name__) - - -@test_routes.route('/test') -def test_route(): - return 'Test route' - - -@test_routes.route('/env') -def show_env(): - env_vars = {key: value for key, value in os.environ.items()} - return jsonify(env_vars) - - -@test_routes.route('/test_db') -def test_db(): - try: - db.session.execute(text('SELECT 1')) - return jsonify({'message': 'Connection to the database successful'}) - except Exception as e: - return jsonify({'error': str(e)}) diff --git a/app/tests/units.py b/app/tests/units.py deleted file mode 100644 index c3620f162..000000000 --- a/app/tests/units.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest - -from app import get_test_client - - -class FlaskAppTestCase(unittest.TestCase): - - def setUp(self): - self.app = get_test_client() - - def tearDown(self): - pass - - def test_show_env(self): - response = self.app.get('/env') - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content_type, 'application/json') - - def test_test_db(self): - response = self.app.get('/test_db') - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content_type, 'application/json') - try: - self.assertEqual(response.json['message'], 'Connection to the database successful') - except KeyError: - print("Received unexpected response: ", response.json) - raise - - -if __name__ == '__main__': - unittest.main() diff --git a/rosemary/cli.py b/rosemary/cli.py index 13cecfd24..96055d3c7 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,4 +1,6 @@ import click + +from rosemary.commands.linter import linter from rosemary.commands.update import update from rosemary.commands.info import info from rosemary.commands.make_module import make_module @@ -18,7 +20,7 @@ def get_command(self, ctx, cmd_name): @click.group(cls=RosemaryCLI) def cli(): - """A CLI tool to help with project management.""" + """A CLI tool to help with project development.""" pass @@ -27,6 +29,7 @@ def cli(): cli.add_command(make_module) cli.add_command(env) cli.add_command(test) +cli.add_command(linter) if __name__ == '__main__': cli() diff --git a/rosemary/commands/linter.py b/rosemary/commands/linter.py new file mode 100644 index 000000000..592132fb4 --- /dev/null +++ b/rosemary/commands/linter.py @@ -0,0 +1,20 @@ +import click +import subprocess + + +@click.command('linter', help="Runs flake8 linter on the 'app' and 'rosemary' directories.") +def linter(): + + # Define the directories to be checked with flake8 + directories = ["app", "rosemary"] + + # Run flake8 in each directory + for directory in directories: + click.echo(f"Running flake8 on {directory}...") + result = subprocess.run(['flake8', directory]) + + # Check if flake8 encountered problems + if result.returncode != 0: + click.echo(click.style(f"flake8 found issues in {directory}.", fg='red')) + else: + click.echo(click.style(f"No issues found in {directory}. Congratulations!", fg='green')) diff --git a/rosemary/commands/make_module.py b/rosemary/commands/make_module.py index 863a70da9..f63d5cf31 100644 --- a/rosemary/commands/make_module.py +++ b/rosemary/commands/make_module.py @@ -32,7 +32,7 @@ def make_module(name): blueprint_path = f'app/blueprints/{name}' if os.path.exists(blueprint_path): - click.echo(f"The module '{name}' already exists.") + click.echo(click.style(f"The module '{name}' already exists.", fg='red')) return env = setup_jinja_env() @@ -54,4 +54,4 @@ def make_module(name): for filename, template_name in files_and_templates.items(): render_and_write_file(env, template_name, os.path.join(blueprint_path, filename), {'blueprint_name': name}) - click.echo(f"Module '{name}' created successfully.") + click.echo(click.style(f"Module '{name}' created successfully.", fg='green')) diff --git a/rosemary/commands/test.py b/rosemary/commands/test.py index 465f392c6..5e6cb345a 100644 --- a/rosemary/commands/test.py +++ b/rosemary/commands/test.py @@ -12,7 +12,7 @@ def test(module_name): if module_name: test_path = os.path.join(base_path, module_name) if not os.path.exists(test_path): - click.echo(f"Module '{module_name}' does not exist.") + click.echo(click.style(f"Module '{module_name}' does not exist.", fg='red')) return click.echo(f"Running tests for the '{module_name}' module...") else: @@ -21,4 +21,4 @@ def test(module_name): try: subprocess.run(['pytest', '-v', test_path], check=True) except subprocess.CalledProcessError as e: - click.echo(f"Error running tests: {e}") + click.echo(click.style(f"Error running tests: {e}", fg='red')) diff --git a/rosemary/commands/update.py b/rosemary/commands/update.py index 9ca4af556..07dc01156 100644 --- a/rosemary/commands/update.py +++ b/rosemary/commands/update.py @@ -24,6 +24,6 @@ def update(): with open(requirements_path, 'w') as f: subprocess.check_call(['pip', 'freeze'], stdout=f) - click.echo('Update completed!') + click.echo(click.style('Update completed!', fg='green')) except subprocess.CalledProcessError as e: - click.echo(f'Error during the update: {e}') + click.echo(click.style(f'Error during the update: {e}', fg='red')) From a6e26b7b05c50740f8823b072ed38e6682ee27e0 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 18:32:27 +0100 Subject: [PATCH 18/46] fix: Fix .env file and init configs --- README.md | 3 ++- app/__init__.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e280cff17..967845f19 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,8 @@ git clone https://github.com/diverso-lab/uvlhub.git Create an `.env` file in the root of the project with this information. It is important to obtain a token in Zenodo first. **We recommend creating the token in the Sandbox version of Zenodo, in order to generate fictitious DOIs and not make intensive use of the real Zenodo SLA.** ``` -FLASK_APP_NAME=UVLHUB.IO +FLASK_APP_NAME="UVLHUB.IO (dev)" +FLASK_ENV=development DOMAIN=localhost MARIADB_HOSTNAME=db MARIADB_PORT=3306 diff --git a/app/__init__.py b/app/__init__.py index e7fd9aab6..c4f3ec835 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -29,6 +29,7 @@ class Config: SQLALCHEMY_TRACK_MODIFICATIONS = False TIMEZONE = 'Europe/Madrid' TEMPLATES_AUTO_RELOAD = True + UPLOAD_FOLDER = 'uploads' class DevelopmentConfig(Config): @@ -37,7 +38,6 @@ class DevelopmentConfig(Config): class TestingConfig(Config): TESTING = True - SECRET_KEY = os.getenv('SECRET_KEY', 'secret_test_key') SQLALCHEMY_DATABASE_URI = ( f"mysql+pymysql://{os.getenv('MARIADB_USER', 'default_user')}:" f"{os.getenv('MARIADB_PASSWORD', 'default_password')}@" @@ -45,7 +45,6 @@ class TestingConfig(Config): f"{os.getenv('MARIADB_PORT', '3306')}/" f"{os.getenv('MARIADB_TEST_DATABASE', 'default_db')}" ) - UPLOAD_FOLDER = 'uploads' WTF_CSRF_ENABLED = False @@ -99,7 +98,7 @@ def load_user(user_id): def inject_vars_into_jinja(): return { 'FLASK_APP_NAME': os.getenv('FLASK_APP_NAME'), - 'FLASK_SERVER_NAME': os.getenv('FLASK_SERVER_NAME'), + 'FLASK_ENV': os.getenv('FLASK_ENV'), 'DOMAIN': os.getenv('DOMAIN', 'localhost') } From c5b9a16a1af1ed4b8bdb435b8b35505ab6369016 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 19:25:40 +0100 Subject: [PATCH 19/46] feat: Implement rosemary coverage command --- .gitignore | 4 +++- README.md | 25 +++++++++++++++++++++++++ requirements.txt | 2 ++ rosemary/cli.py | 2 ++ rosemary/commands/coverage.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 rosemary/commands/coverage.py diff --git a/.gitignore b/.gitignore index 8522fcf08..ae63ad2d1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ uploads/ app.log .DS_Store rosemary.egg-info/ -build/ \ No newline at end of file +build/ +.coverage +htmlcov/ \ No newline at end of file diff --git a/README.md b/README.md index 967845f19..85d1fb153 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,31 @@ use: rosemary test zenodo ``` +### Code Coverage + +The `rosemary coverage` command facilitates running code coverage analysis for your Flask project using `pytest-cov`. +This command simplifies the process of assessing test coverage. + +#### Command Usage + +- **All Modules**: To run coverage analysis for all modules within the `app/blueprints` directory and generate an HTML report, use: + + ``` + rosemary coverage + ``` + +- **Specific Module**: If you wish to run coverage analysis for a specific module, include the +module name: + + ``` + rosemary coverage + ``` + +#### Command Options + +- **--html**: Generates an HTML coverage report. The report is saved in the `htmlcov` directory +at the root of your project. Example: `rosemary coverage --html` + ## Deploy in production (Docker Compose) ``` diff --git a/requirements.txt b/requirements.txt index a8754311f..229d0247f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ certifi==2024.2.2 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 +coverage==7.4.4 cryptography==42.0.5 dnspython==2.6.1 email_validator==2.1.1 @@ -29,6 +30,7 @@ pycparser==2.21 pyflakes==3.2.0 PyMySQL==1.1.0 pytest==8.1.1 +pytest-cov==5.0.0 python-dotenv==1.0.1 requests==2.31.0 rosemary @ file:///app diff --git a/rosemary/cli.py b/rosemary/cli.py index 96055d3c7..a35590bc7 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.coverage import coverage from rosemary.commands.linter import linter from rosemary.commands.update import update from rosemary.commands.info import info @@ -30,6 +31,7 @@ def cli(): cli.add_command(env) cli.add_command(test) cli.add_command(linter) +cli.add_command(coverage) if __name__ == '__main__': cli() diff --git a/rosemary/commands/coverage.py b/rosemary/commands/coverage.py new file mode 100644 index 000000000..ecbb36619 --- /dev/null +++ b/rosemary/commands/coverage.py @@ -0,0 +1,30 @@ +import click +import subprocess +import os + + +@click.command('coverage', help="Runs pytest coverage on the blueprints directory or a specific module.") +@click.argument('module_name', required=False) +@click.option('--html', is_flag=True, help="Generates an HTML coverage report.") +def coverage(module_name, html): + base_path = 'app/blueprints' + test_path = base_path + + if module_name: + test_path = os.path.join(base_path, module_name) + if not os.path.exists(test_path): + click.echo(click.style(f"Module '{module_name}' does not exist.", fg='red')) + return + click.echo(f"Running coverage for the '{module_name}' module...") + else: + click.echo("Running coverage for all modules...") + + coverage_cmd = ['pytest', '--cov=' + test_path, test_path] + + if html: + coverage_cmd.extend(['--cov-report', 'html']) + + try: + subprocess.run(coverage_cmd, check=True) + except subprocess.CalledProcessError as e: + click.echo(click.style(f"Error running coverage: {e}", fg='red')) From cc38335fa89c29342621b866bc3a1522fb0aaee6 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Mon, 25 Mar 2024 19:59:58 +0100 Subject: [PATCH 20/46] feat: Implement rosemary clear command --- docker-compose.dev.yml | 1 + rosemary/cli.py | 2 ++ rosemary/commands/clear.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 rosemary/commands/clear.py diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index fb7a3e68c..639d21434 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -4,6 +4,7 @@ services: web: container_name: web_app_container + image: drorganvidez/uvlhub:dev build: context: . dockerfile: Dockerfile.dev diff --git a/rosemary/cli.py b/rosemary/cli.py index a35590bc7..a6e581fca 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.clear import clear from rosemary.commands.coverage import coverage from rosemary.commands.linter import linter from rosemary.commands.update import update @@ -32,6 +33,7 @@ def cli(): cli.add_command(test) cli.add_command(linter) cli.add_command(coverage) +cli.add_command(clear) if __name__ == '__main__': cli() diff --git a/rosemary/commands/clear.py b/rosemary/commands/clear.py new file mode 100644 index 000000000..44a04d2e1 --- /dev/null +++ b/rosemary/commands/clear.py @@ -0,0 +1,19 @@ +import click +import shutil +import os + + +@click.command('clear', help="Clears the 'uploads' directory.") +def clear(): + uploads_dir = 'uploads' + + # Verify if the 'uploads' folder exists + if os.path.exists(uploads_dir) and os.path.isdir(uploads_dir): + try: + # Use shutil.rmtree to delete the folder and its contents. + shutil.rmtree(uploads_dir) + click.echo(click.style("The 'uploads' directory has been successfully cleared.", fg='green')) + except Exception as e: + click.echo(click.style(f"Error clearing the 'uploads' directory: {e}", fg='red')) + else: + click.echo(click.style("The 'uploads' directory does not exist.", fg='yellow')) From e56c6b576a80edee22910aba2b248625762bc873 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 13:55:10 +0100 Subject: [PATCH 21/46] feat: Implement rosemary clear commands --- rosemary/cli.py | 6 ++++-- rosemary/commands/clear_log.py | 18 ++++++++++++++++++ .../commands/{clear.py => clear_uploads.py} | 4 ++-- 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 rosemary/commands/clear_log.py rename rosemary/commands/{clear.py => clear_uploads.py} (87%) diff --git a/rosemary/cli.py b/rosemary/cli.py index a6e581fca..a0bc2add3 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,6 +1,7 @@ import click -from rosemary.commands.clear import clear +from rosemary.commands.clear_log import clear_log +from rosemary.commands.clear_uploads import clear_uploads from rosemary.commands.coverage import coverage from rosemary.commands.linter import linter from rosemary.commands.update import update @@ -33,7 +34,8 @@ def cli(): cli.add_command(test) cli.add_command(linter) cli.add_command(coverage) -cli.add_command(clear) +cli.add_command(clear_uploads) +cli.add_command(clear_log) if __name__ == '__main__': cli() diff --git a/rosemary/commands/clear_log.py b/rosemary/commands/clear_log.py new file mode 100644 index 000000000..d2e9a0d01 --- /dev/null +++ b/rosemary/commands/clear_log.py @@ -0,0 +1,18 @@ +import click +import os + + +@click.command('clear:log', help="Clears the 'app.log' file.") +def clear_log(): + log_file_path = 'app.log' + + # Check if the log file exists + if os.path.exists(log_file_path): + try: + # Deletes the log file + os.remove(log_file_path) + click.echo(click.style("The 'app.log' file has been successfully cleared.", fg='green')) + except Exception as e: + click.echo(click.style(f"Error clearing the 'app.log' file: {e}", fg='red')) + else: + click.echo(click.style("The 'app.log' file does not exist.", fg='yellow')) diff --git a/rosemary/commands/clear.py b/rosemary/commands/clear_uploads.py similarity index 87% rename from rosemary/commands/clear.py rename to rosemary/commands/clear_uploads.py index 44a04d2e1..68b1ee3ef 100644 --- a/rosemary/commands/clear.py +++ b/rosemary/commands/clear_uploads.py @@ -3,8 +3,8 @@ import os -@click.command('clear', help="Clears the 'uploads' directory.") -def clear(): +@click.command('clear:uploads', help="Clears the 'uploads' directory.") +def clear_uploads(): uploads_dir = 'uploads' # Verify if the 'uploads' folder exists From 32d815d9dfcf0e4e3b2e6299ca557952f46af533 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 17:07:12 +0100 Subject: [PATCH 22/46] fix: Fix bugs in rosemary commands --- app/__init__.py | 31 +++------------ app/blueprint_manager.py | 60 ++++++++++++++++++++++++++++++ app/blueprints/dataset/routes.py | 1 + app/repositories/__init__.py | 0 app/services/__init__.py | 0 rosemary/commands/clear_log.py | 2 +- rosemary/commands/clear_uploads.py | 2 +- rosemary/commands/coverage.py | 2 +- rosemary/commands/env.py | 2 +- rosemary/commands/linter.py | 2 +- rosemary/commands/make_module.py | 8 +++- rosemary/commands/test.py | 2 +- rosemary/commands/update.py | 2 +- 13 files changed, 80 insertions(+), 34 deletions(-) create mode 100644 app/blueprint_manager.py create mode 100644 app/repositories/__init__.py create mode 100644 app/services/__init__.py diff --git a/app/__init__.py b/app/__init__.py index c4f3ec835..e6742506c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -9,6 +9,8 @@ from dotenv import load_dotenv from flask_migrate import Migrate +from app.blueprint_manager import BlueprintManager + # Load environment variables load_dotenv() @@ -72,9 +74,10 @@ def create_app(config_name='development'): migrate.init_app(app, db) # Register blueprints - register_blueprints(app) + blueprint_manager = BlueprintManager(app) + blueprint_manager.register_blueprints() if config_name == 'development': - print_registered_blueprints(app) + blueprint_manager.print_registered_blueprints() from flask_login import LoginManager login_manager = LoginManager() @@ -169,28 +172,4 @@ def feature_models_counter() -> int: return count -def register_blueprints(app): - app.blueprint_url_prefixes = {} - base_dir = os.path.abspath(os.path.dirname(__file__)) - blueprints_dir = os.path.join(base_dir, 'blueprints') - for blueprint_name in os.listdir(blueprints_dir): - blueprint_path = os.path.join(blueprints_dir, blueprint_name) - if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): - try: - routes_module = importlib.import_module(f'app.blueprints.{blueprint_name}.routes') - for item in dir(routes_module): - if isinstance(getattr(routes_module, item), Blueprint): - blueprint = getattr(routes_module, item) - app.register_blueprint(blueprint) - except ModuleNotFoundError as e: - print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") - - -def print_registered_blueprints(app): - print("Registered blueprints") - for name, blueprint in app.blueprints.items(): - url_prefix = app.blueprint_url_prefixes.get(name, 'No URL prefix set') - print(f"Name: {name}, URL prefix: {url_prefix}") - - app = create_app() diff --git a/app/blueprint_manager.py b/app/blueprint_manager.py new file mode 100644 index 000000000..0c3c57999 --- /dev/null +++ b/app/blueprint_manager.py @@ -0,0 +1,60 @@ +# blueprint_manager.py +import os +import importlib.util +from flask import Blueprint + + +class BlueprintManager: + def __init__(self, app): + self.app = app + self.base_dir = os.path.abspath(os.path.dirname(__file__)) + self.blueprints_dir = os.path.join(self.base_dir, 'blueprints') + + def register_blueprints(self): + self.app.blueprints = {} + self.app.blueprint_url_prefixes = {} + base_dir = os.path.abspath(os.path.dirname(__file__)) + blueprints_dir = os.path.join(base_dir, 'blueprints') + for blueprint_name in os.listdir(blueprints_dir): + blueprint_path = os.path.join(blueprints_dir, blueprint_name) + if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): + try: + routes_module = importlib.import_module(f'app.blueprints.{blueprint_name}.routes') + for item in dir(routes_module): + if isinstance(getattr(routes_module, item), Blueprint): + blueprint = getattr(routes_module, item) + self.app.register_blueprint(blueprint) + print(f"Blueprint '{blueprint_name}' registered successfully.") + except ModuleNotFoundError as e: + print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") + + def register_blueprint(self, blueprint_name): + base_dir = os.path.abspath(os.path.dirname(__file__)) + blueprints_dir = os.path.join(base_dir, 'blueprints') + blueprint_path = os.path.join(blueprints_dir, blueprint_name) + if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): + try: + routes_module = importlib.import_module(f'app.blueprints.{blueprint_name}.routes') + for item in dir(routes_module): + if isinstance(getattr(routes_module, item), Blueprint): + blueprint = getattr(routes_module, item) + self.app.register_blueprint(blueprint) + print(f"Blueprint '{blueprint_name}' registered successfully.") + return + except ModuleNotFoundError as e: + print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") + + def unregister_blueprints(self): + for name, blueprint in list(self.app.blueprints.items()): + print(f"Unregistering blueprint: {name}") + self.app.blueprints.pop(name) + + def reload_blueprints(self): + self.unregister_blueprints() + self.register_blueprints() + + def print_registered_blueprints(self): + print("Registered blueprints") + for name, blueprint in self.app.blueprints.items(): + url_prefix = self.app.blueprint_url_prefixes.get(name, 'No URL prefix set') + print(f"Name: {name}, URL prefix: {url_prefix}") diff --git a/app/blueprints/dataset/routes.py b/app/blueprints/dataset/routes.py index 319d804b8..77fe1503f 100644 --- a/app/blueprints/dataset/routes.py +++ b/app/blueprints/dataset/routes.py @@ -28,6 +28,7 @@ def zenodo_test() -> dict: return test_full_zenodo_connection() + @dataset_bp.route('/dataset/upload', methods=['GET', 'POST']) @login_required def create_dataset(): diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/rosemary/commands/clear_log.py b/rosemary/commands/clear_log.py index d2e9a0d01..206120056 100644 --- a/rosemary/commands/clear_log.py +++ b/rosemary/commands/clear_log.py @@ -4,7 +4,7 @@ @click.command('clear:log', help="Clears the 'app.log' file.") def clear_log(): - log_file_path = 'app.log' + log_file_path = '/app/app.log' # Check if the log file exists if os.path.exists(log_file_path): diff --git a/rosemary/commands/clear_uploads.py b/rosemary/commands/clear_uploads.py index 68b1ee3ef..c7fd2ed2b 100644 --- a/rosemary/commands/clear_uploads.py +++ b/rosemary/commands/clear_uploads.py @@ -5,7 +5,7 @@ @click.command('clear:uploads', help="Clears the 'uploads' directory.") def clear_uploads(): - uploads_dir = 'uploads' + uploads_dir = '/app/uploads' # Verify if the 'uploads' folder exists if os.path.exists(uploads_dir) and os.path.isdir(uploads_dir): diff --git a/rosemary/commands/coverage.py b/rosemary/commands/coverage.py index ecbb36619..f7f79bb69 100644 --- a/rosemary/commands/coverage.py +++ b/rosemary/commands/coverage.py @@ -7,7 +7,7 @@ @click.argument('module_name', required=False) @click.option('--html', is_flag=True, help="Generates an HTML coverage report.") def coverage(module_name, html): - base_path = 'app/blueprints' + base_path = '/app/app/blueprints' test_path = base_path if module_name: diff --git a/rosemary/commands/env.py b/rosemary/commands/env.py index b6e4d62a6..e7e71651b 100644 --- a/rosemary/commands/env.py +++ b/rosemary/commands/env.py @@ -8,7 +8,7 @@ def env(): """Displays the current .env file values.""" # Load the .env file - env_values = dotenv_values(".env") + env_values = dotenv_values("/app/.env") # Display keys and values for key, value in env_values.items(): diff --git a/rosemary/commands/linter.py b/rosemary/commands/linter.py index 592132fb4..33e7fbd09 100644 --- a/rosemary/commands/linter.py +++ b/rosemary/commands/linter.py @@ -6,7 +6,7 @@ def linter(): # Define the directories to be checked with flake8 - directories = ["app", "rosemary"] + directories = ["/app/app", "/app/rosemary"] # Run flake8 in each directory for directory in directories: diff --git a/rosemary/commands/make_module.py b/rosemary/commands/make_module.py index f63d5cf31..eb8ba18e9 100644 --- a/rosemary/commands/make_module.py +++ b/rosemary/commands/make_module.py @@ -2,6 +2,8 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape import os +from app import BlueprintManager, app + def pascalcase(s): """Converts string to PascalCase.""" @@ -29,7 +31,7 @@ def render_and_write_file(env, template_name, filename, context): @click.command('make:module', help="Creates a new module with a given name.") @click.argument('name') def make_module(name): - blueprint_path = f'app/blueprints/{name}' + blueprint_path = f'/app/app/blueprints/{name}' if os.path.exists(blueprint_path): click.echo(click.style(f"The module '{name}' already exists.", fg='red')) @@ -54,4 +56,8 @@ def make_module(name): for filename, template_name in files_and_templates.items(): render_and_write_file(env, template_name, os.path.join(blueprint_path, filename), {'blueprint_name': name}) + # Reload blueprints + blueprint_manager = BlueprintManager(app) + blueprint_manager.register_blueprint(blueprint_name=name) + click.echo(click.style(f"Module '{name}' created successfully.", fg='green')) diff --git a/rosemary/commands/test.py b/rosemary/commands/test.py index 5e6cb345a..7359a706e 100644 --- a/rosemary/commands/test.py +++ b/rosemary/commands/test.py @@ -6,7 +6,7 @@ @click.command('test', help="Runs pytest on the blueprints directory or a specific module.") @click.argument('module_name', required=False) def test(module_name): - base_path = 'app/blueprints' + base_path = '/app/app/blueprints' test_path = base_path if module_name: diff --git a/rosemary/commands/update.py b/rosemary/commands/update.py index 07dc01156..331c86a9e 100644 --- a/rosemary/commands/update.py +++ b/rosemary/commands/update.py @@ -20,7 +20,7 @@ def update(): subprocess.check_call(['pip', 'install', '--upgrade', package_name]) # Update requirements.txt - requirements_path = os.path.join(os.getcwd(), 'requirements.txt') + requirements_path = '/app/requirements.txt' with open(requirements_path, 'w') as f: subprocess.check_call(['pip', 'freeze'], stdout=f) From 73ce9b6d548b3fc0069a397dea62b33695aadfcc Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 17:42:23 +0100 Subject: [PATCH 23/46] fix: Fix bugs and refactoring --- app/__init__.py | 58 +------ app/blueprints/dataset/routes.py | 7 +- app/blueprints/zenodo/services.py | 218 +++++++++++++++++++++++ app/flama.py | 23 --- app/managers/__init__.py | 0 app/{ => managers}/blueprint_manager.py | 6 +- app/managers/config_manager.py | 55 ++++++ app/zenodo.py | 221 ------------------------ 8 files changed, 283 insertions(+), 305 deletions(-) delete mode 100644 app/flama.py create mode 100644 app/managers/__init__.py rename app/{ => managers}/blueprint_manager.py (89%) create mode 100644 app/managers/config_manager.py delete mode 100644 app/zenodo.py diff --git a/app/__init__.py b/app/__init__.py index e6742506c..a328c58a9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,15 +1,14 @@ import os -import secrets import logging -import importlib.util -from flask import Flask, render_template, Blueprint +from flask import Flask, render_template from flask_login import current_user from flask_sqlalchemy import SQLAlchemy from dotenv import load_dotenv from flask_migrate import Migrate -from app.blueprint_manager import BlueprintManager +from app.managers.blueprint_manager import BlueprintManager +from app.managers.config_manager import ConfigManager # Load environment variables load_dotenv() @@ -19,55 +18,12 @@ migrate = Migrate() -class Config: - SECRET_KEY = os.getenv('SECRET_KEY', secrets.token_bytes()) - SQLALCHEMY_DATABASE_URI = ( - f"mysql+pymysql://{os.getenv('MARIADB_USER', 'default_user')}:" - f"{os.getenv('MARIADB_PASSWORD', 'default_password')}@" - f"{os.getenv('MARIADB_HOSTNAME', 'localhost')}:" - f"{os.getenv('MARIADB_PORT', '3306')}/" - f"{os.getenv('MARIADB_DATABASE', 'default_db')}" - ) - SQLALCHEMY_TRACK_MODIFICATIONS = False - TIMEZONE = 'Europe/Madrid' - TEMPLATES_AUTO_RELOAD = True - UPLOAD_FOLDER = 'uploads' - - -class DevelopmentConfig(Config): - DEBUG = True - - -class TestingConfig(Config): - TESTING = True - SQLALCHEMY_DATABASE_URI = ( - f"mysql+pymysql://{os.getenv('MARIADB_USER', 'default_user')}:" - f"{os.getenv('MARIADB_PASSWORD', 'default_password')}@" - f"{os.getenv('MARIADB_HOSTNAME', 'localhost')}:" - f"{os.getenv('MARIADB_PORT', '3306')}/" - f"{os.getenv('MARIADB_TEST_DATABASE', 'default_db')}" - ) - WTF_CSRF_ENABLED = False - - -class ProductionConfig(Config): - pass - - def create_app(config_name='development'): app = Flask(__name__) - # If config_name is not provided, use the environment variable FLASK_ENV - if config_name is None: - config_name = os.getenv('FLASK_ENV', 'development') - - # Load configuration - if config_name == 'testing': - app.config.from_object(TestingConfig) - elif config_name == 'production': - app.config.from_object(ProductionConfig) - else: - app.config.from_object(DevelopmentConfig) + # Load configuration according to environment + config_manager = ConfigManager(app) + config_manager.load_config(config_name=config_name) # Initialize SQLAlchemy and Migrate with the app db.init_app(app) @@ -76,8 +32,6 @@ def create_app(config_name='development'): # Register blueprints blueprint_manager = BlueprintManager(app) blueprint_manager.register_blueprints() - if config_name == 'development': - blueprint_manager.print_registered_blueprints() from flask_login import LoginManager login_manager = LoginManager() diff --git a/app/blueprints/dataset/routes.py b/app/blueprints/dataset/routes.py index 77fe1503f..4da102f6e 100644 --- a/app/blueprints/dataset/routes.py +++ b/app/blueprints/dataset/routes.py @@ -19,8 +19,8 @@ PublicationType, DSDownloadRecord, DSViewRecord, FileDownloadRecord from app.blueprints.dataset import dataset_bp from app.blueprints.auth.models import User -from app.zenodo import zenodo_create_new_deposition, zenodo_upload_file, \ - zenodo_publish_deposition, zenodo_get_doi, test_full_zenodo_connection +from app.blueprints.zenodo.services import test_full_zenodo_connection, zenodo_create_new_deposition, \ + zenodo_upload_file, zenodo_publish_deposition, zenodo_get_doi @dataset_bp.route('/zenodo/test', methods=['GET']) @@ -28,7 +28,6 @@ def zenodo_test() -> dict: return test_full_zenodo_connection() - @dataset_bp.route('/dataset/upload', methods=['GET', 'POST']) @login_required def create_dataset(): @@ -298,7 +297,6 @@ def upload(): try: file.save(file_path) - # valid_model = flamapy_valid_model(uvl_filename=new_filename) if True: return jsonify({ 'message': 'UVL uploaded and validated successfully', @@ -499,7 +497,6 @@ def api_create_dataset(): filename = os.path.basename(file.filename) file.save(os.path.join(temp_folder, filename)) # TODO: Change valid model function - # valid_model = flamapy_valid_model(uvl_filename=filename, user=user) if True: continue # TODO else: diff --git a/app/blueprints/zenodo/services.py b/app/blueprints/zenodo/services.py index 3fcb2e5b2..2d7393182 100644 --- a/app/blueprints/zenodo/services.py +++ b/app/blueprints/zenodo/services.py @@ -1,6 +1,224 @@ from app.blueprints.zenodo.repositories import ZenodoRepository from app.services.BaseService import BaseService +import os + +import requests + +from dotenv import load_dotenv +from flask import current_app, jsonify, Response +from flask_login import current_user + +import app +from app.blueprints.dataset.models import DataSet, FeatureModel + +load_dotenv() + +ZENODO_API_URL = 'https://sandbox.zenodo.org/api/deposit/depositions' +ZENODO_ACCESS_TOKEN = os.getenv('ZENODO_ACCESS_TOKEN') + + +def test_zenodo_connection() -> bool: + """ + Test the connection with Zenodo. + + Returns: + bool: True if the connection is successful, False otherwise. + """ + headers = {"Content-Type": "application/json"} + params = {'access_token': ZENODO_ACCESS_TOKEN} + response = requests.get(ZENODO_API_URL, params=params, headers=headers) + return response.status_code == 200 + + +def test_full_zenodo_connection() -> Response: + """ + Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the deposition. + + Returns: + bool: True if the connection, upload, and deletion are successful, False otherwise. + """ + + success = True + + # Create an empty file + file_path = os.path.join(current_app.root_path, "test_file.txt") + with open(file_path, 'w'): + pass + + messages = [] # List to store messages + + # Step 1: Create a deposition on Zenodo + headers = {"Content-Type": "application/json"} + data = { + "metadata": { + "title": "Test Deposition", + "upload_type": "dataset", + "description": "This is a test deposition created via Zenodo API", + "creators": [{"name": "John Doe"}] + } + } + + params = {'access_token': ZENODO_ACCESS_TOKEN} + response = requests.post(ZENODO_API_URL, json=data, params=params, headers=headers) + + if response.status_code != 201: + messages.append(f"Failed to create test deposition on Zenodo. Response code: {response.status_code}") + success = False + + deposition_id = response.json()["id"] + + # Step 2: Upload an empty file to the deposition + data = {'name': "test_file.txt"} + file_path = os.path.join(current_app.root_path, "test_file.txt") + files = {'file': open(file_path, 'rb')} + publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' + response = requests.post(publish_url, params=params, data=data, files=files) + + if response.status_code != 201: + messages.append(f"Failed to upload test file to Zenodo. Response code: {response.status_code}") + success = False + + # Step 3: Delete the deposition + response = requests.delete(f"{ZENODO_API_URL}/{deposition_id}", params=params) + + if os.path.exists(file_path): + os.remove(file_path) + + return jsonify({"success": success, "messages": messages}) + + +def get_all_depositions() -> dict: + """ + Get all depositions from Zenodo. + + Returns: + dict: The response in JSON format with the depositions. + """ + headers = {"Content-Type": "application/json"} + params = {'access_token': ZENODO_ACCESS_TOKEN} + response = requests.get(ZENODO_API_URL, params=params, headers=headers) + if response.status_code != 200: + raise Exception('Failed to get depositions') + return response.json() + + +def zenodo_create_new_deposition(dataset: DataSet) -> dict: + """ + Create a new deposition in Zenodo. + + Args: + dataset (DataSet): The DataSet object containing the metadata of the deposition. + + Returns: + dict: The response in JSON format with the details of the created deposition. + """ + metadata = { + 'title': dataset.ds_meta_data.title, + 'upload_type': 'dataset' if dataset.ds_meta_data.publication_type.value == "none" else 'publication', + 'publication_type': dataset.ds_meta_data.publication_type.value + if dataset.ds_meta_data.publication_type.value != "none" else None, + 'description': dataset.ds_meta_data.description, + 'creators': [{ + 'name': author.name, + **({'affiliation': author.affiliation} if author.affiliation else {}), + **({'orcid': author.orcid} if author.orcid else {}) + } for author in dataset.ds_meta_data.authors], + 'keywords': ["uvlhub"] if not dataset.ds_meta_data.tags else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"], + 'access_right': 'open', + 'license': 'CC-BY-4.0' + } + + data = { + 'metadata': metadata + } + + headers = {"Content-Type": "application/json"} + params = {'access_token': ZENODO_ACCESS_TOKEN} + response = requests.post(ZENODO_API_URL, params=params, json=data, headers=headers) + if response.status_code != 201: + error_message = f'Failed to create deposition. Error details: {response.json()}' + raise Exception(error_message) + return response.json() + + +def zenodo_upload_file(deposition_id: int, feature_model: FeatureModel, user=None) -> dict: + """ + Upload a file to a deposition in Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + feature_model (FeatureModel): The FeatureModel object representing the feature model. + user (FeatureModel): The User object representing the file owner. + + Returns: + dict: The response in JSON format with the details of the uploaded file. + """ + uvl_filename = feature_model.fm_meta_data.uvl_filename + data = {'name': uvl_filename} + user_id = current_user.id if user is None else user.id + file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), uvl_filename) + files = {'file': open(file_path, 'rb')} + + publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' + params = {'access_token': ZENODO_ACCESS_TOKEN} + response = requests.post(publish_url, params=params, data=data, files=files) + if response.status_code != 201: + error_message = f'Failed to upload files. Error details: {response.json()}' + raise Exception(error_message) + return response.json() + + +def zenodo_publish_deposition(deposition_id: int) -> dict: + """ + Publish a deposition in Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + dict: The response in JSON format with the details of the published deposition. + """ + headers = {"Content-Type": "application/json"} + publish_url = f'{ZENODO_API_URL}/{deposition_id}/actions/publish' + params = {'access_token': ZENODO_ACCESS_TOKEN} + response = requests.post(publish_url, params=params, headers=headers) + if response.status_code != 202: + raise Exception('Failed to publish deposition') + return response.json() + + +def zenodo_get_deposition(deposition_id: int) -> dict: + """ + Get a deposition from Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + dict: The response in JSON format with the details of the deposition. + """ + headers = {"Content-Type": "application/json"} + deposition_url = f'{ZENODO_API_URL}/{deposition_id}' + params = {'access_token': ZENODO_ACCESS_TOKEN} + response = requests.get(deposition_url, params=params, headers=headers) + if response.status_code != 200: + raise Exception('Failed to get deposition') + return response.json() + + +def zenodo_get_doi(deposition_id: int) -> str: + """ + Get the DOI of a deposition from Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + str: The DOI of the deposition. + """ + return zenodo_get_deposition(deposition_id).get('doi') + class Zenodo(BaseService): def __init__(self): diff --git a/app/flama.py b/app/flama.py deleted file mode 100644 index b1347212e..000000000 --- a/app/flama.py +++ /dev/null @@ -1,23 +0,0 @@ -import os - -import requests -from flask_login import current_user - -import app - -FLAMAPY_API_URL = "http://flamapyapi:8000" -FLAMAPY_API_VERSION = "api/v1/operations" - - -def flamapy_valid_model(uvl_filename: str, user=None) -> bool: - user_id = current_user.id if user is None else user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), uvl_filename) - files = {'model': open(file_path, 'rb')} - - publish_url = f'{FLAMAPY_API_URL}/{FLAMAPY_API_VERSION}/valid' - - response = requests.post(publish_url, files=files) - if response.status_code != 200: - error_message = 'FlamaPy Error! Failed to send UVL file. Error details: {}'.format(response.json()) - raise Exception(error_message) - return response.json() diff --git a/app/managers/__init__.py b/app/managers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/blueprint_manager.py b/app/managers/blueprint_manager.py similarity index 89% rename from app/blueprint_manager.py rename to app/managers/blueprint_manager.py index 0c3c57999..38daeed8c 100644 --- a/app/blueprint_manager.py +++ b/app/managers/blueprint_manager.py @@ -14,7 +14,7 @@ def register_blueprints(self): self.app.blueprints = {} self.app.blueprint_url_prefixes = {} base_dir = os.path.abspath(os.path.dirname(__file__)) - blueprints_dir = os.path.join(base_dir, 'blueprints') + blueprints_dir = '/app/app/blueprints' for blueprint_name in os.listdir(blueprints_dir): blueprint_path = os.path.join(blueprints_dir, blueprint_name) if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): @@ -24,13 +24,12 @@ def register_blueprints(self): if isinstance(getattr(routes_module, item), Blueprint): blueprint = getattr(routes_module, item) self.app.register_blueprint(blueprint) - print(f"Blueprint '{blueprint_name}' registered successfully.") except ModuleNotFoundError as e: print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") def register_blueprint(self, blueprint_name): base_dir = os.path.abspath(os.path.dirname(__file__)) - blueprints_dir = os.path.join(base_dir, 'blueprints') + blueprints_dir = '/app/app/blueprints' blueprint_path = os.path.join(blueprints_dir, blueprint_name) if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): try: @@ -39,7 +38,6 @@ def register_blueprint(self, blueprint_name): if isinstance(getattr(routes_module, item), Blueprint): blueprint = getattr(routes_module, item) self.app.register_blueprint(blueprint) - print(f"Blueprint '{blueprint_name}' registered successfully.") return except ModuleNotFoundError as e: print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") diff --git a/app/managers/config_manager.py b/app/managers/config_manager.py new file mode 100644 index 000000000..508227ded --- /dev/null +++ b/app/managers/config_manager.py @@ -0,0 +1,55 @@ +import os +import secrets + + +class ConfigManager: + def __init__(self, app): + self.app = app + + def load_config(self, config_name='development'): + # If config_name is not provided, use the environment variable FLASK_ENV + if config_name is None: + config_name = os.getenv('FLASK_ENV', 'development') + + # Load configuration + if config_name == 'testing': + self.app.config.from_object(TestingConfig) + elif config_name == 'production': + self.app.config.from_object(ProductionConfig) + else: + self.app.config.from_object(DevelopmentConfig) + + +class Config: + SECRET_KEY = os.getenv('SECRET_KEY', secrets.token_bytes()) + SQLALCHEMY_DATABASE_URI = ( + f"mysql+pymysql://{os.getenv('MARIADB_USER', 'default_user')}:" + f"{os.getenv('MARIADB_PASSWORD', 'default_password')}@" + f"{os.getenv('MARIADB_HOSTNAME', 'localhost')}:" + f"{os.getenv('MARIADB_PORT', '3306')}/" + f"{os.getenv('MARIADB_DATABASE', 'default_db')}" + ) + SQLALCHEMY_TRACK_MODIFICATIONS = False + TIMEZONE = 'Europe/Madrid' + TEMPLATES_AUTO_RELOAD = True + UPLOAD_FOLDER = 'uploads' + + +class DevelopmentConfig(Config): + DEBUG = True + + +class TestingConfig(Config): + TESTING = True + SQLALCHEMY_DATABASE_URI = ( + f"mysql+pymysql://{os.getenv('MARIADB_USER', 'default_user')}:" + f"{os.getenv('MARIADB_PASSWORD', 'default_password')}@" + f"{os.getenv('MARIADB_HOSTNAME', 'localhost')}:" + f"{os.getenv('MARIADB_PORT', '3306')}/" + f"{os.getenv('MARIADB_TEST_DATABASE', 'default_db')}" + ) + WTF_CSRF_ENABLED = False + + +class ProductionConfig(Config): + pass diff --git a/app/zenodo.py b/app/zenodo.py deleted file mode 100644 index 38b79636f..000000000 --- a/app/zenodo.py +++ /dev/null @@ -1,221 +0,0 @@ -""" -This module contains functions to interact with Zenodo and perform operations related to depositions -""" - -import os - -import requests - -from dotenv import load_dotenv -from flask import current_app, jsonify, Response -from flask_login import current_user - -import app -from app.blueprints.dataset.models import DataSet, FeatureModel - -load_dotenv() - -ZENODO_API_URL = 'https://sandbox.zenodo.org/api/deposit/depositions' -ZENODO_ACCESS_TOKEN = os.getenv('ZENODO_ACCESS_TOKEN') - - -def test_zenodo_connection() -> bool: - """ - Test the connection with Zenodo. - - Returns: - bool: True if the connection is successful, False otherwise. - """ - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(ZENODO_API_URL, params=params, headers=headers) - return response.status_code == 200 - - -def test_full_zenodo_connection() -> Response: - """ - Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the deposition. - - Returns: - bool: True if the connection, upload, and deletion are successful, False otherwise. - """ - - success = True - - # Create an empty file - file_path = os.path.join(current_app.root_path, "test_file.txt") - with open(file_path, 'w'): - pass - - messages = [] # List to store messages - - # Step 1: Create a deposition on Zenodo - headers = {"Content-Type": "application/json"} - data = { - "metadata": { - "title": "Test Deposition", - "upload_type": "dataset", - "description": "This is a test deposition created via Zenodo API", - "creators": [{"name": "John Doe"}] - } - } - - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(ZENODO_API_URL, json=data, params=params, headers=headers) - - if response.status_code != 201: - messages.append(f"Failed to create test deposition on Zenodo. Response code: {response.status_code}") - success = False - - deposition_id = response.json()["id"] - - # Step 2: Upload an empty file to the deposition - data = {'name': "test_file.txt"} - file_path = os.path.join(current_app.root_path, "test_file.txt") - files = {'file': open(file_path, 'rb')} - publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' - response = requests.post(publish_url, params=params, data=data, files=files) - - if response.status_code != 201: - messages.append(f"Failed to upload test file to Zenodo. Response code: {response.status_code}") - success = False - - # Step 3: Delete the deposition - response = requests.delete(f"{ZENODO_API_URL}/{deposition_id}", params=params) - - if os.path.exists(file_path): - os.remove(file_path) - - return jsonify({"success": success, "messages": messages}) - - -def get_all_depositions() -> dict: - """ - Get all depositions from Zenodo. - - Returns: - dict: The response in JSON format with the depositions. - """ - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(ZENODO_API_URL, params=params, headers=headers) - if response.status_code != 200: - raise Exception('Failed to get depositions') - return response.json() - - -def zenodo_create_new_deposition(dataset: DataSet) -> dict: - """ - Create a new deposition in Zenodo. - - Args: - dataset (DataSet): The DataSet object containing the metadata of the deposition. - - Returns: - dict: The response in JSON format with the details of the created deposition. - """ - metadata = { - 'title': dataset.ds_meta_data.title, - 'upload_type': 'dataset' if dataset.ds_meta_data.publication_type.value == "none" else 'publication', - 'publication_type': dataset.ds_meta_data.publication_type.value - if dataset.ds_meta_data.publication_type.value != "none" else None, - 'description': dataset.ds_meta_data.description, - 'creators': [{ - 'name': author.name, - **({'affiliation': author.affiliation} if author.affiliation else {}), - **({'orcid': author.orcid} if author.orcid else {}) - } for author in dataset.ds_meta_data.authors], - 'keywords': ["uvlhub"] if not dataset.ds_meta_data.tags else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"], - 'access_right': 'open', - 'license': 'CC-BY-4.0' - } - - data = { - 'metadata': metadata - } - - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(ZENODO_API_URL, params=params, json=data, headers=headers) - if response.status_code != 201: - error_message = f'Failed to create deposition. Error details: {response.json()}' - raise Exception(error_message) - return response.json() - - -def zenodo_upload_file(deposition_id: int, feature_model: FeatureModel, user=None) -> dict: - """ - Upload a file to a deposition in Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - feature_model (FeatureModel): The FeatureModel object representing the feature model. - user (FeatureModel): The User object representing the file owner. - - Returns: - dict: The response in JSON format with the details of the uploaded file. - """ - uvl_filename = feature_model.fm_meta_data.uvl_filename - data = {'name': uvl_filename} - user_id = current_user.id if user is None else user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), uvl_filename) - files = {'file': open(file_path, 'rb')} - - publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(publish_url, params=params, data=data, files=files) - if response.status_code != 201: - error_message = f'Failed to upload files. Error details: {response.json()}' - raise Exception(error_message) - return response.json() - - -def zenodo_publish_deposition(deposition_id: int) -> dict: - """ - Publish a deposition in Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - dict: The response in JSON format with the details of the published deposition. - """ - headers = {"Content-Type": "application/json"} - publish_url = f'{ZENODO_API_URL}/{deposition_id}/actions/publish' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(publish_url, params=params, headers=headers) - if response.status_code != 202: - raise Exception('Failed to publish deposition') - return response.json() - - -def zenodo_get_deposition(deposition_id: int) -> dict: - """ - Get a deposition from Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - dict: The response in JSON format with the details of the deposition. - """ - headers = {"Content-Type": "application/json"} - deposition_url = f'{ZENODO_API_URL}/{deposition_id}' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(deposition_url, params=params, headers=headers) - if response.status_code != 200: - raise Exception('Failed to get deposition') - return response.json() - - -def zenodo_get_doi(deposition_id: int) -> str: - """ - Get the DOI of a deposition from Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - str: The DOI of the deposition. - """ - return zenodo_get_deposition(deposition_id).get('doi') From df4fe395296d789bb9ba291ad1a46e4248d9d943 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 17:43:48 +0100 Subject: [PATCH 24/46] fix: Fix bugs and refactoring --- app/managers/blueprint_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/managers/blueprint_manager.py b/app/managers/blueprint_manager.py index 38daeed8c..979be2a59 100644 --- a/app/managers/blueprint_manager.py +++ b/app/managers/blueprint_manager.py @@ -13,7 +13,6 @@ def __init__(self, app): def register_blueprints(self): self.app.blueprints = {} self.app.blueprint_url_prefixes = {} - base_dir = os.path.abspath(os.path.dirname(__file__)) blueprints_dir = '/app/app/blueprints' for blueprint_name in os.listdir(blueprints_dir): blueprint_path = os.path.join(blueprints_dir, blueprint_name) @@ -28,7 +27,6 @@ def register_blueprints(self): print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") def register_blueprint(self, blueprint_name): - base_dir = os.path.abspath(os.path.dirname(__file__)) blueprints_dir = '/app/app/blueprints' blueprint_path = os.path.join(blueprints_dir, blueprint_name) if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): From 9839fa6f414c2ad6aaea07ef4307576e264db8d8 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 17:59:09 +0100 Subject: [PATCH 25/46] fix: Fix CI --- .github/workflows/tests.yml | 1 + app/managers/blueprint_manager.py | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d65d71ee5..eb0f3be57 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,6 +41,7 @@ jobs: - name: Run Tests env: FLASK_ENV: testing + BLUEPRINTS_DIR: app/blueprints MARIADB_HOSTNAME: 127.0.0.1 MARIADB_PORT: 3306 MARIADB_TEST_DATABASE: uvlhubdb_test diff --git a/app/managers/blueprint_manager.py b/app/managers/blueprint_manager.py index 979be2a59..6ab1cd17b 100644 --- a/app/managers/blueprint_manager.py +++ b/app/managers/blueprint_manager.py @@ -8,14 +8,13 @@ class BlueprintManager: def __init__(self, app): self.app = app self.base_dir = os.path.abspath(os.path.dirname(__file__)) - self.blueprints_dir = os.path.join(self.base_dir, 'blueprints') + self.blueprints_dir = os.getenv('BLUEPRINTS_DIR', '/app/app/blueprints') def register_blueprints(self): self.app.blueprints = {} self.app.blueprint_url_prefixes = {} - blueprints_dir = '/app/app/blueprints' - for blueprint_name in os.listdir(blueprints_dir): - blueprint_path = os.path.join(blueprints_dir, blueprint_name) + for blueprint_name in os.listdir(self.blueprints_dir): + blueprint_path = os.path.join(self.blueprints_dir, blueprint_name) if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): try: routes_module = importlib.import_module(f'app.blueprints.{blueprint_name}.routes') @@ -27,8 +26,7 @@ def register_blueprints(self): print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") def register_blueprint(self, blueprint_name): - blueprints_dir = '/app/app/blueprints' - blueprint_path = os.path.join(blueprints_dir, blueprint_name) + blueprint_path = os.path.join(self.blueprints_dir, blueprint_name) if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): try: routes_module = importlib.import_module(f'app.blueprints.{blueprint_name}.routes') From 49f064646f7c2642372ad2005783f96f215da640 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 19:27:45 +0100 Subject: [PATCH 26/46] feat: Add 'test' folder to rosemary make:module command --- rosemary/commands/make_module.py | 24 +++++++++++++++---- rosemary/commands/update.py | 1 - .../templates/blueprint_tests_test_unit.py.j2 | 22 +++++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 rosemary/templates/blueprint_tests_test_unit.py.j2 diff --git a/rosemary/commands/make_module.py b/rosemary/commands/make_module.py index eb8ba18e9..15fc48b51 100644 --- a/rosemary/commands/make_module.py +++ b/rosemary/commands/make_module.py @@ -39,6 +39,9 @@ def make_module(name): env = setup_jinja_env() + # Defines the directories to create. 'tests' is handled separately to avoid subfolders. + directories = {'templates'} + files_and_templates = { '__init__.py': 'blueprint_init.py.j2', 'routes.py': 'blueprint_routes.py.j2', @@ -46,15 +49,26 @@ def make_module(name): 'repositories.py': 'blueprint_repositories.py.j2', 'services.py': 'blueprint_services.py.j2', 'forms.py': 'blueprint_forms.py.j2', - os.path.join('templates', name, 'index.html'): 'blueprint_templates_index.html.j2' + os.path.join('templates', name, 'index.html'): 'blueprint_templates_index.html.j2', + 'tests/test_unit.py': 'blueprint_tests_test_unit.py.j2' } - # Create necessary directories - os.makedirs(os.path.join(blueprint_path, 'templates', name), exist_ok=True) + # Create the necessary directories, explicitly excluding 'tests' from the creation of subfolders. + for directory in directories: + os.makedirs(os.path.join(blueprint_path, directory, name), exist_ok=True) + + # Create 'tests' directory directly under blueprint_path, without additional subfolders. + os.makedirs(os.path.join(blueprint_path, 'tests'), exist_ok=True) + + # Create empty __init__.py file directly in the 'tests' directory. + open(os.path.join(blueprint_path, 'tests', '__init__.py'), 'a').close() - # Render and write files + # Render and write files, including 'test_unit.py' directly in 'tests'. for filename, template_name in files_and_templates.items(): - render_and_write_file(env, template_name, os.path.join(blueprint_path, filename), {'blueprint_name': name}) + if template_name: # Check if there is a defined template. + render_and_write_file(env, template_name, os.path.join(blueprint_path, filename), {'blueprint_name': name}) + else: + open(os.path.join(blueprint_path, filename), 'a').close() # Create empty file if there is no template. # Reload blueprints blueprint_manager = BlueprintManager(app) diff --git a/rosemary/commands/update.py b/rosemary/commands/update.py index 331c86a9e..0e10aeddf 100644 --- a/rosemary/commands/update.py +++ b/rosemary/commands/update.py @@ -2,7 +2,6 @@ import click import subprocess -import os @click.command() diff --git a/rosemary/templates/blueprint_tests_test_unit.py.j2 b/rosemary/templates/blueprint_tests_test_unit.py.j2 new file mode 100644 index 000000000..8f9c9af4f --- /dev/null +++ b/rosemary/templates/blueprint_tests_test_unit.py.j2 @@ -0,0 +1,22 @@ +import pytest +from app import create_app, db +from app.blueprints.auth.models import User + + +@pytest.fixture(scope='module') +def test_client(): + flask_app = create_app('testing') + + with flask_app.test_client() as testing_client: + with flask_app.app_context(): + db.create_all() + + user_test = User(email='test@example.com') + user_test.set_password('test1234') + db.session.add(user_test) + db.session.commit() + + yield testing_client + + db.session.remove() + db.drop_all() From 3a5ffb86a90aaa14269ff725d5bdec94a86124a8 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 20:47:05 +0100 Subject: [PATCH 27/46] feat: Improve test suite --- app/blueprints/conftest.py | 53 +++++++++++++++++ app/blueprints/profile/tests/test_unit.py | 57 ++++--------------- app/managers/blueprint_manager.py | 2 +- rosemary/commands/make_module.py | 4 -- .../templates/blueprint_tests_test_unit.py.j2 | 33 ++++++----- 5 files changed, 82 insertions(+), 67 deletions(-) create mode 100644 app/blueprints/conftest.py diff --git a/app/blueprints/conftest.py b/app/blueprints/conftest.py new file mode 100644 index 000000000..bd62fa990 --- /dev/null +++ b/app/blueprints/conftest.py @@ -0,0 +1,53 @@ +import pytest +from app import create_app, db +from app.blueprints.auth.models import User + + +@pytest.fixture(scope='module') +def test_client(): + flask_app = create_app('testing') + with flask_app.test_client() as testing_client: + with flask_app.app_context(): + db.create_all() + """ + The test suite always includes the following user in order to avoid repetition + of its creation + """ + user_test = User(email='test@example.com', password='test1234') + db.session.add(user_test) + db.session.commit() + yield testing_client + db.session.remove() + db.drop_all() + + +def login(test_client, email, password): + """ + Authenticates the user with the credentials provided. + + Args: + test_client: Flask test client. + email (str): User's email address. + password (str): User's password. + + Returns: + response: POST login request response. + """ + response = test_client.post('/login', data=dict( + email=email, + password=password + ), follow_redirects=True) + return response + + +def logout(test_client): + """ + Logs out the user. + + Args: + test_client: Flask test client. + + Returns: + response: Response to GET request to log out. + """ + return test_client.get('/logout', follow_redirects=True) diff --git a/app/blueprints/profile/tests/test_unit.py b/app/blueprints/profile/tests/test_unit.py index d14ddcc6a..57c43fcd9 100644 --- a/app/blueprints/profile/tests/test_unit.py +++ b/app/blueprints/profile/tests/test_unit.py @@ -1,58 +1,21 @@ import pytest from flask import url_for -from app import create_app, db -from app.blueprints.auth.models import User +from app.blueprints.conftest import login, logout -@pytest.fixture(scope='module') -def test_client(): - flask_app = create_app('testing') - - with flask_app.test_client() as testing_client: - with flask_app.app_context(): - db.create_all() - - user_test = User(email='test@example.com') - user_test.set_password('test1234') - db.session.add(user_test) - db.session.commit() - - yield testing_client - - db.session.remove() - db.drop_all() - - -def login(test_client, email, password): - """ - Authenticates the user with the credentials provided. - Args: - test_client: Flask test client. - email (str): User's email address. - password (str): User's password. - - Returns: - response: POST login request response. +@pytest.fixture(scope='module') +def test_client(test_client): """ - response = test_client.post('/login', data=dict( - email=email, - password=password - ), follow_redirects=True) - return response - - -def logout(test_client): + Extends the test_client fixture to add additional specific data for module testing. + for module testing (por example, new users) """ - Logs out the user. + with test_client.application.app_context(): + # Add HERE new elements to the database that you want to exist in the test context. + # DO NOT FORGET to use db.session.add() and db.session.commit() to save the data. + pass - Args: - test_client: Flask test client. - - Returns: - response: Response to GET request to log out. - """ - return test_client.get('/logout', follow_redirects=True) + yield test_client def test_login_success(test_client): diff --git a/app/managers/blueprint_manager.py b/app/managers/blueprint_manager.py index 6ab1cd17b..dc583beab 100644 --- a/app/managers/blueprint_manager.py +++ b/app/managers/blueprint_manager.py @@ -23,7 +23,7 @@ def register_blueprints(self): blueprint = getattr(routes_module, item) self.app.register_blueprint(blueprint) except ModuleNotFoundError as e: - print(f"Could not load the module for Blueprint '{blueprint_name}': {e}") + print(f"Error registering blueprints: Could not load the module for Blueprint '{blueprint_name}': {e}") def register_blueprint(self, blueprint_name): blueprint_path = os.path.join(self.blueprints_dir, blueprint_name) diff --git a/rosemary/commands/make_module.py b/rosemary/commands/make_module.py index 15fc48b51..ffd442a81 100644 --- a/rosemary/commands/make_module.py +++ b/rosemary/commands/make_module.py @@ -70,8 +70,4 @@ def make_module(name): else: open(os.path.join(blueprint_path, filename), 'a').close() # Create empty file if there is no template. - # Reload blueprints - blueprint_manager = BlueprintManager(app) - blueprint_manager.register_blueprint(blueprint_name=name) - click.echo(click.style(f"Module '{name}' created successfully.", fg='green')) diff --git a/rosemary/templates/blueprint_tests_test_unit.py.j2 b/rosemary/templates/blueprint_tests_test_unit.py.j2 index 8f9c9af4f..722bd6b0e 100644 --- a/rosemary/templates/blueprint_tests_test_unit.py.j2 +++ b/rosemary/templates/blueprint_tests_test_unit.py.j2 @@ -1,22 +1,25 @@ import pytest -from app import create_app, db -from app.blueprints.auth.models import User @pytest.fixture(scope='module') -def test_client(): - flask_app = create_app('testing') +def test_client(test_client): + """ + Extends the test_client fixture to add additional specific data for module testing. + for module testing (por example, new users) + """ + with test_client.application.app_context(): + # Add HERE new elements to the database that you want to exist in the test context. + # DO NOT FORGET to use db.session.add() and db.session.commit() to save the data. + pass - with flask_app.test_client() as testing_client: - with flask_app.app_context(): - db.create_all() + yield test_client - user_test = User(email='test@example.com') - user_test.set_password('test1234') - db.session.add(user_test) - db.session.commit() - yield testing_client - - db.session.remove() - db.drop_all() +def test_sample_assertion(test_client): + """ + Sample test to verify that the test framework and environment are working correctly. + It does not communicate with the Flask application; it only performs a simple assertion to + confirm that the tests in this module can be executed. + """ + greeting = "Hello, World!" + assert greeting == "Hello, World!", "The greeting does not coincide with 'Hello, World!'" From 6983b0635273ed3c840b5cb9b323b0d18f5c3d90 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 20:49:33 +0100 Subject: [PATCH 28/46] fix: Fix linter errors --- app/managers/blueprint_manager.py | 5 ++++- rosemary/commands/make_module.py | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/managers/blueprint_manager.py b/app/managers/blueprint_manager.py index dc583beab..a87c35374 100644 --- a/app/managers/blueprint_manager.py +++ b/app/managers/blueprint_manager.py @@ -23,7 +23,10 @@ def register_blueprints(self): blueprint = getattr(routes_module, item) self.app.register_blueprint(blueprint) except ModuleNotFoundError as e: - print(f"Error registering blueprints: Could not load the module for Blueprint '{blueprint_name}': {e}") + print( + f"Error registering blueprints: Could not load the module " + f"for Blueprint '{blueprint_name}': {e}" + ) def register_blueprint(self, blueprint_name): blueprint_path = os.path.join(self.blueprints_dir, blueprint_name) diff --git a/rosemary/commands/make_module.py b/rosemary/commands/make_module.py index ffd442a81..0fe103559 100644 --- a/rosemary/commands/make_module.py +++ b/rosemary/commands/make_module.py @@ -2,8 +2,6 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape import os -from app import BlueprintManager, app - def pascalcase(s): """Converts string to PascalCase.""" From 5054bdc2ca93cb3f2639307cd48c5a79b33aaee6 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Wed, 27 Mar 2024 21:08:57 +0100 Subject: [PATCH 29/46] feat: Add auth test --- app/blueprints/auth/tests/__init__.py | 0 app/blueprints/auth/tests/test_unit.py | 49 +++++++++++++++++++++++ app/blueprints/profile/tests/test_unit.py | 34 ---------------- 3 files changed, 49 insertions(+), 34 deletions(-) create mode 100644 app/blueprints/auth/tests/__init__.py create mode 100644 app/blueprints/auth/tests/test_unit.py diff --git a/app/blueprints/auth/tests/__init__.py b/app/blueprints/auth/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/blueprints/auth/tests/test_unit.py b/app/blueprints/auth/tests/test_unit.py new file mode 100644 index 000000000..c4b08e36d --- /dev/null +++ b/app/blueprints/auth/tests/test_unit.py @@ -0,0 +1,49 @@ +import pytest +from flask import url_for + + +@pytest.fixture(scope='module') +def test_client(test_client): + """ + Extends the test_client fixture to add additional specific data for module testing. + for module testing (por example, new users) + """ + with test_client.application.app_context(): + # Add HERE new elements to the database that you want to exist in the test context. + # DO NOT FORGET to use db.session.add() and db.session.commit() to save the data. + pass + + yield test_client + + +def test_login_success(test_client): + response = test_client.post('/login', data=dict( + email='test@example.com', + password='test1234' + ), follow_redirects=True) + + assert response.request.path != url_for('auth.login'), "Login was unsuccessful" + + test_client.get('/logout', follow_redirects=True) + + +def test_login_unsuccessful_bad_email(test_client): + response = test_client.post('/login', data=dict( + email='bademail@example.com', + password='test1234' + ), follow_redirects=True) + + assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + + test_client.get('/logout', follow_redirects=True) + + +def test_login_unsuccessful_bad_password(test_client): + response = test_client.post('/login', data=dict( + email='test@example.com', + password='basspassword' + ), follow_redirects=True) + + assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + + test_client.get('/logout', follow_redirects=True) diff --git a/app/blueprints/profile/tests/test_unit.py b/app/blueprints/profile/tests/test_unit.py index 57c43fcd9..7f6bf9c00 100644 --- a/app/blueprints/profile/tests/test_unit.py +++ b/app/blueprints/profile/tests/test_unit.py @@ -1,5 +1,4 @@ import pytest -from flask import url_for from app.blueprints.conftest import login, logout @@ -18,39 +17,6 @@ def test_client(test_client): yield test_client -def test_login_success(test_client): - response = test_client.post('/login', data=dict( - email='test@example.com', - password='test1234' - ), follow_redirects=True) - - assert response.request.path != url_for('auth.login'), "Login was unsuccessful" - - test_client.get('/logout', follow_redirects=True) - - -def test_login_unsuccessful_bad_email(test_client): - response = test_client.post('/login', data=dict( - email='bademail@example.com', - password='test1234' - ), follow_redirects=True) - - assert response.request.path == url_for('auth.login'), "Login was unsuccessful" - - test_client.get('/logout', follow_redirects=True) - - -def test_login_unsuccessful_bad_password(test_client): - response = test_client.post('/login', data=dict( - email='test@example.com', - password='basspassword' - ), follow_redirects=True) - - assert response.request.path == url_for('auth.login'), "Login was unsuccessful" - - test_client.get('/logout', follow_redirects=True) - - def test_edit_profile_page_get(test_client): """ Tests access to the profile editing page via a GET request. From eae4f25353738eca4e255a4028254cb0ff88feca Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Thu, 28 Mar 2024 13:11:11 +0100 Subject: [PATCH 30/46] feat: Implement rosemary db:reset command --- requirements.txt | 1 + rosemary/cli.py | 2 ++ rosemary/commands/db_reset.py | 42 +++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 rosemary/commands/db_reset.py diff --git a/requirements.txt b/requirements.txt index 229d0247f..d683279ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,7 @@ python-dotenv==1.0.1 requests==2.31.0 rosemary @ file:///app SQLAlchemy==2.0.29 +SQLAlchemy-Utils==0.41.2 typing_extensions==4.10.0 Unidecode==1.3.8 urllib3==2.2.1 diff --git a/rosemary/cli.py b/rosemary/cli.py index a0bc2add3..400ee35d2 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.db_reset import db_reset from rosemary.commands.clear_log import clear_log from rosemary.commands.clear_uploads import clear_uploads from rosemary.commands.coverage import coverage @@ -36,6 +37,7 @@ def cli(): cli.add_command(coverage) cli.add_command(clear_uploads) cli.add_command(clear_log) +cli.add_command(db_reset) if __name__ == '__main__': cli() diff --git a/rosemary/commands/db_reset.py b/rosemary/commands/db_reset.py new file mode 100644 index 000000000..94f7852f8 --- /dev/null +++ b/rosemary/commands/db_reset.py @@ -0,0 +1,42 @@ +import click +from flask.cli import with_appcontext +from app import db +from sqlalchemy import MetaData +from flask_migrate import upgrade +from rosemary.commands.clear_uploads import clear_uploads + + +@click.command('db:reset', help="Drops all tables except 'alembic_version', then recreates them from migrations, " + "and clears the uploads directory.") +@with_appcontext +def db_reset(): + if not click.confirm('WARNING: This will delete all data except migration data and clear uploads. Are you sure?', + abort=True): + return + + try: + meta = MetaData() + meta.reflect(bind=db.engine) + with db.engine.connect() as conn: + trans = conn.begin() # Initiate a transaction + for table in reversed(meta.sorted_tables): + if table.name != 'alembic_version': + conn.execute(table.delete()) + trans.commit() # Transaction Commit + click.echo(click.style("All table data cleared except 'alembic_version'.", fg='yellow')) + except Exception as e: + click.echo(click.style(f"Error clearing table data: {e}", fg='red')) + if trans: + trans.rollback() + return + + # Invoke the clear:uploads command + ctx = click.get_current_context() + ctx.invoke(clear_uploads) + + # Recreate the tables and execute the migrations + try: + upgrade() + click.echo(click.style("Tables recreated from migrations.", fg='green')) + except Exception as e: + click.echo(click.style(f"Error recreating tables from migrations: {e}", fg='red')) From f8a16e7cc882ff03994e45ce8bbe3c96a7cb358e Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Thu, 28 Mar 2024 13:42:03 +0100 Subject: [PATCH 31/46] feat: Improve rosemary db:reset command --- README.md | 31 ++++++ migrations/env.py | 9 +- migrations/versions/8ceabe6bd17a_.py | 38 ------- migrations/versions/a9aad0f9ef14_.py | 152 --------------------------- migrations/versions/fc6010b002ee_.py | 37 ------- rosemary/commands/db_reset.py | 88 ++++++++++------ 6 files changed, 90 insertions(+), 265 deletions(-) delete mode 100644 migrations/versions/8ceabe6bd17a_.py delete mode 100644 migrations/versions/a9aad0f9ef14_.py delete mode 100644 migrations/versions/fc6010b002ee_.py diff --git a/README.md b/README.md index 85d1fb153..38682a61c 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,37 @@ rosemary update Note: it is the responsibility of the developer to check that the update of the dependencies has not broken any functionality and each dependency maintains backwards compatibility. Use the script with care! +### Resetting the Database + +The `rosemary db:reset` command is a powerful tool for resetting your project's database to its +initial state. This command deletes all the data in your database, making it ideal for fixing any inconsistencies +we may have created during development. + +#### Basic Usage + +To reset your database and clear all table data except for migration records, run: + +``` +rosemary db:reset +``` + +The `rosemary db:reset` command also clears the uploads directory as part of the reset process, ensuring that any files +uploaded during development or testing are removed. + +#### Clearing Migrations with --clear-migrations + +If you need to completely rebuild your database from scratch, including removing all migration history and starting +fresh, you can use the `--clear-migrations` option: + +``` +rosemary db:reset --clear-migrations +``` + +- Delete all data from the database, including the migration history. +- Clear the migrations directory. +- Initialize a new set of migrations. +- Apply the migrations to rebuild the database schema. + ### Extending the Project with New Modules To quickly generate a new module within the project, including necessary boilerplate files diff --git a/migrations/env.py b/migrations/env.py index 89f80b211..4c9709271 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -19,7 +19,7 @@ def get_engine(): try: # this works with Flask-SQLAlchemy<3 and Alchemical return current_app.extensions['migrate'].db.get_engine() - except TypeError: + except (TypeError, AttributeError): # this works with Flask-SQLAlchemy>=3 return current_app.extensions['migrate'].db.engine @@ -90,14 +90,17 @@ def process_revision_directives(context, revision, directives): directives[:] = [] logger.info('No changes in schema detected.') + conf_args = current_app.extensions['migrate'].configure_args + if conf_args.get("process_revision_directives") is None: + conf_args["process_revision_directives"] = process_revision_directives + connectable = get_engine() with connectable.connect() as connection: context.configure( connection=connection, target_metadata=get_metadata(), - process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args + **conf_args ) with context.begin_transaction(): diff --git a/migrations/versions/8ceabe6bd17a_.py b/migrations/versions/8ceabe6bd17a_.py deleted file mode 100644 index bf8d1fc47..000000000 --- a/migrations/versions/8ceabe6bd17a_.py +++ /dev/null @@ -1,38 +0,0 @@ -"""empty message - -Revision ID: 8ceabe6bd17a -Revises: fc6010b002ee -Create Date: 2024-02-19 12:48:49.168477 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '8ceabe6bd17a' -down_revision = 'fc6010b002ee' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('user', schema=None) as batch_op: - batch_op.alter_column('password', - existing_type=mysql.VARCHAR(length=128), - type_=sa.String(length=256), - existing_nullable=False) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('user', schema=None) as batch_op: - batch_op.alter_column('password', - existing_type=sa.String(length=256), - type_=mysql.VARCHAR(length=128), - existing_nullable=False) - - # ### end Alembic commands ### diff --git a/migrations/versions/a9aad0f9ef14_.py b/migrations/versions/a9aad0f9ef14_.py deleted file mode 100644 index 86758c121..000000000 --- a/migrations/versions/a9aad0f9ef14_.py +++ /dev/null @@ -1,152 +0,0 @@ -"""empty message - -Revision ID: a9aad0f9ef14 -Revises: -Create Date: 2023-06-21 09:14:11.858032 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'a9aad0f9ef14' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('ds_metrics', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('number_of_models', sa.String(length=120), nullable=True), - sa.Column('number_of_features', sa.String(length=120), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('fm_metrics', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('solver', sa.Text(), nullable=True), - sa.Column('not_solver', sa.Text(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('user', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('email', sa.String(length=256), nullable=False), - sa.Column('password', sa.String(length=128), nullable=False), - sa.Column('created_at', sa.DateTime(), nullable=False), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('email') - ) - op.create_table('ds_meta_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('deposition_id', sa.Integer(), nullable=True), - sa.Column('title', sa.String(length=120), nullable=False), - sa.Column('description', sa.Text(), nullable=False), - sa.Column('publication_type', sa.Enum('NONE', 'ANNOTATION_COLLECTION', 'BOOK', 'BOOK_SECTION', 'CONFERENCE_PAPER', 'DATA_MANAGEMENT_PLAN', 'JOURNAL_ARTICLE', 'PATENT', 'PREPRINT', 'PROJECT_DELIVERABLE', 'PROJECT_MILESTONE', 'PROPOSAL', 'REPORT', 'SOFTWARE_DOCUMENTATION', 'TAXONOMIC_TREATMENT', 'TECHNICAL_NOTE', 'THESIS', 'WORKING_PAPER', 'OTHER', name='publicationtype'), nullable=False), - sa.Column('publication_doi', sa.String(length=120), nullable=True), - sa.Column('dataset_doi', sa.String(length=120), nullable=True), - sa.Column('tags', sa.String(length=120), nullable=True), - sa.Column('ds_metrics_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['ds_metrics_id'], ['ds_metrics.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('fm_meta_data', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uvl_filename', sa.String(length=120), nullable=False), - sa.Column('title', sa.String(length=120), nullable=False), - sa.Column('description', sa.Text(), nullable=False), - sa.Column('publication_type', sa.Enum('NONE', 'ANNOTATION_COLLECTION', 'BOOK', 'BOOK_SECTION', 'CONFERENCE_PAPER', 'DATA_MANAGEMENT_PLAN', 'JOURNAL_ARTICLE', 'PATENT', 'PREPRINT', 'PROJECT_DELIVERABLE', 'PROJECT_MILESTONE', 'PROPOSAL', 'REPORT', 'SOFTWARE_DOCUMENTATION', 'TAXONOMIC_TREATMENT', 'TECHNICAL_NOTE', 'THESIS', 'WORKING_PAPER', 'OTHER', name='publicationtype'), nullable=False), - sa.Column('publication_doi', sa.String(length=120), nullable=True), - sa.Column('tags', sa.String(length=120), nullable=True), - sa.Column('uvl_version', sa.String(length=120), nullable=True), - sa.Column('fm_metrics_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['fm_metrics_id'], ['fm_metrics.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('user_profile', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('orcid', sa.String(length=19), nullable=True), - sa.Column('affiliation', sa.String(length=100), nullable=True), - sa.Column('name', sa.String(length=100), nullable=False), - sa.Column('surname', sa.String(length=100), nullable=False), - sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('user_id') - ) - op.create_table('author', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=120), nullable=False), - sa.Column('affiliation', sa.String(length=120), nullable=True), - sa.Column('orcid', sa.String(length=120), nullable=True), - sa.Column('ds_meta_data_id', sa.Integer(), nullable=True), - sa.Column('fm_meta_data_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['ds_meta_data_id'], ['ds_meta_data.id'], ), - sa.ForeignKeyConstraint(['fm_meta_data_id'], ['fm_meta_data.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('data_set', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('ds_meta_data_id', sa.Integer(), nullable=False), - sa.Column('created_at', sa.DateTime(), nullable=False), - sa.ForeignKeyConstraint(['ds_meta_data_id'], ['ds_meta_data.id'], ), - sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('ds_download_record', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=True), - sa.Column('dataset_id', sa.Integer(), nullable=True), - sa.Column('download_date', sa.DateTime(), nullable=False), - sa.Column('download_cookie', sa.String(length=36), nullable=False), - sa.ForeignKeyConstraint(['dataset_id'], ['data_set.id'], ), - sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('ds_view_record', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=True), - sa.Column('dataset_id', sa.Integer(), nullable=True), - sa.Column('view_date', sa.DateTime(), nullable=False), - sa.Column('view_cookie', sa.String(length=36), nullable=False), - sa.ForeignKeyConstraint(['dataset_id'], ['data_set.id'], ), - sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('feature_model', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('data_set_id', sa.Integer(), nullable=False), - sa.Column('fm_meta_data_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['data_set_id'], ['data_set.id'], ), - sa.ForeignKeyConstraint(['fm_meta_data_id'], ['fm_meta_data.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('file', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=120), nullable=False), - sa.Column('checksum', sa.String(length=120), nullable=False), - sa.Column('size', sa.Integer(), nullable=False), - sa.Column('feature_model_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['feature_model_id'], ['feature_model.id'], ), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('file') - op.drop_table('feature_model') - op.drop_table('ds_view_record') - op.drop_table('ds_download_record') - op.drop_table('data_set') - op.drop_table('author') - op.drop_table('user_profile') - op.drop_table('fm_meta_data') - op.drop_table('ds_meta_data') - op.drop_table('user') - op.drop_table('fm_metrics') - op.drop_table('ds_metrics') - # ### end Alembic commands ### diff --git a/migrations/versions/fc6010b002ee_.py b/migrations/versions/fc6010b002ee_.py deleted file mode 100644 index 186489f95..000000000 --- a/migrations/versions/fc6010b002ee_.py +++ /dev/null @@ -1,37 +0,0 @@ -"""empty message - -Revision ID: fc6010b002ee -Revises: a9aad0f9ef14 -Create Date: 2023-06-21 09:31:06.107189 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'fc6010b002ee' -down_revision = 'a9aad0f9ef14' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('file_download_record', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=True), - sa.Column('file_id', sa.Integer(), nullable=True), - sa.Column('download_date', sa.DateTime(), nullable=False), - sa.Column('download_cookie', sa.String(length=36), nullable=False), - sa.ForeignKeyConstraint(['file_id'], ['file.id'], ), - sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('file_download_record') - # ### end Alembic commands ### diff --git a/rosemary/commands/db_reset.py b/rosemary/commands/db_reset.py index 94f7852f8..c84b13294 100644 --- a/rosemary/commands/db_reset.py +++ b/rosemary/commands/db_reset.py @@ -1,42 +1,60 @@ import click +import shutil +import os +import subprocess from flask.cli import with_appcontext -from app import db +from app import create_app, db from sqlalchemy import MetaData -from flask_migrate import upgrade + from rosemary.commands.clear_uploads import clear_uploads -@click.command('db:reset', help="Drops all tables except 'alembic_version', then recreates them from migrations, " - "and clears the uploads directory.") +@click.command('db:reset', help="Resets the database, optionally clears migrations and recreates them.") +@click.option('--clear-migrations', is_flag=True, + help="Remove all tables including 'alembic_version', clear migrations folder, and recreate migrations.") @with_appcontext -def db_reset(): - if not click.confirm('WARNING: This will delete all data except migration data and clear uploads. Are you sure?', - abort=True): - return - - try: - meta = MetaData() - meta.reflect(bind=db.engine) - with db.engine.connect() as conn: - trans = conn.begin() # Initiate a transaction - for table in reversed(meta.sorted_tables): - if table.name != 'alembic_version': - conn.execute(table.delete()) - trans.commit() # Transaction Commit - click.echo(click.style("All table data cleared except 'alembic_version'.", fg='yellow')) - except Exception as e: - click.echo(click.style(f"Error clearing table data: {e}", fg='red')) - if trans: - trans.rollback() - return - - # Invoke the clear:uploads command - ctx = click.get_current_context() - ctx.invoke(clear_uploads) - - # Recreate the tables and execute the migrations - try: - upgrade() - click.echo(click.style("Tables recreated from migrations.", fg='green')) - except Exception as e: - click.echo(click.style(f"Error recreating tables from migrations: {e}", fg='red')) +def db_reset(clear_migrations): + app = create_app() + with app.app_context(): + if not click.confirm('WARNING: This will delete all data and clear uploads. Are you sure?', abort=True): + return + + # Deletes data from all tables + try: + meta = MetaData() + meta.reflect(bind=db.engine) + with db.engine.connect() as conn: + trans = conn.begin() # Begin transaction + for table in reversed(meta.sorted_tables): + if not clear_migrations or table.name != 'alembic_version': + conn.execute(table.delete()) + trans.commit() # End transaction + click.echo(click.style("All table data cleared.", fg='yellow')) + except Exception as e: + click.echo(click.style(f"Error clearing table data: {e}", fg='red')) + if trans: + trans.rollback() + return + + # Delete the uploads folder + ctx = click.get_current_context() + ctx.invoke(clear_uploads) + + if clear_migrations: + # Delete the migration folder if it exists. + migrations_dir = '/app/migrations' + if os.path.isdir(migrations_dir): + shutil.rmtree(migrations_dir) + click.echo(click.style("Migrations directory cleared.", fg='yellow')) + + # Run flask db init, migrate and upgrade + try: + subprocess.run(['flask', 'db', 'init'], check=True) + subprocess.run(['flask', 'db', 'migrate'], check=True) + subprocess.run(['flask', 'db', 'upgrade'], check=True) + click.echo(click.style("Database recreated from new migrations.", fg='green')) + except subprocess.CalledProcessError as e: + click.echo(click.style(f"Error during migrations reset: {e}", fg='red')) + return + + click.echo(click.style("Database reset successfully.", fg='green')) From fbc2647cce8e10630014b4dce2799ea987e74669 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Thu, 28 Mar 2024 13:56:38 +0100 Subject: [PATCH 32/46] feat: Implement rosemary db:migrate command --- README.md | 20 +++++++++++--------- rosemary/cli.py | 2 ++ rosemary/commands/db_migrate.py | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 rosemary/commands/db_migrate.py diff --git a/README.md b/README.md index 38682a61c..e98c84459 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,6 @@ This will apply the migrations to the database and run the Flask application. **If everything worked correctly, you should see the deployed version of UVLHub in development at `http://localhost`.** -### Migrations - -However, if during development there are new changes in the model, run inside the `web` container: - -``` -flask db migrate -flask db upgrade -``` - ## Using Rosemary CLI `Rosemary` is a CLI tool developed to facilitate project management and development tasks. @@ -82,6 +73,17 @@ rosemary update Note: it is the responsibility of the developer to check that the update of the dependencies has not broken any functionality and each dependency maintains backwards compatibility. Use the script with care! +### Migrations + +If during development there are new changes in the model, run: + +``` +rosemary db:migrate +``` + +This command will detect all changes in the model (new tables, modified fields, etc.) and apply those changes to the database. +those changes to the database. + ### Resetting the Database The `rosemary db:reset` command is a powerful tool for resetting your project's database to its diff --git a/rosemary/cli.py b/rosemary/cli.py index 400ee35d2..c827f3d55 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.db_migrate import db_migrate from rosemary.commands.db_reset import db_reset from rosemary.commands.clear_log import clear_log from rosemary.commands.clear_uploads import clear_uploads @@ -38,6 +39,7 @@ def cli(): cli.add_command(clear_uploads) cli.add_command(clear_log) cli.add_command(db_reset) +cli.add_command(db_migrate) if __name__ == '__main__': cli() diff --git a/rosemary/commands/db_migrate.py b/rosemary/commands/db_migrate.py new file mode 100644 index 000000000..8afd63fd1 --- /dev/null +++ b/rosemary/commands/db_migrate.py @@ -0,0 +1,25 @@ +import click +import subprocess +from flask.cli import with_appcontext + + +@click.command('db:migrate', help="Generates and applies database migrations.") +@with_appcontext +def db_migrate(): + # Generates migrations + try: + click.echo("Generating database migrations...") + subprocess.run(['flask', 'db', 'migrate'], check=True) + click.echo(click.style("Migrations generated successfully.", fg='green')) + except subprocess.CalledProcessError as e: + click.echo(click.style(f"Error generating migrations: {e}", fg='red')) + return + + # Applies to migrations + try: + click.echo("Applying database migrations...") + subprocess.run(['flask', 'db', 'upgrade'], check=True) + click.echo(click.style("Migrations applied successfully.", fg='green')) + except subprocess.CalledProcessError as e: + click.echo(click.style(f"Error applying migrations: {e}", fg='red')) + return From 140422b134c83af0f3253bf21e435784f758a31e Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Thu, 28 Mar 2024 14:22:57 +0100 Subject: [PATCH 33/46] feat: Implement rosemary db:console command --- README.md | 12 ++++++++++++ rosemary/cli.py | 2 ++ rosemary/commands/db_console.py | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 rosemary/commands/db_console.py diff --git a/README.md b/README.md index e98c84459..187c829a6 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,18 @@ rosemary db:reset --clear-migrations - Initialize a new set of migrations. - Apply the migrations to rebuild the database schema. +### MariaDB Console + +To directly use the MariaDB console to execute native SQL statements, use: + +``` +rosemary db:console +``` + +This command connects to the MariaDB container using the credentials defined in the `.env` file. + +To exit the MariaDB console, type `exit;`. + ### Extending the Project with New Modules To quickly generate a new module within the project, including necessary boilerplate files diff --git a/rosemary/cli.py b/rosemary/cli.py index c827f3d55..a99be3535 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.db_console import db_console from rosemary.commands.db_migrate import db_migrate from rosemary.commands.db_reset import db_reset from rosemary.commands.clear_log import clear_log @@ -40,6 +41,7 @@ def cli(): cli.add_command(clear_log) cli.add_command(db_reset) cli.add_command(db_migrate) +cli.add_command(db_console) if __name__ == '__main__': cli() diff --git a/rosemary/commands/db_console.py b/rosemary/commands/db_console.py new file mode 100644 index 000000000..ec179e8d6 --- /dev/null +++ b/rosemary/commands/db_console.py @@ -0,0 +1,22 @@ +import click +import subprocess +from dotenv import load_dotenv +import os + +@click.command('db:console', help="Opens a MariaDB console with credentials from .env.") +def db_console(): + load_dotenv() + + mariadb_hostname = os.getenv('MARIADB_HOSTNAME') + mariadb_user = os.getenv('MARIADB_USER') + mariadb_password = os.getenv('MARIADB_PASSWORD') + mariadb_database = os.getenv('MARIADB_DATABASE') + + # Build the command to connect to MariaDB + mariadb_connect_cmd = f'mysql -h{mariadb_hostname} -u{mariadb_user} -p{mariadb_password} {mariadb_database}' + + # Execute the command + try: + subprocess.run(mariadb_connect_cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + click.echo(click.style(f"Error opening MariaDB console: {e}", fg='red')) From 703301e5e334895efad4ee7b7fcf315863d16019 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Thu, 28 Mar 2024 21:07:53 +0100 Subject: [PATCH 34/46] feat: Update to Python 3.12 --- .github/workflows/tests.yml | 2 +- .gitignore | 3 +- Dockerfile.dev | 2 +- Dockerfile.prod | 2 +- app/blueprints/pytest.ini | 3 + app/managers/blueprint_manager.py | 7 +- migrations/versions/68a0039f3bd2_.py | 168 +++++++++++++++++++++++++++ 7 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 app/blueprints/pytest.ini create mode 100644 migrations/versions/68a0039f3bd2_.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eb0f3be57..1dd296d7b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Prepare environment run: | diff --git a/.gitignore b/.gitignore index ae63ad2d1..5a7b1992c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ app.log rosemary.egg-info/ build/ .coverage -htmlcov/ \ No newline at end of file +htmlcov/ +.pytest_cache \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev index c2166395e..8c6548089 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM python:3.11-alpine +FROM python:3.12-alpine # Set this environment variable to suppress the "Running as root" warning from pip ENV PIP_ROOT_USER_ACTION=ignore diff --git a/Dockerfile.prod b/Dockerfile.prod index caffdad3e..2b586fefb 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image, Alpine version for a lighter footprint -FROM python:3.11-alpine +FROM python:3.12-alpine # Install MySQL client and temporary build dependencies RUN apk add --no-cache mysql-client \ diff --git a/app/blueprints/pytest.ini b/app/blueprints/pytest.ini new file mode 100644 index 000000000..b0e5a945f --- /dev/null +++ b/app/blueprints/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +filterwarnings = + ignore::DeprecationWarning \ No newline at end of file diff --git a/app/managers/blueprint_manager.py b/app/managers/blueprint_manager.py index a87c35374..a81a7a981 100644 --- a/app/managers/blueprint_manager.py +++ b/app/managers/blueprint_manager.py @@ -15,7 +15,9 @@ def register_blueprints(self): self.app.blueprint_url_prefixes = {} for blueprint_name in os.listdir(self.blueprints_dir): blueprint_path = os.path.join(self.blueprints_dir, blueprint_name) - if os.path.isdir(blueprint_path) and not blueprint_name.startswith('__'): + if (os.path.isdir(blueprint_path) and not blueprint_name.startswith('__') and + os.path.exists(os.path.join(blueprint_path, '__init__.py')) and + blueprint_name != '.pytest_cache'): try: routes_module = importlib.import_module(f'app.blueprints.{blueprint_name}.routes') for item in dir(routes_module): @@ -25,8 +27,7 @@ def register_blueprints(self): except ModuleNotFoundError as e: print( f"Error registering blueprints: Could not load the module " - f"for Blueprint '{blueprint_name}': {e}" - ) + f"for Blueprint '{blueprint_name}': {e}") def register_blueprint(self, blueprint_name): blueprint_path = os.path.join(self.blueprints_dir, blueprint_name) diff --git a/migrations/versions/68a0039f3bd2_.py b/migrations/versions/68a0039f3bd2_.py new file mode 100644 index 000000000..1927d554a --- /dev/null +++ b/migrations/versions/68a0039f3bd2_.py @@ -0,0 +1,168 @@ +"""empty message + +Revision ID: 68a0039f3bd2 +Revises: +Create Date: 2024-03-28 19:33:17.327306 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '68a0039f3bd2' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('ds_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('number_of_models', sa.String(length=120), nullable=True), + sa.Column('number_of_features', sa.String(length=120), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('fm_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('solver', sa.Text(), nullable=True), + sa.Column('not_solver', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('email', sa.String(length=256), nullable=False), + sa.Column('password', sa.String(length=256), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('email') + ) + op.create_table('zenodo', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ds_meta_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('deposition_id', sa.Integer(), nullable=True), + sa.Column('title', sa.String(length=120), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('publication_type', sa.Enum('NONE', 'ANNOTATION_COLLECTION', 'BOOK', 'BOOK_SECTION', 'CONFERENCE_PAPER', 'DATA_MANAGEMENT_PLAN', 'JOURNAL_ARTICLE', 'PATENT', 'PREPRINT', 'PROJECT_DELIVERABLE', 'PROJECT_MILESTONE', 'PROPOSAL', 'REPORT', 'SOFTWARE_DOCUMENTATION', 'TAXONOMIC_TREATMENT', 'TECHNICAL_NOTE', 'THESIS', 'WORKING_PAPER', 'OTHER', name='publicationtype'), nullable=False), + sa.Column('publication_doi', sa.String(length=120), nullable=True), + sa.Column('dataset_doi', sa.String(length=120), nullable=True), + sa.Column('tags', sa.String(length=120), nullable=True), + sa.Column('ds_metrics_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['ds_metrics_id'], ['ds_metrics.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('fm_meta_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uvl_filename', sa.String(length=120), nullable=False), + sa.Column('title', sa.String(length=120), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('publication_type', sa.Enum('NONE', 'ANNOTATION_COLLECTION', 'BOOK', 'BOOK_SECTION', 'CONFERENCE_PAPER', 'DATA_MANAGEMENT_PLAN', 'JOURNAL_ARTICLE', 'PATENT', 'PREPRINT', 'PROJECT_DELIVERABLE', 'PROJECT_MILESTONE', 'PROPOSAL', 'REPORT', 'SOFTWARE_DOCUMENTATION', 'TAXONOMIC_TREATMENT', 'TECHNICAL_NOTE', 'THESIS', 'WORKING_PAPER', 'OTHER', name='publicationtype'), nullable=False), + sa.Column('publication_doi', sa.String(length=120), nullable=True), + sa.Column('tags', sa.String(length=120), nullable=True), + sa.Column('uvl_version', sa.String(length=120), nullable=True), + sa.Column('fm_metrics_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['fm_metrics_id'], ['fm_metrics.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_profile', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('orcid', sa.String(length=19), nullable=True), + sa.Column('affiliation', sa.String(length=100), nullable=True), + sa.Column('name', sa.String(length=100), nullable=False), + sa.Column('surname', sa.String(length=100), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user_id') + ) + op.create_table('author', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=120), nullable=False), + sa.Column('affiliation', sa.String(length=120), nullable=True), + sa.Column('orcid', sa.String(length=120), nullable=True), + sa.Column('ds_meta_data_id', sa.Integer(), nullable=True), + sa.Column('fm_meta_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['ds_meta_data_id'], ['ds_meta_data.id'], ), + sa.ForeignKeyConstraint(['fm_meta_data_id'], ['fm_meta_data.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('data_set', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('ds_meta_data_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['ds_meta_data_id'], ['ds_meta_data.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ds_download_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('download_date', sa.DateTime(), nullable=False), + sa.Column('download_cookie', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['dataset_id'], ['data_set.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ds_view_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('view_date', sa.DateTime(), nullable=False), + sa.Column('view_cookie', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['dataset_id'], ['data_set.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('feature_model', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('data_set_id', sa.Integer(), nullable=False), + sa.Column('fm_meta_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['data_set_id'], ['data_set.id'], ), + sa.ForeignKeyConstraint(['fm_meta_data_id'], ['fm_meta_data.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('file', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=120), nullable=False), + sa.Column('checksum', sa.String(length=120), nullable=False), + sa.Column('size', sa.Integer(), nullable=False), + sa.Column('feature_model_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['feature_model_id'], ['feature_model.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('file_download_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('file_id', sa.Integer(), nullable=True), + sa.Column('download_date', sa.DateTime(), nullable=False), + sa.Column('download_cookie', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['file_id'], ['file.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('file_download_record') + op.drop_table('file') + op.drop_table('feature_model') + op.drop_table('ds_view_record') + op.drop_table('ds_download_record') + op.drop_table('data_set') + op.drop_table('author') + op.drop_table('user_profile') + op.drop_table('fm_meta_data') + op.drop_table('ds_meta_data') + op.drop_table('zenodo') + op.drop_table('user') + op.drop_table('fm_metrics') + op.drop_table('ds_metrics') + # ### end Alembic commands ### From 73f30f2090214dd65c6a2ddbfd265a1dfb065e91 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Thu, 28 Mar 2024 21:29:24 +0100 Subject: [PATCH 35/46] feat: Implement rosemary clear:cache command --- rosemary/cli.py | 2 ++ rosemary/commands/clear_cache.py | 34 ++++++++++++++++++++++++++++++++ rosemary/commands/db_console.py | 1 + 3 files changed, 37 insertions(+) create mode 100644 rosemary/commands/clear_cache.py diff --git a/rosemary/cli.py b/rosemary/cli.py index a99be3535..d5c9b309e 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.clear_cache import clear_cache from rosemary.commands.db_console import db_console from rosemary.commands.db_migrate import db_migrate from rosemary.commands.db_reset import db_reset @@ -39,6 +40,7 @@ def cli(): cli.add_command(coverage) cli.add_command(clear_uploads) cli.add_command(clear_log) +cli.add_command(clear_cache) cli.add_command(db_reset) cli.add_command(db_migrate) cli.add_command(db_console) diff --git a/rosemary/commands/clear_cache.py b/rosemary/commands/clear_cache.py new file mode 100644 index 000000000..682c1cba7 --- /dev/null +++ b/rosemary/commands/clear_cache.py @@ -0,0 +1,34 @@ +import click +import shutil +import os + + +@click.command('clear:cache', help="Clears pytest cache in app/blueprints and the build directory at the root.") +def clear_cache(): + + if click.confirm('Are you sure you want to clear the pytest cache and the build directory?'): + + pytest_cache_dir = '/app/app/blueprints/.pytest_cache' + build_dir = '/app/build' + + if os.path.exists(pytest_cache_dir): + try: + shutil.rmtree(pytest_cache_dir) + click.echo(click.style("Pytest cache cleared.", fg='green')) + except Exception as e: + click.echo(click.style(f"Failed to clear pytest cache: {e}", fg='red')) + else: + click.echo(click.style("No pytest cache found. Nothing to clear.", fg='yellow')) + + if os.path.exists(build_dir): + try: + shutil.rmtree(build_dir) + click.echo(click.style("Build directory cleared.", fg='green')) + except Exception as e: + click.echo(click.style(f"Failed to clear build directory: {e}", fg='red')) + else: + click.echo(click.style("No cache or build directory found. Nothing to clear.", fg='yellow')) + else: + click.echo(click.style("Clear operation cancelled.", fg='yellow')) + +# No olvides registrar este comando en tu CLI, como lo has hecho con los otros comandos. diff --git a/rosemary/commands/db_console.py b/rosemary/commands/db_console.py index ec179e8d6..86dffc2df 100644 --- a/rosemary/commands/db_console.py +++ b/rosemary/commands/db_console.py @@ -3,6 +3,7 @@ from dotenv import load_dotenv import os + @click.command('db:console', help="Opens a MariaDB console with credentials from .env.") def db_console(): load_dotenv() From 3c9981e97ac72a34211ce7ce0d8afa48d6be4c6e Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Fri, 29 Mar 2024 15:26:37 +0100 Subject: [PATCH 36/46] feat: Implement Seeders --- Dockerfile.dev | 12 ++-- README.md | 39 ++++++++++ app/blueprints/auth/seeder.py | 13 ++++ app/seeders/BaseSeeder.py | 31 ++++++++ app/seeders/__init__.py | 0 docker-compose.dev.yml | 4 +- .../{68a0039f3bd2_.py => 513400d5d415_.py} | 6 +- requirements.txt | 3 +- rosemary/__main__.py | 4 ++ rosemary/cli.py | 2 + rosemary/commands/clear_cache.py | 23 +++++- rosemary/commands/db_reset.py | 6 +- rosemary/commands/db_seed.py | 72 +++++++++++++++++++ setup.py | 11 ++- 14 files changed, 201 insertions(+), 25 deletions(-) create mode 100644 app/blueprints/auth/seeder.py create mode 100644 app/seeders/BaseSeeder.py create mode 100644 app/seeders/__init__.py rename migrations/versions/{68a0039f3bd2_.py => 513400d5d415_.py} (98%) create mode 100644 rosemary/__main__.py create mode 100644 rosemary/commands/db_seed.py diff --git a/Dockerfile.dev b/Dockerfile.dev index 8c6548089..404137b7b 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -10,9 +10,6 @@ RUN apk add --no-cache mariadb-client # Set the working directory in the container to /app WORKDIR /app -# Copy the entire project into the container -COPY . . - # Copy requirements.txt at the /app working directory COPY requirements.txt . @@ -23,10 +20,11 @@ COPY --chmod=+x scripts/wait-for-db.sh ./scripts/ COPY --chmod=+x scripts/init-db.sh ./scripts/ # Install any needed packages specified in requirements.txt -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install -r requirements.txt -# Install rosemary CLI tool -RUN pip install --no-cache-dir . +# Install rosemary CLI package in editable mode +COPY setup.py ./ +RUN pip install -e ./ # Update pip RUN pip install --no-cache-dir --upgrade pip @@ -35,4 +33,4 @@ RUN pip install --no-cache-dir --upgrade pip EXPOSE 5000 # Sets the CMD command to correctly execute the wait-for-db.sh script -CMD sh ./scripts/wait-for-db.sh && sh ./scripts/init-db.sh && flask db upgrade && flask run --host=0.0.0.0 --port=5000 --reload --debug +CMD sh ./scripts/wait-for-db.sh && sh ./scripts/init-db.sh && rosemary db:migrate && rosemary db:reset -y && rosemary db:seed && flask run --host=0.0.0.0 --port=5000 --reload --debug \ No newline at end of file diff --git a/README.md b/README.md index 187c829a6..05a737d16 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,45 @@ rosemary db:migrate This command will detect all changes in the model (new tables, modified fields, etc.) and apply those changes to the database. those changes to the database. +### Seeders + +#### Basic Usage + +It is possible to populate the database with predefined test data. It is very useful for testing certain +that require existing data. + +To popularize all test data of all modules, run: + +``` +rosemary db:seed +``` + +If we only want to popularize the test data of a specific module, run: + +``` +rosemary db:seed +``` + +Replace `` with the name of the module you want to populate +(for example, `auth` for the authentication module). + +#### Reset database before populating + +If you want to make sure that the database is in a clean state before populating it with test data, you can use the --reset flag. +populating it with test data, you can use the `--reset` flag. +This will reset the database to its initial state before running the seeders: + +``` +rosemary db:seed --reset +``` + +You can also combine the `--reset` flag with a module specification if you want to reset the database before populating +only the test data of a specific module: + +``` +rosemary db:seed --reset +``` + ### Resetting the Database The `rosemary db:reset` command is a powerful tool for resetting your project's database to its diff --git a/app/blueprints/auth/seeder.py b/app/blueprints/auth/seeder.py new file mode 100644 index 000000000..7b08aa703 --- /dev/null +++ b/app/blueprints/auth/seeder.py @@ -0,0 +1,13 @@ +from app.blueprints.auth.models import User +from app.seeders.BaseSeeder import BaseSeeder + + +class AuthSeeder(BaseSeeder): + def run(self): + + users = [ + User(email='user1@example.com', password='1234'), + User(email='user2@example.com', password='1234'), + ] + + self.seed(users) diff --git a/app/seeders/BaseSeeder.py b/app/seeders/BaseSeeder.py new file mode 100644 index 000000000..4151f1f42 --- /dev/null +++ b/app/seeders/BaseSeeder.py @@ -0,0 +1,31 @@ +from sqlalchemy.exc import IntegrityError + +from app import db + + +class BaseSeeder: + def __init__(self): + self.db = db + + def run(self): + raise NotImplementedError("The 'run' method must be implemented by the child class.") + + def seed(self, data): + """ + Attempts to insert a list of model objects. Throws an exception if data insertion fails. + + :param data: List of model objects to insert. + """ + if not data: + return + + model = type(data[0]) + if not all(isinstance(obj, model) for obj in data): + raise ValueError("All objects must be of the same model.") + + try: + self.db.session.bulk_save_objects(data) + self.db.session.commit() + except IntegrityError as e: + self.db.session.rollback() + raise Exception(f"Failed to insert data into `{model.__tablename__}` table. Error: {e}") diff --git a/app/seeders/__init__.py b/app/seeders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 639d21434..26511d600 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -9,9 +9,7 @@ services: context: . dockerfile: Dockerfile.dev volumes: - - type: bind - source: . - target: /app + - .:/app expose: - "5000" environment: diff --git a/migrations/versions/68a0039f3bd2_.py b/migrations/versions/513400d5d415_.py similarity index 98% rename from migrations/versions/68a0039f3bd2_.py rename to migrations/versions/513400d5d415_.py index 1927d554a..6d1341969 100644 --- a/migrations/versions/68a0039f3bd2_.py +++ b/migrations/versions/513400d5d415_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: 68a0039f3bd2 +Revision ID: 513400d5d415 Revises: -Create Date: 2024-03-28 19:33:17.327306 +Create Date: 2024-03-29 14:23:47.776214 """ from alembic import op @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. -revision = '68a0039f3bd2' +revision = '513400d5d415' down_revision = None branch_labels = None depends_on = None diff --git a/requirements.txt b/requirements.txt index d683279ce..1fb17ce3c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,11 +33,10 @@ pytest==8.1.1 pytest-cov==5.0.0 python-dotenv==1.0.1 requests==2.31.0 -rosemary @ file:///app SQLAlchemy==2.0.29 SQLAlchemy-Utils==0.41.2 typing_extensions==4.10.0 Unidecode==1.3.8 urllib3==2.2.1 Werkzeug==3.0.1 -WTForms==3.1.2 +WTForms==3.1.2 \ No newline at end of file diff --git a/rosemary/__main__.py b/rosemary/__main__.py new file mode 100644 index 000000000..d603733c3 --- /dev/null +++ b/rosemary/__main__.py @@ -0,0 +1,4 @@ +from rosemary.cli import cli + +if __name__ == '__main__': + cli() diff --git a/rosemary/cli.py b/rosemary/cli.py index d5c9b309e..a0e6a7a7a 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.db_seed import db_seed from rosemary.commands.clear_cache import clear_cache from rosemary.commands.db_console import db_console from rosemary.commands.db_migrate import db_migrate @@ -44,6 +45,7 @@ def cli(): cli.add_command(db_reset) cli.add_command(db_migrate) cli.add_command(db_console) +cli.add_command(db_seed) if __name__ == '__main__': cli() diff --git a/rosemary/commands/clear_cache.py b/rosemary/commands/clear_cache.py index 682c1cba7..a1d393d49 100644 --- a/rosemary/commands/clear_cache.py +++ b/rosemary/commands/clear_cache.py @@ -1,3 +1,5 @@ +from pathlib import Path + import click import shutil import os @@ -8,6 +10,7 @@ def clear_cache(): if click.confirm('Are you sure you want to clear the pytest cache and the build directory?'): + project_root = Path('/app') pytest_cache_dir = '/app/app/blueprints/.pytest_cache' build_dir = '/app/build' @@ -28,7 +31,23 @@ def clear_cache(): click.echo(click.style(f"Failed to clear build directory: {e}", fg='red')) else: click.echo(click.style("No cache or build directory found. Nothing to clear.", fg='yellow')) + + pycache_dirs = project_root.rglob('__pycache__') + for dir in pycache_dirs: + try: + shutil.rmtree(dir) + except Exception as e: + click.echo(click.style(f"Failed to clear __pycache__ directory {dir}: {e}", fg='red')) + click.echo(click.style("All __pycache__ directories cleared.", fg='green')) + + pyc_files = project_root.rglob('*.pyc') + for file in pyc_files: + try: + file.unlink() + except Exception as e: + click.echo(click.style(f"Failed to clear .pyc file {file}: {e}", fg='red')) + + click.echo(click.style("All cache cleared.", fg='green')) + else: click.echo(click.style("Clear operation cancelled.", fg='yellow')) - -# No olvides registrar este comando en tu CLI, como lo has hecho con los otros comandos. diff --git a/rosemary/commands/db_reset.py b/rosemary/commands/db_reset.py index c84b13294..a58c9aa29 100644 --- a/rosemary/commands/db_reset.py +++ b/rosemary/commands/db_reset.py @@ -12,11 +12,13 @@ @click.command('db:reset', help="Resets the database, optionally clears migrations and recreates them.") @click.option('--clear-migrations', is_flag=True, help="Remove all tables including 'alembic_version', clear migrations folder, and recreate migrations.") +@click.option('-y', '--yes', is_flag=True, help="Confirm the operation without prompting.") @with_appcontext -def db_reset(clear_migrations): +def db_reset(clear_migrations, yes): app = create_app() with app.app_context(): - if not click.confirm('WARNING: This will delete all data and clear uploads. Are you sure?', abort=True): + if not yes and not click.confirm('WARNING: This will delete all data and clear uploads. Are you sure?', + abort=True): return # Deletes data from all tables diff --git a/rosemary/commands/db_seed.py b/rosemary/commands/db_seed.py new file mode 100644 index 000000000..684661be9 --- /dev/null +++ b/rosemary/commands/db_seed.py @@ -0,0 +1,72 @@ +import os +import importlib +import click +from flask.cli import with_appcontext + +from app.seeders.BaseSeeder import BaseSeeder +from rosemary.commands.db_reset import db_reset + + +def get_module_seeders(module_path, specific_module=None): + seeders = [] + for root, dirs, files in os.walk(module_path): + if 'seeder.py' in files: + relative_path = os.path.relpath(root, module_path) + module_name = relative_path.replace(os.path.sep, '.') + full_module_name = f'app.blueprints.{module_name}.seeder' + + # Si se especificó un módulo y no coincide con el actual, continúa con el siguiente + if specific_module and specific_module != module_name.split('.')[0]: + continue + + seeder_module = importlib.import_module(full_module_name) + importlib.reload(seeder_module) # Recargar el módulo + + for attr in dir(seeder_module): + if attr.endswith('Seeder'): + seeder_class = getattr(seeder_module, attr) + if issubclass(seeder_class, BaseSeeder) and seeder_class is not BaseSeeder: + seeders.append(seeder_class()) + return seeders + + +@click.command('db:seed', help="Populates the database with the seeders defined in each module.") +@click.option('--reset', is_flag=True, help="Reset the database before seeding.") +@click.argument('module', required=False) +@with_appcontext +def db_seed(reset, module): + + if reset: + if click.confirm(click.style('This will reset the database, do you want to continue?', fg='red'), abort=True): + click.echo(click.style("Resetting the database...", fg='yellow')) + ctx = click.get_current_context() + ctx.invoke(db_reset, clear_migrations=False, yes=True) + click.echo(click.style("Database reset successfully.", fg='green')) + else: + click.echo(click.style("Database reset cancelled.", fg='yellow')) + return + + blueprints_module_path = '/app/app/blueprints' + seeders = get_module_seeders(blueprints_module_path, specific_module=module) + success = True # Flag to control the successful flow of the operation + + if module: + click.echo(click.style(f"Seeding data for the '{module}' module...", fg='green')) + else: + click.echo(click.style("Seeding data for all modules...", fg='green')) + + for seeder in seeders: + try: + seeder.run() + click.echo(click.style(f'{seeder.__class__.__name__} performed.', fg='blue')) + except Exception as e: + click.echo(click.style(f'Error running seeder {seeder.__class__.__name__}: {e}', fg='red')) + click.echo(click.style(f'Rolled back the transaction of {seeder.__class__.__name__} to keep the session ' + f'clean.', + fg='yellow')) + + success = False + break + + if success: + click.echo(click.style('Database populated with test data.', fg='green')) diff --git a/setup.py b/setup.py index 45cddb146..007a560c5 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -# setup.py - from setuptools import setup, find_packages setup( @@ -14,8 +12,9 @@ author='David Romero', author_email='drorganvidez@us.es', description="Rosemary is a CLI to be able to work on UVLHub development more easily.", - entry_points=''' - [console_scripts] - rosemary=rosemary.cli:cli - ''', + entry_points={ + 'console_scripts': [ + 'rosemary=rosemary.cli:cli' + ], + }, ) From ba50c11eda968ce1d7265eac2fde4c1fb8fed446 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Fri, 29 Mar 2024 17:00:59 +0100 Subject: [PATCH 37/46] fix: Fix bugs in Seeders --- app/blueprints/auth/seeder.py | 13 --------- app/blueprints/auth/seeders.py | 32 ++++++++++++++++++++++ app/seeders/BaseSeeder.py | 13 +++++++-- rosemary/commands/db_seed.py | 15 +++++----- rosemary/commands/make_module.py | 1 + rosemary/templates/blueprint_seeders.py.j2 | 12 ++++++++ 6 files changed, 63 insertions(+), 23 deletions(-) delete mode 100644 app/blueprints/auth/seeder.py create mode 100644 app/blueprints/auth/seeders.py create mode 100644 rosemary/templates/blueprint_seeders.py.j2 diff --git a/app/blueprints/auth/seeder.py b/app/blueprints/auth/seeder.py deleted file mode 100644 index 7b08aa703..000000000 --- a/app/blueprints/auth/seeder.py +++ /dev/null @@ -1,13 +0,0 @@ -from app.blueprints.auth.models import User -from app.seeders.BaseSeeder import BaseSeeder - - -class AuthSeeder(BaseSeeder): - def run(self): - - users = [ - User(email='user1@example.com', password='1234'), - User(email='user2@example.com', password='1234'), - ] - - self.seed(users) diff --git a/app/blueprints/auth/seeders.py b/app/blueprints/auth/seeders.py new file mode 100644 index 000000000..9355bc926 --- /dev/null +++ b/app/blueprints/auth/seeders.py @@ -0,0 +1,32 @@ +from app.blueprints.auth.models import User +from app.blueprints.profile.models import UserProfile +from app.seeders.BaseSeeder import BaseSeeder + + +class AuthSeeder(BaseSeeder): + def run(self): + + # Seeding users + users = [ + User(email='user1@example.com', password='1234'), + User(email='user2@example.com', password='1234'), + ] + + # Inserted users with their assigned IDs are returned by `self.seed`. + seeded_users = self.seed(users) + + # Create profiles for each user inserted. + user_profiles = [] + for user in seeded_users: + profile_data = { + "user_id": user.id, + "orcid": "0000-000X-XXXX-XXXX", + "affiliation": "Some University", + "name": "User", + "surname": "Surname", + } + user_profile = UserProfile(**profile_data) + user_profiles.append(user_profile) + + # Seeding user profiles + self.seed(user_profiles) diff --git a/app/seeders/BaseSeeder.py b/app/seeders/BaseSeeder.py index 4151f1f42..c62d77b39 100644 --- a/app/seeders/BaseSeeder.py +++ b/app/seeders/BaseSeeder.py @@ -10,22 +10,29 @@ def __init__(self): def run(self): raise NotImplementedError("The 'run' method must be implemented by the child class.") + from sqlalchemy.exc import IntegrityError + def seed(self, data): """ - Attempts to insert a list of model objects. Throws an exception if data insertion fails. + Attempts to insert a list of model objects and returns them with their IDs assigned after insertion. + Throws an exception if data insertion fails. :param data: List of model objects to insert. + :return: List of model objects with IDs assigned. """ if not data: - return + return [] model = type(data[0]) if not all(isinstance(obj, model) for obj in data): raise ValueError("All objects must be of the same model.") try: - self.db.session.bulk_save_objects(data) + self.db.session.add_all(data) self.db.session.commit() except IntegrityError as e: self.db.session.rollback() raise Exception(f"Failed to insert data into `{model.__tablename__}` table. Error: {e}") + + # After committing, the `data` objects should have their IDs assigned. + return data diff --git a/rosemary/commands/db_seed.py b/rosemary/commands/db_seed.py index 684661be9..6b29e9b0f 100644 --- a/rosemary/commands/db_seed.py +++ b/rosemary/commands/db_seed.py @@ -1,3 +1,4 @@ +import inspect import os import importlib import click @@ -10,10 +11,10 @@ def get_module_seeders(module_path, specific_module=None): seeders = [] for root, dirs, files in os.walk(module_path): - if 'seeder.py' in files: + if 'seeders.py' in files: relative_path = os.path.relpath(root, module_path) module_name = relative_path.replace(os.path.sep, '.') - full_module_name = f'app.blueprints.{module_name}.seeder' + full_module_name = f'app.blueprints.{module_name}.seeders' # Si se especificó un módulo y no coincide con el actual, continúa con el siguiente if specific_module and specific_module != module_name.split('.')[0]: @@ -23,10 +24,11 @@ def get_module_seeders(module_path, specific_module=None): importlib.reload(seeder_module) # Recargar el módulo for attr in dir(seeder_module): - if attr.endswith('Seeder'): - seeder_class = getattr(seeder_module, attr) - if issubclass(seeder_class, BaseSeeder) and seeder_class is not BaseSeeder: - seeders.append(seeder_class()) + potential_seeder_class = getattr(seeder_module, attr) + if (inspect.isclass(potential_seeder_class) and + issubclass(potential_seeder_class, BaseSeeder) and + potential_seeder_class is not BaseSeeder): + seeders.append(potential_seeder_class()) return seeders @@ -41,7 +43,6 @@ def db_seed(reset, module): click.echo(click.style("Resetting the database...", fg='yellow')) ctx = click.get_current_context() ctx.invoke(db_reset, clear_migrations=False, yes=True) - click.echo(click.style("Database reset successfully.", fg='green')) else: click.echo(click.style("Database reset cancelled.", fg='yellow')) return diff --git a/rosemary/commands/make_module.py b/rosemary/commands/make_module.py index 0fe103559..5a1173599 100644 --- a/rosemary/commands/make_module.py +++ b/rosemary/commands/make_module.py @@ -47,6 +47,7 @@ def make_module(name): 'repositories.py': 'blueprint_repositories.py.j2', 'services.py': 'blueprint_services.py.j2', 'forms.py': 'blueprint_forms.py.j2', + 'seeders.py': 'blueprint_seeders.py.j2', os.path.join('templates', name, 'index.html'): 'blueprint_templates_index.html.j2', 'tests/test_unit.py': 'blueprint_tests_test_unit.py.j2' } diff --git a/rosemary/templates/blueprint_seeders.py.j2 b/rosemary/templates/blueprint_seeders.py.j2 new file mode 100644 index 000000000..52e87a24a --- /dev/null +++ b/rosemary/templates/blueprint_seeders.py.j2 @@ -0,0 +1,12 @@ +from app.seeders.BaseSeeder import BaseSeeder + + +class {{ blueprint_name | pascalcase }}Seeder(BaseSeeder): + + def run(self): + + data = [ + # Create any Model object you want to make seed + ] + + self.seed(data) From 4194174d2c4545c074b67127b3daf2be7bf85378 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Fri, 29 Mar 2024 17:15:38 +0100 Subject: [PATCH 38/46] fix: Fix minor bug in AuthSeeder --- app/blueprints/auth/seeders.py | 6 +++--- app/blueprints/profile/forms.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/blueprints/auth/seeders.py b/app/blueprints/auth/seeders.py index 9355bc926..14397315b 100644 --- a/app/blueprints/auth/seeders.py +++ b/app/blueprints/auth/seeders.py @@ -20,10 +20,10 @@ def run(self): for user in seeded_users: profile_data = { "user_id": user.id, - "orcid": "0000-000X-XXXX-XXXX", + "orcid": "", "affiliation": "Some University", - "name": "User", - "surname": "Surname", + "name": "John", + "surname": "Doe", } user_profile = UserProfile(**profile_data) user_profiles.append(user_profile) diff --git a/app/blueprints/profile/forms.py b/app/blueprints/profile/forms.py index 3b3d5c86f..868a382b2 100644 --- a/app/blueprints/profile/forms.py +++ b/app/blueprints/profile/forms.py @@ -4,8 +4,8 @@ class UserProfileForm(FlaskForm): - name = StringField('Name', validators=[DataRequired(), Length(min=5, max=100)]) - surname = StringField('Surname', validators=[DataRequired(), Length(min=5, max=100)]) + name = StringField('Name', validators=[DataRequired(), Length(max=100)]) + surname = StringField('Surname', validators=[DataRequired(), Length(max=100)]) orcid = StringField('ORCID', validators=[ Optional(), Length(min=19, max=19, message='ORCID must have 16 numbers separated by dashes'), From bb838c86128bc9aa2dc0634fc2ce11c5189dab92 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Fri, 29 Mar 2024 18:23:06 +0100 Subject: [PATCH 39/46] feat: Implement rosemary route:list command --- rosemary/cli.py | 2 ++ rosemary/commands/route_list.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 rosemary/commands/route_list.py diff --git a/rosemary/cli.py b/rosemary/cli.py index a0e6a7a7a..0d562d0d4 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.route_list import route_list from rosemary.commands.db_seed import db_seed from rosemary.commands.clear_cache import clear_cache from rosemary.commands.db_console import db_console @@ -46,6 +47,7 @@ def cli(): cli.add_command(db_migrate) cli.add_command(db_console) cli.add_command(db_seed) +cli.add_command(route_list) if __name__ == '__main__': cli() diff --git a/rosemary/commands/route_list.py b/rosemary/commands/route_list.py new file mode 100644 index 000000000..8ba71b5ac --- /dev/null +++ b/rosemary/commands/route_list.py @@ -0,0 +1,51 @@ +import os +import click +from flask import current_app +from flask.cli import with_appcontext +from collections import defaultdict + + +@click.command('route:list', help="Lists all routes of the Flask application.") +@click.argument('module_name', required=False) +@click.option('--group', is_flag=True, help="Group routes by module when no specific module is provided.") +@with_appcontext +def route_list(module_name, group): + base_path = '/app/app/blueprints' + + # Checks if a module was specified and if it exists + if module_name: + module_path = os.path.join(base_path, module_name) + if not os.path.exists(module_path): + click.echo(click.style(f"Module '{module_name}' does not exist.", fg='red')) + return + click.echo(f"Listing routes for the '{module_name}' module...") + # Path filtering for a specific module + filtered_rules = [ + rule for rule in current_app.url_map.iter_rules() + if rule.endpoint.startswith(f"{module_name}.") + ] + print_route_table(filtered_rules) + else: + if group: # Group routes by module + click.echo("Listing routes for all modules, grouped by module...") + rules = sorted(current_app.url_map.iter_rules(), key=lambda rule: rule.endpoint) + grouped_rules = defaultdict(list) + for rule in rules: + module = rule.endpoint.split('.')[0] + grouped_rules[module].append(rule) + + for module, rules in sorted(grouped_rules.items()): + click.echo(click.style(f"\nModule: {module}", fg='yellow')) + print_route_table(rules) + else: # Lists all routes without grouping + click.echo("Listing routes for all modules...") + rules = sorted(current_app.url_map.iter_rules(), key=lambda rule: rule.endpoint) + print_route_table(rules) + + +def print_route_table(rules): + click.echo(f"{'Endpoint':<50} {'Methods':<30} {'Route':<100}") + click.echo('-' * 180) + for rule in rules: + methods = ', '.join(sorted(rule.methods.difference({'HEAD', 'OPTIONS'}))) + click.echo(f"{rule.endpoint:<50} {methods:<30} {rule.rule:<100}") From f2dfdeb6bca74813629d6ca5d7af9454e2451e73 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Fri, 29 Mar 2024 21:18:52 +0100 Subject: [PATCH 40/46] feat: Improve README --- README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md index 05a737d16..d65b221b4 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,41 @@ module name: - **--html**: Generates an HTML coverage report. The report is saved in the `htmlcov` directory at the root of your project. Example: `rosemary coverage --html` + +### Listing Application Routes + +The rosemary command `route:list` allows you to list all the routes available in your application. +Flask application. This command is useful for getting a quick overview of available endpoints +and their corresponding HTTP methods. + +#### Basic Usage + +To list all routes: + +``` +rosemary route:list +``` + +#### Group routes by module + +To get a grouped view of the paths by module, you can use the `--group` option. This is especially useful +for applications with a complex modular structure, as it allows you to quickly see how the paths are organized within different parts of your application. + +``` +rosemary route:list --group +``` + +#### List routes of a specific module + +It may be useful to see the paths associated with a specific module. To do this, simply provide the module +name as an argument: + +``` +rosemary route:list +``` + +Replace `` with the actual name of the module for which you want to see the paths. + ## Deploy in production (Docker Compose) ``` From 6d3c65122fcc67413d5228c3ef59671f4a5f5754 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Fri, 29 Mar 2024 23:00:23 +0100 Subject: [PATCH 41/46] feat: Implement rosemary compose:env --- .env.example | 10 ++++++++ Dockerfile.dev | 2 +- README.md | 40 +++++++++++++++++++++--------- app/blueprints/zenodo/.env.example | 1 + rosemary/cli.py | 2 ++ rosemary/commands/compose_env.py | 35 ++++++++++++++++++++++++++ rosemary/commands/db_seed.py | 5 ++-- 7 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 .env.example create mode 100644 app/blueprints/zenodo/.env.example create mode 100644 rosemary/commands/compose_env.py diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..1da9c8d7a --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +FLASK_APP_NAME=UVLHUB.IO (dev) +FLASK_ENV=development +DOMAIN=localhost +MARIADB_HOSTNAME=db +MARIADB_PORT=3306 +MARIADB_DATABASE=uvlhubdb +MARIADB_TEST_DATABASE=uvlhubdb_test +MARIADB_USER=uvlhubdb_user +MARIADB_PASSWORD=uvlhubdb_password +MARIADB_ROOT_PASSWORD=uvlhubdb_root_password diff --git a/Dockerfile.dev b/Dockerfile.dev index 404137b7b..edd5c6dd4 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -33,4 +33,4 @@ RUN pip install --no-cache-dir --upgrade pip EXPOSE 5000 # Sets the CMD command to correctly execute the wait-for-db.sh script -CMD sh ./scripts/wait-for-db.sh && sh ./scripts/init-db.sh && rosemary db:migrate && rosemary db:reset -y && rosemary db:seed && flask run --host=0.0.0.0 --port=5000 --reload --debug \ No newline at end of file +CMD sh ./scripts/wait-for-db.sh && sh ./scripts/init-db.sh && rosemary db:migrate && rosemary db:seed -y --reset && flask run --host=0.0.0.0 --port=5000 --reload --debug \ No newline at end of file diff --git a/README.md b/README.md index d65b221b4..d6abdf014 100644 --- a/README.md +++ b/README.md @@ -19,23 +19,26 @@ Repository of feature models in UVL format integrated with Zenodo and FlamaPy - git clone https://github.com/diverso-lab/uvlhub.git ``` -## Set `.env` file in root with: +## Preparing environment: -Create an `.env` file in the root of the project with this information. It is important to obtain a token in Zenodo first. **We recommend creating the token in the Sandbox version of Zenodo, in order to generate fictitious DOIs and not make intensive use of the real Zenodo SLA.** +To create an `.env` file according to a basic template, run: ``` -FLASK_APP_NAME="UVLHUB.IO (dev)" -FLASK_ENV=development -DOMAIN=localhost -MARIADB_HOSTNAME=db -MARIADB_PORT=3306 -MARIADB_DATABASE=uvlhubdb -MARIADB_USER=uvlhubuser -MARIADB_PASSWORD=uvlhubpass -MARIADB_ROOT_PASSWORD=uvlhubrootpass -ZENODO_ACCESS_TOKEN= +cp .env.example .env ``` +To use Zenodo, it is important to obtain a token in Zenodo first. +**We recommend creating the token in the Sandbox version of Zenodo, in order to generate fictitious DOIs +and not make intensive use of the real Zenodo SLA.** + +To generate the Zenodo `.env` file, run: + +``` +cp app/blueprints/zenodo/.env.example app/blueprints/zenodo/.env +``` + +To perform the composition of all environment variables, refer to section [Composing Environment Variables](#composing-environment-variables). + ## Deploy in develop To deploy the software under development environment, run: @@ -62,6 +65,18 @@ docker exec -it web_app_container /bin/sh In the terminal, you should see the prefix `/app #`. You are now ready to use Rosemary's commands. +### Composing Environment Variables + +It is possible to make a final composition of the `.env` file based on the individual `.env` files of each module. + +To execute this command and automatically combine the environment variables: + +``` +rosemary compose:env +``` +**Note: it is important to restart our Docker container for any changes to `.env` to take effect.** + + ### Update Project Dependencies To update all project dependencies, run: @@ -273,6 +288,7 @@ rosemary route:list Replace `` with the actual name of the module for which you want to see the paths. + ## Deploy in production (Docker Compose) ``` diff --git a/app/blueprints/zenodo/.env.example b/app/blueprints/zenodo/.env.example new file mode 100644 index 000000000..973e94755 --- /dev/null +++ b/app/blueprints/zenodo/.env.example @@ -0,0 +1 @@ +ZENODO_ACCESS_TOKEN= \ No newline at end of file diff --git a/rosemary/cli.py b/rosemary/cli.py index 0d562d0d4..c32887629 100644 --- a/rosemary/cli.py +++ b/rosemary/cli.py @@ -1,5 +1,6 @@ import click +from rosemary.commands.compose_env import compose_env from rosemary.commands.route_list import route_list from rosemary.commands.db_seed import db_seed from rosemary.commands.clear_cache import clear_cache @@ -48,6 +49,7 @@ def cli(): cli.add_command(db_console) cli.add_command(db_seed) cli.add_command(route_list) +cli.add_command(compose_env) if __name__ == '__main__': cli() diff --git a/rosemary/commands/compose_env.py b/rosemary/commands/compose_env.py new file mode 100644 index 000000000..cbcf99de3 --- /dev/null +++ b/rosemary/commands/compose_env.py @@ -0,0 +1,35 @@ +import os +import click +from dotenv import dotenv_values +from flask.cli import with_appcontext + + +@click.command('compose:env', help="Combines .env files from blueprints with the root .env, checking for conflicts.") +@with_appcontext +def compose_env(): + + base_path = '/app/app/blueprints' + root_env_path = '/app/.env' + + # Loads the current root .env variables into a dictionary + root_env_vars = dotenv_values(root_env_path) + + # Finds and processes all blueprints .env files + blueprint_env_paths = [os.path.join(root, '.env') for root, dirs, files in os.walk(base_path) if '.env' in files] + for env_path in blueprint_env_paths: + blueprint_env_vars = dotenv_values(env_path) + # Add or update the blueprint variables in the root .env dictionary + for key, value in blueprint_env_vars.items(): + if key in root_env_vars and root_env_vars[key] != value: + conflict_msg = (f"Conflict found for variable '{key}' in {env_path}. " + "Keeping the original value.") + click.echo(click.style(conflict_msg, fg='yellow')) + continue + root_env_vars[key] = value + + # Write back to the root .env file + with open(root_env_path, 'w') as root_env_file: + for key, value in root_env_vars.items(): + root_env_file.write(f"{key}={value}\n") + + click.echo(click.style("Successfully merged .env files without conflicts.", fg='green')) diff --git a/rosemary/commands/db_seed.py b/rosemary/commands/db_seed.py index 6b29e9b0f..f74eed7d5 100644 --- a/rosemary/commands/db_seed.py +++ b/rosemary/commands/db_seed.py @@ -34,12 +34,13 @@ def get_module_seeders(module_path, specific_module=None): @click.command('db:seed', help="Populates the database with the seeders defined in each module.") @click.option('--reset', is_flag=True, help="Reset the database before seeding.") +@click.option('-y', '--yes', is_flag=True, help="Confirm the operation without prompting.") @click.argument('module', required=False) @with_appcontext -def db_seed(reset, module): +def db_seed(reset, yes, module): if reset: - if click.confirm(click.style('This will reset the database, do you want to continue?', fg='red'), abort=True): + if yes or click.confirm(click.style('This will reset the database, do you want to continue?', fg='red'), abort=True): click.echo(click.style("Resetting the database...", fg='yellow')) ctx = click.get_current_context() ctx.invoke(db_reset, clear_migrations=False, yes=True) From ade39bdc9276b46b00e9630bfe95180a9b5a2c07 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Sat, 30 Mar 2024 20:30:06 +0100 Subject: [PATCH 42/46] feat: First steps for wiki --- docs/1.Home.md | 17 +++++++++++++++++ docs/2.GettingStarted.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 docs/1.Home.md create mode 100644 docs/2.GettingStarted.md diff --git a/docs/1.Home.md b/docs/1.Home.md new file mode 100644 index 000000000..f56db014f --- /dev/null +++ b/docs/1.Home.md @@ -0,0 +1,17 @@ +# Welcome to UVLHub.io + +Welcome to the official documentation wiki of UVLHub.io, the repository of feature models in UVL format. Developed by DiversoLab, UVLHub.io integrates seamlessly with Zenodo and FlamaPy to provide a robust platform for feature model management. + +![UVLHub.io Logo](https://www.uvlhub.io/static/img/logos/logo-light.svg) + +For more information, visit our [GitHub repository](https://github.com/diverso-lab/uvlhub). + +## Features + +- Integration with Zenodo for DOI generation. +- Support for the UVL format for feature models. +- A suite of tools for feature model analysis through FlamaPy. + +## Getting Started + +To get started with UVLHub.io, see the [Getting Started](GettingStarted.md) guide. diff --git a/docs/2.GettingStarted.md b/docs/2.GettingStarted.md new file mode 100644 index 000000000..a6d8fa98e --- /dev/null +++ b/docs/2.GettingStarted.md @@ -0,0 +1,32 @@ +# Getting Started + +This guide covers the basics of cloning the UVLHub.io repository and setting up your environment. + +## Clone the Repository + +To clone the UVLHub.io repository, run the following command: + +``` +git clone https://github.com/diverso-lab/uvlhub.git +``` + +## Preparing Your Environment + +### Create an .env File + +Copy the example environment file to get started: + +``` +cp .env.example .env +``` + +### Zenodo Token + +To use Zenodo integration, obtain a token from Zenodo. We recommend using the Sandbox version for testing: + +``` +cp app/blueprints/zenodo/.env.example app/blueprints/zenodo/.env +``` + +For a detailed guide on environment variables composition, see Composing Environment Variables. + From 148c18699a41305f25219d844240e024496f7125 Mon Sep 17 00:00:00 2001 From: David Romero Date: Thu, 4 Apr 2024 11:52:34 +0200 Subject: [PATCH 43/46] fix: Fix rosemary --- .gitignore | 3 ++- Dockerfile.dev | 12 +++++++----- requirements.txt | 3 ++- rosemary/commands/update.py | 16 ++++++++++------ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 5a7b1992c..c9e088506 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ rosemary.egg-info/ build/ .coverage htmlcov/ -.pytest_cache \ No newline at end of file +.pytest_cache +venv/ \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev index edd5c6dd4..dcf54cd28 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -5,7 +5,8 @@ FROM python:3.12-alpine ENV PIP_ROOT_USER_ACTION=ignore # Install the MariaDB client to be able to use it in the standby script. -RUN apk add --no-cache mariadb-client +RUN apk add --no-cache mariadb-client \ + && apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev libffi-dev # Set the working directory in the container to /app WORKDIR /app @@ -19,15 +20,16 @@ COPY --chmod=+x scripts/wait-for-db.sh ./scripts/ # Copy the init-db.sh script and set execution permissions COPY --chmod=+x scripts/init-db.sh ./scripts/ -# Install any needed packages specified in requirements.txt -RUN pip install -r requirements.txt +# Update pip +RUN pip install --no-cache-dir --upgrade pip setuptools # Install rosemary CLI package in editable mode +COPY rosemary/ ./rosemary COPY setup.py ./ RUN pip install -e ./ -# Update pip -RUN pip install --no-cache-dir --upgrade pip +# Install any needed packages specified in requirements.txt +RUN pip install -r requirements.txt # Expose port 5000 EXPOSE 5000 diff --git a/requirements.txt b/requirements.txt index 1fb17ce3c..6f7ccb630 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,4 +39,5 @@ typing_extensions==4.10.0 Unidecode==1.3.8 urllib3==2.2.1 Werkzeug==3.0.1 -WTForms==3.1.2 \ No newline at end of file +WTForms==3.1.2 +setuptools \ No newline at end of file diff --git a/rosemary/commands/update.py b/rosemary/commands/update.py index 0e10aeddf..84f841511 100644 --- a/rosemary/commands/update.py +++ b/rosemary/commands/update.py @@ -12,16 +12,20 @@ def update(): subprocess.check_call(['pip', 'install', '--upgrade', 'pip']) # Get the list of installed packages and update them - packages = subprocess.check_output(['pip', 'freeze']).decode('utf-8').split('\n') + packages = subprocess.check_output(['pip', 'list', '--format=freeze']).decode('utf-8').split('\n') for package in packages: - package_name = package.split('==')[0] - if package_name: # Check if the package name is not empty - subprocess.check_call(['pip', 'install', '--upgrade', package_name]) + if not package.startswith("-e"): # Ignore packages installed in editable mode + package_name = package.split('==')[0] + if package_name: # Check if the package name is not empty + subprocess.check_call(['pip', 'install', '--upgrade', package_name]) - # Update requirements.txt + # Update requirements.txt, excluding editable installations requirements_path = '/app/requirements.txt' with open(requirements_path, 'w') as f: - subprocess.check_call(['pip', 'freeze'], stdout=f) + # Use pip freeze but filter out lines starting with -e + freeze_output = subprocess.check_output(['pip', 'freeze']).decode('utf-8') + filtered_packages = [line for line in freeze_output.split('\n') if not line.startswith("-e")] + f.write('\n'.join(filtered_packages)) click.echo(click.style('Update completed!', fg='green')) except subprocess.CalledProcessError as e: From 65f6f02619e230b413195689b36505ccd3373800 Mon Sep 17 00:00:00 2001 From: David Romero Date: Thu, 4 Apr 2024 13:35:49 +0200 Subject: [PATCH 44/46] fix: Fix rosemary bug install with entrypoint --- Dockerfile.dev | 22 +++++++++++++++------- rosemary/commands/db_migrate.py | 24 +++++++++++------------- scripts/entrypoint.sh | 3 +++ setup.py | 1 - 4 files changed, 29 insertions(+), 21 deletions(-) create mode 100644 scripts/entrypoint.sh diff --git a/Dockerfile.dev b/Dockerfile.dev index dcf54cd28..57fffee18 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -20,19 +20,27 @@ COPY --chmod=+x scripts/wait-for-db.sh ./scripts/ # Copy the init-db.sh script and set execution permissions COPY --chmod=+x scripts/init-db.sh ./scripts/ -# Update pip -RUN pip install --no-cache-dir --upgrade pip setuptools - -# Install rosemary CLI package in editable mode +# Copy files COPY rosemary/ ./rosemary COPY setup.py ./ -RUN pip install -e ./ + +# Copy the entrypoint script +COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh + +# Update pip +RUN pip install --no-cache-dir --upgrade pip # Install any needed packages specified in requirements.txt RUN pip install -r requirements.txt +# Make the entrypoint script executable +RUN chmod +x /usr/local/bin/entrypoint.sh + # Expose port 5000 EXPOSE 5000 -# Sets the CMD command to correctly execute the wait-for-db.sh script -CMD sh ./scripts/wait-for-db.sh && sh ./scripts/init-db.sh && rosemary db:migrate && rosemary db:seed -y --reset && flask run --host=0.0.0.0 --port=5000 --reload --debug \ No newline at end of file +# Set the entrypoint to run the entrypoint script +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +# Sets the default command to start the Flask application +CMD sh /app/scripts/wait-for-db.sh && sh /app/scripts/init-db.sh && rosemary db:migrate && rosemary db:seed -y --reset && flask run --host=0.0.0.0 --port=5000 --reload --debug \ No newline at end of file diff --git a/rosemary/commands/db_migrate.py b/rosemary/commands/db_migrate.py index 8afd63fd1..e88803bd3 100644 --- a/rosemary/commands/db_migrate.py +++ b/rosemary/commands/db_migrate.py @@ -1,5 +1,5 @@ -import click import subprocess +import click from flask.cli import with_appcontext @@ -7,19 +7,17 @@ @with_appcontext def db_migrate(): # Generates migrations - try: - click.echo("Generating database migrations...") - subprocess.run(['flask', 'db', 'migrate'], check=True) + click.echo("Generating database migrations...") + result_migrate = subprocess.run(['flask', 'db', 'migrate']) + if result_migrate.returncode == 0: click.echo(click.style("Migrations generated successfully.", fg='green')) - except subprocess.CalledProcessError as e: - click.echo(click.style(f"Error generating migrations: {e}", fg='red')) - return + else: + click.echo(click.style("Note: No new migrations needed or an error occurred while generating migrations.", fg='yellow')) # Applies to migrations - try: - click.echo("Applying database migrations...") - subprocess.run(['flask', 'db', 'upgrade'], check=True) + click.echo("Applying database migrations...") + result_upgrade = subprocess.run(['flask', 'db', 'upgrade']) + if result_upgrade.returncode == 0: click.echo(click.style("Migrations applied successfully.", fg='green')) - except subprocess.CalledProcessError as e: - click.echo(click.style(f"Error applying migrations: {e}", fg='red')) - return + else: + click.echo(click.style("Error applying migrations. This may be due to the database being already up-to-date.", fg='yellow')) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100644 index 000000000..94ef69131 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +pip install -e /app/ +exec "$@" \ No newline at end of file diff --git a/setup.py b/setup.py index 007a560c5..afd5b9c9f 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ name='rosemary', version='0.1.0', packages=find_packages(), - include_package_data=True, install_requires=[ 'click', 'python-dotenv', From 3aecfb74cecdfb7b0e83b437790b22b9c6257af6 Mon Sep 17 00:00:00 2001 From: David Romero Date: Fri, 5 Apr 2024 17:06:02 +0200 Subject: [PATCH 45/46] fix: Refactoring rosemary update command --- requirements.txt | 9 ++++---- rosemary/commands/db_migrate.py | 6 +++-- rosemary/commands/db_seed.py | 3 ++- rosemary/commands/update.py | 39 +++++++++++++++++++-------------- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6f7ccb630..6e16eaa18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,18 +26,19 @@ mccabe==0.7.0 packaging==24.0 pluggy==1.4.0 pycodestyle==2.11.1 -pycparser==2.21 +pycparser==2.22 pyflakes==3.2.0 PyMySQL==1.1.0 pytest==8.1.1 pytest-cov==5.0.0 python-dotenv==1.0.1 requests==2.31.0 +setuptools==69.2.0 SQLAlchemy==2.0.29 SQLAlchemy-Utils==0.41.2 -typing_extensions==4.10.0 +typing_extensions==4.11.0 Unidecode==1.3.8 urllib3==2.2.1 -Werkzeug==3.0.1 +Werkzeug==3.0.2 +wheel==0.43.0 WTForms==3.1.2 -setuptools \ No newline at end of file diff --git a/rosemary/commands/db_migrate.py b/rosemary/commands/db_migrate.py index e88803bd3..367d38b5c 100644 --- a/rosemary/commands/db_migrate.py +++ b/rosemary/commands/db_migrate.py @@ -12,7 +12,8 @@ def db_migrate(): if result_migrate.returncode == 0: click.echo(click.style("Migrations generated successfully.", fg='green')) else: - click.echo(click.style("Note: No new migrations needed or an error occurred while generating migrations.", fg='yellow')) + click.echo(click.style("Note: No new migrations needed or an error occurred " + "while generating migrations.", fg='yellow')) # Applies to migrations click.echo("Applying database migrations...") @@ -20,4 +21,5 @@ def db_migrate(): if result_upgrade.returncode == 0: click.echo(click.style("Migrations applied successfully.", fg='green')) else: - click.echo(click.style("Error applying migrations. This may be due to the database being already up-to-date.", fg='yellow')) + click.echo(click.style("Error applying migrations. This may be due to the database " + "being already up-to-date.", fg='yellow')) diff --git a/rosemary/commands/db_seed.py b/rosemary/commands/db_seed.py index f74eed7d5..65068b0f8 100644 --- a/rosemary/commands/db_seed.py +++ b/rosemary/commands/db_seed.py @@ -40,7 +40,8 @@ def get_module_seeders(module_path, specific_module=None): def db_seed(reset, yes, module): if reset: - if yes or click.confirm(click.style('This will reset the database, do you want to continue?', fg='red'), abort=True): + if yes or click.confirm(click.style('This will reset the database, do you want ' + 'to continue?', fg='red'), abort=True): click.echo(click.style("Resetting the database...", fg='yellow')) ctx = click.get_current_context() ctx.invoke(db_reset, clear_migrations=False, yes=True) diff --git a/rosemary/commands/update.py b/rosemary/commands/update.py index 84f841511..4bd31929e 100644 --- a/rosemary/commands/update.py +++ b/rosemary/commands/update.py @@ -1,32 +1,37 @@ -# rosemary/commands/update.py - import click import subprocess @click.command() def update(): - """This command updates pip, all packages, and updates requirements.txt.""" + """This command updates all packages based on the requirements.txt, excluding editable installations, and updates + the file with concrete versions.""" + requirements_path = '/app/requirements.txt' try: - # Update pip + # Update pip first subprocess.check_call(['pip', 'install', '--upgrade', 'pip']) - # Get the list of installed packages and update them - packages = subprocess.check_output(['pip', 'list', '--format=freeze']).decode('utf-8').split('\n') - for package in packages: - if not package.startswith("-e"): # Ignore packages installed in editable mode - package_name = package.split('==')[0] - if package_name: # Check if the package name is not empty - subprocess.check_call(['pip', 'install', '--upgrade', package_name]) + # Read current requirements, excluding -e packages + with open(requirements_path, 'r') as f: + requirements = [line.strip() for line in f if not line.startswith("-e")] + + # Update each package + for requirement in requirements: + package_name = requirement.split('==')[0] + if package_name: # Ensure it's not an empty package name + subprocess.check_call(['pip', 'install', '--upgrade', package_name]) - # Update requirements.txt, excluding editable installations - requirements_path = '/app/requirements.txt' + # Generate a new requirements.txt, excluding editable installations + freeze_output = subprocess.check_output(['pip', 'freeze']).decode('utf-8') with open(requirements_path, 'w') as f: - # Use pip freeze but filter out lines starting with -e - freeze_output = subprocess.check_output(['pip', 'freeze']).decode('utf-8') - filtered_packages = [line for line in freeze_output.split('\n') if not line.startswith("-e")] - f.write('\n'.join(filtered_packages)) + for line in freeze_output.split('\n'): + if not line.startswith("-e"): + f.write(line + '\n') click.echo(click.style('Update completed!', fg='green')) except subprocess.CalledProcessError as e: click.echo(click.style(f'Error during the update: {e}', fg='red')) + + +if __name__ == '__main__': + update() From f866dd4d09651ee21bc90ab3ad8261d8cd0f2373 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Sun, 7 Apr 2024 11:41:51 +0200 Subject: [PATCH 46/46] feat: Update requirements --- requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requirements.txt b/requirements.txt index 6e16eaa18..f3bc3f4e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ alembic==1.13.1 +aniso8601==9.0.1 blinker==1.7.0 certifi==2024.2.2 cffi==1.16.0 @@ -12,6 +13,7 @@ flake8==7.0.0 Flask==3.0.2 Flask-Login==0.6.3 Flask-Migrate==4.0.7 +Flask-RESTful==0.3.10 Flask-SQLAlchemy==3.1.1 Flask-WTF==1.2.1 greenlet==3.0.3 @@ -32,8 +34,10 @@ PyMySQL==1.1.0 pytest==8.1.1 pytest-cov==5.0.0 python-dotenv==1.0.1 +pytz==2024.1 requests==2.31.0 setuptools==69.2.0 +six==1.16.0 SQLAlchemy==2.0.29 SQLAlchemy-Utils==0.41.2 typing_extensions==4.11.0 @@ -42,3 +46,4 @@ urllib3==2.2.1 Werkzeug==3.0.2 wheel==0.43.0 WTForms==3.1.2 +