diff --git a/.github/workflows/deploy-pre-release.yml b/.github/workflows/deploy-pre-release.yml index 518fbe1..76e4c72 100644 --- a/.github/workflows/deploy-pre-release.yml +++ b/.github/workflows/deploy-pre-release.yml @@ -14,7 +14,6 @@ env: LC_ALL: en_US.utf-8 PYTHON_VERSION: '3.10' PROJECT_NAME: volttron-lib-sql-historian - POETRY_VERSION: '1.2.2' jobs: diff --git a/.github/workflows/dispatch-to-pypi.yml b/.github/workflows/dispatch-to-pypi.yml deleted file mode 100644 index b114704..0000000 --- a/.github/workflows/dispatch-to-pypi.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# Documentation located -# https://github.com/marketplace/actions/publish-python-poetry-package -name: Dispatch to PyPi - -on: - workflow_dispatch: - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - -jobs: - - publish_to_pypi: - - runs-on: ubuntu-22.04 - - steps: - - run: echo "πŸŽ‰ The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "πŸ”Ž The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Build and publish to pypi - uses: JRubics/poetry-publish@v1.7 - with: - # These are only needed when using test.pypi - #repository_name: testpypi - #repository_url: https://test.pypi.org/legacy/ - pypi_token: ${{ secrets.PYPI_TOKEN }} - ignore_dev_requirements: "yes" \ No newline at end of file diff --git a/.github/workflows/downstream-test-response.yml b/.github/workflows/downstream-test-response.yml deleted file mode 100644 index 8ad1116..0000000 --- a/.github/workflows/downstream-test-response.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Respond to downstream test runs as part of downstream testing - - -on: - repository_dispatch: - # To add more test responses from volttron modular repos, - # put the name of the event type in the list below. - # The name of the event type should come from Workflow - # that is triggering this workflow. - # For example, 'VOLTTRON/volttron-listener-agent` repo has a workflow - # called downstream-testing-dispatch.yml that will send a repository - # dispatch to this repo using with an event type called "listener-downstream-testing-response" - types: [listener-downstream-testing-response] - -jobs: - test-response: - runs-on: ubuntu-22.04 - - steps: - - name: Repository Dispatch Triggered - run: | - echo "πŸŽ‰ The job was automatically triggered by a ${{ github.event_name }} event." - echo "Event '${{ github.event.action }}' received from '${{ github.event.client_payload.repository }}'" - echo "Payload from downstream workflow: '${{ toJson(github.event.client_payload) }}'" \ No newline at end of file diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml new file mode 100644 index 0000000..4449eb1 --- /dev/null +++ b/.github/workflows/make-release.yml @@ -0,0 +1,59 @@ +name: Make full release + +on: + workflow_dispatch: + inputs: + publish-to-test-pypi: + description: 'Publish to test pypi instead of pypi' + required: false + default: false + type: boolean + bump-rule: + description: 'Rule for computing next release version' + required: false + default: 'prerelease' + type: choice + options: + - patch + - minor + - major + - prepatch + - preminor + - premajor + - prerelease + release-version: + description: 'Version number to use(instead of computing). Should be of the format x.y.z[rcn]. Do not use hyphens.' + required: false + default: '' + type: string + merge-strategy: + description: 'Merge strategy and strategy options. Used only in case of merge conflicts' + required: false + default: '' + type: string + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: '3.10' + +jobs: + call-deploy-release: + permissions: + contents: write # To push a branch + pull-requests: write # To create a PR from that branch + + uses: eclipse-volttron/github-tooling/.github/workflows/deploy-release.yml@main + with: + merge-strategy: ${{ inputs.merge-strategy }} + release-version: ${{ inputs.release-version }} + bump-rule: ${{ inputs.bump-rule }} + run-tests-wait: 600 + publish-to-test-pypi: false + secrets: + git-token: ${{ secrets.AUTO_PROJECT_PAT }} + pypi-token: ${{ secrets.PYPI_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml deleted file mode 100644 index b378626..0000000 --- a/.github/workflows/publish-to-pypi.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# Documentation located -# https://github.com/marketplace/actions/publish-python-poetry-package -name: Publish to PyPi - -on: - release: - types: [published] - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - -jobs: - - publish_to_pypi: - - runs-on: ubuntu-22.04 - - steps: - - run: echo "πŸŽ‰ The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "πŸ”Ž The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Build and publish to pypi - uses: JRubics/poetry-publish@v1.7 - with: - # These are only needed when using test.pypi - #repository_name: testpypi - #repository_url: https://test.pypi.org/legacy/ - pypi_token: ${{ secrets.PYPI_TOKEN }} - ignore_dev_requirements: "yes" diff --git a/.github/workflows/run-downstream-tests.yml b/.github/workflows/run-downstream-tests.yml deleted file mode 100644 index dddaa2f..0000000 --- a/.github/workflows/run-downstream-tests.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Start downstream tests - -on: - push: - branches: - - main - pull_request: - branches: - - develop - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYTHON_VERSION: '3.10' - EVENT_TYPE: downstream-testing - -jobs: - initiate_downstream_testing: - strategy: - matrix: - # To add more Volttron modular repos that need to be tested against a version of volttron-core, - # add the name of repo to the list below. - # The name of the repo takes the following form: / - # For example, to add the OpenADR agent repo, use the following name: - # VOLTTRON/volttron-openadr-ven - repo: [ eclipse-volttron/volttron-sqlite-historian ] - # , 'VOLTTRON/volttron-openadr-ven' ] - - runs-on: ubuntu-22.04 - - steps: - - run: env - - run: echo "πŸŽ‰ The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "πŸ”Ž The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python ${{ env.PYTHON_VERSION }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_VERSION }} - - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction - - - name: Create build artifacts - run: | - poetry build -vvv - - - name: Check if wheels were built - run: | - ls -lh dist - - - name: Upload wheels as artifact - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist - - - run: echo "Workflow run_id ${{ github.run_id }}" - - - name: Repository Dispatch - uses: peter-evans/repository-dispatch@v1.1.3 - with: - token: ${{ secrets.ACTION_HOOK_TOKEN }} - repository: ${{ matrix.repo }} - event-type: ${{ env.EVENT_TYPE }} - client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "repository": "${{ github.repository }}", "api_url": "${{ github.api_url }}", "run_id": "${{ github.run_id }}" }' \ No newline at end of file diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 444eb91..9055563 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -3,17 +3,20 @@ name: Run Pytests on: push: paths: - - '**.py' - - '**.yaml' - - '**.yml' - - '**.toml' + - '**.py' + - '**.yaml' + - '**.yml' + - '**.toml' pull_request: types: - - opened - - synchronize - - reopened + - opened + - synchronize + - reopened paths: - - '**.py' + - '**.py' + - '**.yaml' + - '**.yml' + - '**.toml' defaults: run: @@ -23,74 +26,60 @@ env: LANG: en_US.utf-8 LC_ALL: en_US.utf-8 PYTHON_VERSION: '3.10' - PROJECT_NAME: volttron-lib-base-historian + PROJECT_NAME: volttron-lib-sql-historian jobs: run-tests: strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04] - python: ['3.8', '3.9', '3.10'] + os: [ "ubuntu-22.04" ] + python: [ "3.10", "3.11" ] runs-on: ${{ matrix.os }} steps: - - run: echo "πŸŽ‰ The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "πŸ”Ž The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + - run: echo "πŸŽ‰ The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "πŸ”Ž The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." #---------------------------------------------- # check-out repo and set-up python #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v3 + - name: Checkout code + uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} + - name: Set up Python ${{ matrix.python }} + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.2.2 - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true -# #---------------------------------------------- -# # load cached venv if cache exists -# #---------------------------------------------- -# - name: Load cached venv -# id: cached-poetry-dependencies -# uses: actions/cache@v2.1.7 -# with: -# path: .venv -# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} -# #---------------------------------------------- -# # install dependencies if cache does not exist -# #---------------------------------------------- -# - name: Install dependencies -# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' -# run: poetry install --no-interaction --no-root + #---------------------------------------------- + # install your root project, and run tests. + #---------------------------------------------- + - name: Check file existence + id: check_files + uses: andstor/file-existence-action@v2 + with: + files: tests #---------------------------------------------- # install your root project, and run tests. #---------------------------------------------- - - name: Check file existence - id: check_files - uses: andstor/file-existence-action@v2 - with: - files: tests - - - name: Install library and run tests - if: steps.check_files.outputs.files_exists == 'true' - run: | - poetry install --no-interaction - poetry add --group dev pytest-github-actions-annotate-failures - poetry run pytest --cov=src tests/ + - name: Install library and run tests + if: steps.check_files.outputs.files_exists == 'true' + run: | + poetry install --no-interaction + poetry add --group dev pytest-github-actions-annotate-failures + poetry run pytest --cov=src diff --git a/README.md b/README.md index 76f8011..5cd9868 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![Eclipse VOLTTRONβ„’](https://img.shields.io/badge/Eclips%20VOLTTRON--red.svg)](https://volttron.readthedocs.io/en/latest/) +![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg) +![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg) [![pypi version](https://img.shields.io/pypi/v/volttron-lib-sql-historian.svg)](https://pypi.org/project/volttron-lib-sql-historian/) ![Passing?](https://github.com/VOLTTRON/volttron-lib-sql-historian/actions/workflows/run-tests.yml/badge.svg) @@ -8,7 +11,7 @@ installed as a VOLTTRON agent. ## Requirements - - Python >= 3.8 + - Python >= 3.10 ## Installation diff --git a/docs/source/index.rst b/docs/source/index.rst index 0eaf1a5..e1d1861 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,22 +1,87 @@ -Welcome to VOLTTRON SQL Historian's documentation! -=================================== +.. _SQLHistorian-Library: -**VOLTTRON SQL Historian** is ... +SQLHistorian Library +==================== -Check out the :doc:`usage` section for further information, including -how to :ref:`installation` the project. +volttron-lib-sql-historian library provides a SQLHistorian class that extends the +`BaseHistorian library `_ and could serves as a base class for any +historian that stores its data in an RDBMS based data store. It inherits from the +`BaseHistorian class `_ and provides all the necessary validation and processing +logic needed by a historian and abstracts only the actual database specific SQL queries to be implemented by an +inheriting historian class. -.. note:: +An historian that extends from SQLHistorian need to create only a single class that inherits the +`DbDriver class `_ +and save it in a module named "historian.{database_type}.{database_type}functs.py" where database_type is the type of +the database as configured in the historian's configuration file. For example, below shows a minimal configuration +for SQLiteHistorian that store data in a sqlite database. - This project is under active development. +.. code-block:: -Contents --------- + { + "connection": { + "type": "sqlite", + "params": { + "database": "data/historian.sqlite" + } + } + } -.. toctree:: +The SQLiteHistorian contains a single module named sqlitefuncts.py in the package historian.sqlite. This module +contains the class that inherit the `DbDriver class `_ - usage - agent - quick-start - fail-url-test-on-purpose +SQLHistorian assume that the data is store in two tables +1. A table to store topics and its metadata. Default table name is "topics" +2. A table to store the actual data. Default table name is "data" + + +.. _SQL-Historian-Configurations: + +Optional Configuration +---------------------- + +The table names can be configured using the following configuration parameters. + +.. code-block:: + + "tables_def": { + "table_prefix": "volttron", + "data_table": "data_table", + "topics_table": "topics_table" + } + +If a *table_prefix* is configured, the resultant table name would be .. A +*table_prefix* configuration is useful when multiple historians' store data in the same database. + +Below example, shows a configuration for a historian that store data in a MySQL data. + +.. code-block:: + + { + "connection": { + "type": "mysql", + "params": { + "host": "localhost", + "port": 3306, + "database": "test_historian", + "user": "historian", + "passwd": "historian" + } + }, + "tables_def": { + "table_prefix": "volttron", + "data_table": "data_table", + "topics_table": "topics_table" + } + } + +A class that implements DBDriver should implement all the abstract methods defined in the DBDriver class, where each +method provides the database specific SQL query for a specific database operation such as setting up the historian +tables, insert a topic into topics table, update metadata for a topic, insert data for a topic etc. + +Additional Configurations +------------------------- + +Since SQLHistorian extends from the BaseHistorian, all the configurations supported by the BaseHistorian are also +available to any class that extends from the SQLHistorian. Please refer to :ref:`Base Historian Configurations ` diff --git a/pyproject.toml b/pyproject.toml index b7fcd4a..103eebb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,6 @@ [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=1.2.0"] build-backend = "poetry.core.masonry.api" -git-changelog = ">=0.5.0" -httpx = ">=0.16.1" -jinja2-cli = ">=0.7.0" -toml = ">=0.10.2" [tool.poetry] name = "volttron-lib-sql-historian" @@ -19,8 +15,8 @@ keywords = [] packages = [ { include = "historian", from = "src" } ] [tool.poetry.dependencies] -python = ">=3.8,<4.0" -volttron-lib-base-historian = "^0.2.0rc0" +python = ">=3.10,<4.0" +volttron-lib-base-historian = "^0.2.0rc4" [tool.poetry.group.dev.dependencies] pytest = "^6.2.5" @@ -33,8 +29,8 @@ safety = "^1.10.3" mypy = "^0.942" coverage = "^6.3.2" pytest-cov = "^3.0.0" -Sphinx = "^4.5.0" -sphinx-rtd-theme = "^1.0.0" +Sphinx = "^6.0.0" +sphinx-rtd-theme = "^1.2.0" [tool.yapf] based_on_style = "pep8"