From 2fbade7fa2ef4f4863570e8e6ddd1fc04c32f7c5 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 27 Apr 2022 11:11:51 -0700 Subject: [PATCH 01/24] Initial transfer from volttron devlop branch commit reference 127dff6 --- .copier-answers.yml | 20 + .github/ISSUE_TEMPLATE/bug_report.md | 32 ++ .github/ISSUE_TEMPLATE/feature_request.md | 20 + .github/workflows/ci.yml | 101 ++++ .github/workflows/code_analysis.yml | 70 +++ .github/workflows/create_release.yml | 116 +++++ .github/workflows/publish_to_pypi.yml | 39 ++ README.md | 117 +++++ config | 10 + docs/Makefile | 22 + docs/source/conf.py | 36 ++ docs/source/index.rst | 22 + mypy.ini | 8 + pyproject.toml | 58 +++ src/volttron/historian/sql/__init__.py | 10 + src/volttron/historian/sql/basedb.py | 559 ++++++++++++++++++++++ src/volttron/historian/sql/historian.py | 379 +++++++++++++++ src/volttron/historian/sql/sqlutils.py | 60 +++ 18 files changed, 1679 insertions(+) create mode 100644 .copier-answers.yml create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/code_analysis.yml create mode 100644 .github/workflows/create_release.yml create mode 100644 .github/workflows/publish_to_pypi.yml create mode 100644 README.md create mode 100644 config create mode 100644 docs/Makefile create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 mypy.ini create mode 100644 pyproject.toml create mode 100644 src/volttron/historian/sql/__init__.py create mode 100644 src/volttron/historian/sql/basedb.py create mode 100644 src/volttron/historian/sql/historian.py create mode 100644 src/volttron/historian/sql/sqlutils.py diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..ad4f45e --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,20 @@ +# Changes here will be overwritten by Copier +_commit: v0.3.0-beta-2-g3b3bf71 +_src_path: gh:VOLTTRON/copier-poetry-volttron-agent +author_email: volttron@pnnl.gov +author_fullname: VOLTTRON +author_username: VOLTTRON +copyright_date: '2022' +copyright_holder: PNNL +copyright_holder_email: volttron@pnnl.gov +copyright_license: Apache License 2.0 +project_description: null +project_name: VOLTTRON SQL Historian +python_agent_class_name: Volttron_sql_historian +python_package_command_line_name: volttron-sql-historian +python_package_distribution_name: volttron-sql-historian +python_package_import_name: volttron.historian.sql +repository_name: volttron-sql-historian +repository_namespace: VOLTTRON +repository_provider: github.com +use_precommit: true diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..4ffcf24 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: unconfirmed +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Run command '...' +3. Scroll down to '...' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**System (please complete the following information):** +- `VOLTTRON SQL Historian` version: [e.g. 0.2.1] +- Python version: [e.g. 3.8] +- OS: [Windows/Linux] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..4fe86d5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e7db949 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,101 @@ +name: ci + +on: + push: + branches: + - main + pull_request: + branches: + - main + - develop + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHONIOENCODING: UTF-8 + + # To fix an error when running Poetry on Windows + # (https://github.com/python-poetry/poetry/issues/2629), + # we set Poetry's cache directory to .poetry_cache in the current directory. + # It makes it easier to later remove the virtualenv when it's broken. + # Absolute path is necessary to avoid this issue: + # https://github.com/python-poetry/poetry/issues/3049 + POETRY_CACHE_DIR: ${{ github.workspace }}/.poetry_cache + +jobs: + + quality: + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Set up Poetry + run: pip install poetry + + - name: Set up the cache + uses: actions/cache@v1 + with: + path: .poetry_cache + key: quality-poetry-cache + + - name: Set up the project + run: poetry install -vv + + - name: Check if the documentation builds correctly + run: poetry run duty check-docs + + - name: Check the code quality + run: poetry run duty check-code-quality + + - name: Check if the code is correctly typed + run: poetry run duty check-types + + - name: Check for vulnerabilities in dependencies + run: | + pip install safety + poetry run duty check-dependencies + + tests: + + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.8, 3.9, 3.10.0] + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up Poetry + run: pip install poetry + + - name: Set up the cache + uses: actions/cache@v1 + with: + path: .poetry_cache + key: tests-poetry-cache-${{ matrix.os }}-py${{ matrix.python-version }} + + - name: Set up the project + run: poetry install -vv || { rm -rf .poetry_cache/virtualenvs/*; poetry install -vv; } + + - name: Run the test suite + run: poetry run duty test diff --git a/.github/workflows/code_analysis.yml b/.github/workflows/code_analysis.yml new file mode 100644 index 0000000..8fae426 --- /dev/null +++ b/.github/workflows/code_analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: CodeQL + +on: + push: + branches: [main, develop, releases] + pull_request: + # The branches below must be a subset of the branches above + branches: [main, develop, releases] + schedule: + - cron: 34 10 * * 4 + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-20.04 + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [python] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml new file mode 100644 index 0000000..c02617c --- /dev/null +++ b/.github/workflows/create_release.yml @@ -0,0 +1,116 @@ +name: Create Release + +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - v* # Push events to matching v*, i.e. v1.0, v20.15.10 + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: '3.8' + PROJECT_NAME: volttron-core + +jobs: + + autorelease: + + runs-on: ubuntu-20.04 + + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2.1.7 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: poetry install --no-interaction + + #---------------------------------------------------------------- + # get project version from pyproject.toml; store in env var + #---------------------------------------------------------------- + - name: Add version to environment vars + run: | + PROJECT_VERSION=$(poetry version --short) + echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV + - name: Guardrail to check if tag version matches project version + run: | + TAG=$(git describe HEAD --tags --abbrev=0) + echo $TAG + echo $PROJECT_VERSION + if [[ "$TAG" != "v$PROJECT_VERSION" ]]; then exit 1; fi + + #--------------------------------------------------------------- + # create build artifacts to be included as part of release + #--------------------------------------------------------------- + - name: Create build artifacts + run: | + poetry build -vvv + + #---------------------------------------------------------------- + # create release notes; + # the git log command will check all commits since the last tag to HEAD. + # For each commit, appends the commit hash, the commit message + # subject, the author name, and the author email to the release template. + #---------------------------------------------------------------- + - name: Release Notes + run: git log $(git describe HEAD~ --tags --abbrev=0)..HEAD --pretty='format:* %h %s%n * %an <%ae>' --no-merges >> ".github/RELEASE-TEMPLATE.md" + + #---------------------------------------------------------------- + # create release draft which will be visible at https://github.com/VOLTTRON/volttron-core/releases + #---------------------------------------------------------------- + - name: Create Release Draft + uses: softprops/action-gh-release@v1 + with: + body_path: .github/RELEASE-TEMPLATE.md + draft: true + files: | + dist/${{ env.PROJECT_NAME }}-${{env.PROJECT_VERSION}}-py3-none-any.whl + dist/${{ env.PROJECT_NAME }}-${{env.PROJECT_VERSION}}.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml new file mode 100644 index 0000000..85963cc --- /dev/null +++ b/.github/workflows/publish_to_pypi.yml @@ -0,0 +1,39 @@ +# Documentation located +# https://github.com/marketplace/actions/publish-python-poetry-package +name: Publish to PyPi + +on: + release: + types: [published] + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + +jobs: + + publish_to_pypi: + + runs-on: ubuntu-20.04 + + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build and publish to pypi + uses: JRubics/poetry-publish@v1.7 + with: + # These are only needed when using test.pypi + #repository_name: testpypi + #repository_url: https://test.pypi.org/legacy/ + pypi_token: ${{ secrets.PYPI_TOKEN }} + ignore_dev_requirements: yes diff --git a/README.md b/README.md new file mode 100644 index 0000000..73690f9 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +# VOLTTRON SQL Historian + +[![ci](https://github.com/VOLTTRON/volttron-sql-historian/workflows/ci/badge.svg)](https://github.com/VOLTTRON/volttron-sql-historian/actions?query=workflow%3Aci) +[![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://VOLTTRON.github.io/volttron-sql-historian/) +[![pypi version](https://img.shields.io/pypi/v/volttron-sql-historian.svg)](https://pypi.org/project/volttron-sql-historian/) + + +None + +## Prerequisites + +* Python 3.8 +* Poetry + +### Python +VOLTTRON SQL Historian requires Python 3.8 or above. + + +To install Python 3.8, we recommend using [pyenv](https://github.com/pyenv/pyenv). + +```bash +# install pyenv +git clone https://github.com/pyenv/pyenv ~/.pyenv + +# setup pyenv (you should also put these three lines in .bashrc or similar) +export PATH="${HOME}/.pyenv/bin:${PATH}" +export PYENV_ROOT="${HOME}/.pyenv" +eval "$(pyenv init -)" + +# install Python 3.8 +pyenv install 3.8.10 + +# make it available globally +pyenv global system 3.8.10 +``` + +### Poetry + +This project uses `poetry` to install and manage dependencies. To install poetry, +follow these [instructions](https://python-poetry.org/docs/master/#installation). + + + +## Installation and Virtual Environment Setup + +If you want to install all your dependencies, including dependencies to help with developing your agent, run this command: + +```poetry install``` + +If you want to install only the dependencies needed to run your agent, run this command: + +```poetry install --no-dev``` + +Set the environment to be in your project directory: + +```poetry config virtualenvs.in-project true``` + +Activate the virtual environment: + +```poetry shell``` + + +## Git Setup + +1. To use git to manage version control, create a new git repository in your local agent project. + +``` +git init +``` + +2. Then create a new repo in your Github or Gitlab account. Copy the URL that points to that new repo in +your Github or Gitlab account. This will be known as our 'remote'. + +3. Add the remote (i.e. the new repo URL from your Github or Gitlab account) to your local repository. Run the following command: + +```git remote add origin ``` + +When you push to your repo, note that the default branch is called 'main'. + + +## Optional Configurations + +## Precommit + +Install pre-commit hooks: + +```pre-commit install``` + +To run pre-commit on all your files, run this command: + +```pre-commit run --all-files``` + +If you have precommit installed and you want to ignore running the commit hooks +every time you run a commit, include the `--no-verify` flag in your commit. The following +is an example: + +```git commit -m "Some message" --no-verify``` + +# Documentation + +To build the docs, navigate to the 'docs' directory and build the documentation: + +```shell +cd docs +make html +``` + +After the documentation is built, view the documentation in html form in your browser. +The html files will be located in `~/docs/build/html`. + +**PROTIP: To open the landing page of your documentation directly from the command line, run the following command:** + +```shell +open /docs/build/html/index.html +``` + +This will open the documentation landing page in your default browsert (e.g. Chrome, Firefox). diff --git a/config b/config new file mode 100644 index 0000000..0966014 --- /dev/null +++ b/config @@ -0,0 +1,10 @@ +{ + # VOLTTRON config files are JSON with support for python style comments. + "setting1": 2, # Integers + "setting2": "some/random/topic2", #Strings + "setting3": true, # Booleans: remember that in JSON true and false are not capitalized. + "setting4": false, + "setting5": 5.1, # Floating point numbers. + "setting6": [1,2,3,4], #Lists + "setting7": {"setting7a": "a", "setting7b": "b"} #Objects +} diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..4b65e60 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,22 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build +SPHINXPROJ = VOLTTRON SQL Historian + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..59a00f8 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,36 @@ +# Configuration file for the Sphinx documentation builder. + +# -- Project information + +project = 'VOLTTRON SQL Historian' +copyright = '2022, PNNL' +author = 'PNNL' + +release = '0.1' +version = '0.1.0' + +# -- General configuration + +extensions = [ + 'sphinx.ext.duration', + 'sphinx.ext.doctest', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', +] + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), +} +intersphinx_disabled_domains = ['std'] + +templates_path = ['_templates'] + +# -- Options for HTML output + +html_theme = 'sphinx_rtd_theme' + +# -- Options for EPUB output +# epub_show_urls = 'footnote' + diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..0eaf1a5 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,22 @@ +Welcome to VOLTTRON SQL Historian's documentation! +=================================== + +**VOLTTRON SQL Historian** is ... + +Check out the :doc:`usage` section for further information, including +how to :ref:`installation` the project. + +.. note:: + + This project is under active development. + +Contents +-------- + +.. toctree:: + + usage + agent + quick-start + fail-url-test-on-purpose + diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..e4c5a50 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +; For a complete list of options, see https://mypy.readthedocs.io/en/stable/config_file.html +python_version = 3.9 +show_error_context = True +pretty = True +show_column_numbers = True +warn_return_any = True +warn_unused_configs = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..66cc775 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,58 @@ +[tool.poetry] +name = "volttron-sql-historian" +version = "0.1.0" +description = "None" +authors = ["VOLTTRON "] +license = "Apache License 2.0" +readme = "README.md" +repository = "https://github.com/VOLTTRON/volttron-sql-historian" +homepage = "https://github.com/VOLTTRON/volttron-sql-historian" +keywords = [] +packages = [ { include = "volttron", from = "src" } ] + +[tool.poetry.dependencies] +python = ">=3.8,<4.0" +# TODO: uncomment when volttron 0.1.0 is available +# volttron = "^0.1.0" +volttron-base-historian = {path = "../volttron-base-historian", develop = true} + +[tool.poetry.dev-dependencies] +# formatting, quality, tests +pytest = "^6.2.5" +mock = "^4.0.3" +pre-commit = "^2.17.0" +yapf = "^0.32.0" +toml = "^0.10.2" +isort = "^5.10.1" +safety = "^1.10.3" +mypy = "^0.942" +coverage = "^6.3.2" +Sphinx = "^4.5.0" +sphinx-rtd-theme = "^1.0.0" + +[tool.yapfignore] +ignore_patterns = [ + ".venv/**", + ".pytest_cache/**", + "dist/**", + "docs/**" +] + +[tool.yapf] +based_on_style = "pep8" +spaces_before_comment = 4 +column_limit = 99 +split_before_logical_operator = true + +[tool.poetry.scripts] +volttron-sql-historian = "volttron.historian.sql.agent:main" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +# tasks +git-changelog = ">=0.5.0" +httpx = ">=0.16.1" +jinja2-cli = ">=0.7.0" +toml = ">=0.10.2" diff --git a/src/volttron/historian/sql/__init__.py b/src/volttron/historian/sql/__init__.py new file mode 100644 index 0000000..69ae4b2 --- /dev/null +++ b/src/volttron/historian/sql/__init__.py @@ -0,0 +1,10 @@ +""" +VOLTTRON SQL Historian package. + +None + +""" + +from typing import List + +__all__: List[str] = [] # noqa: WPS410 (the only __variable__ we use) diff --git a/src/volttron/historian/sql/basedb.py b/src/volttron/historian/sql/basedb.py new file mode 100644 index 0000000..7fe4f92 --- /dev/null +++ b/src/volttron/historian/sql/basedb.py @@ -0,0 +1,559 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} + + +import contextlib +import importlib +import logging +import threading +import sqlite3 +import sys +from abc import abstractmethod +from gevent.local import local + +from volttron.platform.agent import utils +from volttron.platform import jsonapi + +utils.setup_logging() +_log = logging.getLogger(__name__) + + +class ConnectionError(Exception): + """ + Custom class for connection errors + """ + pass + + +@contextlib.contextmanager +def closing(obj): + try: + yield obj + finally: + try: + obj.close() + except BaseException as exc: + # if exc.__class__.__module__ == 'exceptions': + if exc.__class__.__module__ == 'builtins': + # Don't ignore built-in exceptions because they likely indicate a bug that should stop execution. + # psycopg2.Error subclasses Exception, so the module must also be checked. + raise + _log.exception('An exception was raised while closing the cursor and is being ignored.') + + +class DbDriver: + """ + Parent class used by :py:class:`sqlhistorian.historian.SQLHistorian` to + do the database operations. This class is inherited by + - :py:class:`volttron.platform.dbutils.mysqlfuncts.MySqlFuncts` + - :py:class:`volttron.platform.dbutils.sqlitefuncts.SqlLiteFuncts` + """ + def __init__(self, dbapimodule, **kwargs): + thread_name = threading.currentThread().getName() + if callable(dbapimodule): + _log.debug("Constructing Driver for %s in thread: %s", dbapimodule.__name__, thread_name) + connect = dbapimodule + else: + _log.debug("Constructing Driver for %s in thread: %s", dbapimodule, thread_name) + _log.debug("kwargs for connect is %r", kwargs) + dbapimodule = importlib.import_module(dbapimodule) + connect = lambda: dbapimodule.connect(**kwargs) + self.__connect = connect + self.__connection = None + self.stash = local() + + @contextlib.contextmanager + def bulk_insert(self): + """ + Function to meet bulk insert requirements. This function can be overridden by historian drivers to yield the + required method for data insertion during bulk inserts in the respective historians. In this generic case it + will yield the single insert method + :yields: insert method + """ + yield self.insert_data + + @contextlib.contextmanager + def bulk_insert_meta(self): + """ + Function to meet bulk insert requirements. This function can be overridden by historian drivers to yield the + required method for meta insertion during bulk inserts in the respective historians. In this generic case it + will yield the single insert method + :yields: insert method + """ + yield self.insert_meta + + def cursor(self): + + self.stash.cursor = None + if self.__connection is not None and not getattr(self.__connection, "closed", False): + try: + self.stash.cursor = self.__connection.cursor() + return self.stash.cursor + except Exception: + _log.warning("An exception occurred while creating a cursor. Will try establishing connection again") + self.__connection = None + try: + self.__connection = self.__connect() + except Exception as e: + _log.error("Could not connect to database. Raise ConnectionError") + raise ConnectionError(e).with_traceback(sys.exc_info()[2]) + if self.__connection is None: + raise ConnectionError("Unknown error. Could not connect to database") + + # if any exception happens here have it go to the caller. + self.stash.cursor = self.__connection.cursor() + + return self.stash.cursor + + @abstractmethod + def setup_historian_tables(self): + """ + Create historian tables if necessary + """ + pass + + @abstractmethod + def get_topic_map(self): + """ + Returns details of topics in database + :return: two dictionaries. + - First one maps topic_name.lower() to topic id and + - Second one maps topic_name.lower() to topic name + """ + pass + + @abstractmethod + def get_agg_topics(self): + """ + Get the list of aggregate topics available + :return: list of tuples containing + (agg_topic_name, agg_type, agg_time_period, configured topics/topic name pattern) + """ + pass + + @abstractmethod + def get_agg_topic_map(self): + """ + Get a map of aggregate_topics to aggregate_topic_id + :return: dict of format + {(agg_topic_name, agg_type, agg_time_period):agg_topic_id} + """ + pass + + @abstractmethod + def query_topics_by_pattern(self, topic_pattern): + """ + Return a map of {topic_name.lower():topic_id} that matches the given pattern + :param topic_pattern: pattern to match against topic_name + :return: + """ + pass + + @abstractmethod + def get_topic_meta_map(self): + """ + Returns details of metadata in the database + :return: dictionary of format {topic_id:{metadata}} + """ + + @abstractmethod + def insert_data_query(self): + """ + :return: query string to insert data into database + """ + pass + + @abstractmethod + def insert_topic_query(self): + """ + :return: query string to insert a topic into database + """ + pass + + @abstractmethod + def insert_topic_and_meta_query(self): + """ + Return insert statement to insert both topic and meta data into the same table. + This is used if topic table contains metadata column instead of storing metadata in a separate table + """ + pass + + @abstractmethod + def update_topic_query(self): + """ + :return: query string to update a topic in database + """ + pass + + @abstractmethod + def insert_meta_query(self): + """ + :return: query string to insert metadata for a topic into database + """ + pass + + @abstractmethod + def update_topic_and_meta_query(self): + """ + :return: query string to update both metadata and topic_name field in self.topics_table. This is used from + SQLHistorian version 4.0.0 + """ + + @abstractmethod + def update_meta_query(self): + """ + :return: query string to update metadata field in self.topics_table. This is used from + SQLHistorian version 4.0.0 + """ + + @abstractmethod + def get_aggregation_list(self): + """ + Return list of aggregation supported by the specific data store + :return: list of aggregations + """ + pass + + @abstractmethod + def insert_agg_topic_stmt(self): + """ + :return: query string to insert an aggregate topic into database + """ + pass + + @abstractmethod + def update_agg_topic_stmt(self): + """ + :return: query string to update an aggregate topic in database + """ + pass + + @abstractmethod + def replace_agg_meta_stmt(self): + """ + :return: query string to insert metadata for an aggregate topic into + database + """ + pass + + def manage_db_size(self, history_limit_timestamp, storage_limit_gb): + """ + Optional function to manage database size. + :param history_limit_timestamp: remove all data older than this timestamp + :param storage_limit_gb: remove oldest data until database is smaller than this value. + """ + pass + + def insert_meta(self, topic_id, metadata): + """ + Inserts metadata for topic + :param topic_id: topic id for which metadata is inserted + :param metadata: metadata + :return: True if execution completes. Raises exception if unable to connect to database + """ + self.execute_stmt(self.insert_meta_query(), (topic_id, jsonapi.dumps(metadata)), commit=False) + return True + + def update_meta(self, topic_id, metadata): + """ + Inserts metadata for topic + :param topic_id: topic id for which metadata is inserted + :param metadata: metadata + :return: True if execution completes. Raises exception if unable to connect to database + """ + self.execute_stmt(self.update_meta_query(), (jsonapi.dumps(metadata), topic_id), commit=False) + return True + + def insert_data(self, ts, topic_id, data): + """ + Inserts data for topic + :param ts: timestamp + :param topic_id: topic id for which data is inserted + :param data: data value + :return: True if execution completes. raises Exception if unable to connect to database + """ + self.execute_stmt(self.insert_data_query(), (ts, topic_id, jsonapi.dumps(data)), commit=False) + return True + + def insert_topic(self, topic, **kwargs): + """ + Insert a new topic + :param topic: topic to insert + :return: id of the topic inserted if insert was successful. Raises exception if unable to connect to database + """ + meta = kwargs.get('metadata') + insert_topic_only = True + if self.meta_table == self.topics_table and topic and meta: + value = (topic, jsonapi.dumps(kwargs.get("metadata"))) + query = self.insert_topic_and_meta_query() + else: + value = (topic,) + query = self.insert_topic_query() + + with closing(self.cursor()) as cursor: + _log.debug(f"Inserting topic {query} {value}") + cursor.execute(query, value) + return cursor.lastrowid + + def update_topic(self, topic, topic_id, **kwargs): + """ + Update a topic name + :param topic: new topic name + :param topic_id: topic id for which update is done + :return: True if execution is complete. Raises exception if unable to connect to database + """ + meta = kwargs.get('metadata') + if self.meta_table == self.topics_table and topic and meta: + self.execute_stmt(self.update_topic_and_meta_query(), (topic, jsonapi.dumps(meta), topic_id), + commit=False) + else: + # either topic and meta table are separate or no meta was sent + self.execute_stmt(self.update_topic_query(), (topic, topic_id), commit=False) + return True + + def insert_agg_meta(self, topic_id, metadata): + """ + Inserts metadata for aggregate topic + :param topic_id: aggregate topic id for which metadata is inserted + :param metadata: metadata + :return: True if execution completes. Raises exception if connection to database fails + """ + self.execute_stmt(self.replace_agg_meta_stmt(), (topic_id, jsonapi.dumps(metadata)), commit=False) + return True + + def insert_agg_topic(self, topic, agg_type, agg_time_period): + """ + Insert a new aggregate topic + :param topic: topic name to insert + :param agg_type: type of aggregation + :param agg_time_period: time period of aggregation + :return: id of the topic inserted if insert was successful. Raises exception if unable to connect to database + """ + with closing(self.cursor()) as cursor: + cursor.execute(self.insert_agg_topic_stmt(), (topic, agg_type, agg_time_period)) + return cursor.lastrowid + + def update_agg_topic(self, agg_id, agg_topic_name): + """ + Update a aggregate topic name + :param agg_id: topic id for which update is done + :param agg_topic_name: new aggregate topic name + :return: True if execution is complete. Raises exception if unable to + connect to database + """ + self.execute_stmt(self.update_agg_topic_stmt(), (agg_topic_name, agg_id),commit=False) + return True + + def commit(self): + """ + Commit a transaction + + :return: True if successful, False otherwise + """ + if self.__connection is not None: + try: + self.__connection.commit() + return True + except sqlite3.OperationalError as e: + if "database is locked" in str(e): + _log.error("EXCEPTION: SQLITE3 Database is locked. This error could occur when there are multiple " + "simultaneous read and write requests, making individual request to wait more than the " + "default timeout period. If you are using sqlite for frequent reads and write, please " + "configure a higher timeout in agent configuration under \nconfig[\"connection\"]" + "[\"params\"][\"timeout\"] Default value is 10. Timeout units is seconds") + raise + _log.warning('connection was null during commit phase.') + return False + + def rollback(self): + """ + Rollback a transaction + :return: True if successful, False otherwise + """ + if self.__connection is not None: + self.__connection.rollback() + return True + _log.warning('connection was null during rollback phase.') + return False + + def close(self): + """ + Close connection to database + :return: + """ + if self.__connection is not None: + self.__connection.close() + + def select(self, query, args=None, fetch_all=True): + """ + Execute a select statement + :param query: select statement + :param args: arguments for the where clause + :param fetch_all: Set to True if function should return retrieve all + the records from cursors and return it. Set to False to return cursor. + :return: resultant rows if fetch_all is True else returns the cursor + It is up to calling method to close the cursor + """ + if not args: + args = () + cursor = self.cursor() + try: + cursor.execute(query, args) + except Exception: + cursor.close() + raise + if fetch_all: + with closing(cursor): + return cursor.fetchall() + return cursor + + def execute_stmt(self, stmt, args=None, commit=False): + """ + Execute a sql statement + :param stmt: the statement to execute + :param args: optional arguments + :param commit: True if transaction should be committed. Defaults to False + :return: count of the number of affected rows + """ + if args is None: + args = () + with closing(self.cursor()) as cursor: + cursor.execute(stmt, args) + if commit: + self.commit() + return cursor.rowcount + + def execute_many(self, stmt, args, commit=False): + """ + Execute a sql statement with multiple args + :param stmt: the statement to execute + :param args: optional arguments + :param commit: True if transaction should be committed. Defaults to False + :return: count of the number of affected rows + """ + with closing(self.cursor()) as cursor: + cursor.executemany(stmt, args) + if commit: + self.commit() + return cursor.rowcount + + @abstractmethod + def query(self, topic_ids, id_name_map, start=None, end=None, agg_type=None, agg_period=None, skip=0, count=None, + order="FIRST_TO_LAST"): + """ + Queries the raw historian data or aggregate data and returns the results of the query + :param topic_ids: list of topic ids to query for. + :param id_name_map: dictionary that maps topic id to topic name + :param start: Start of query timestamp as a datetime. + :param end: End of query timestamp as a datetime. + :param agg_type: If this is a query for aggregate data, the type of aggregation ( for example, sum, avg) + :param agg_period: If this is a query for aggregate data, the time period of aggregation + :param skip: Skip this number of results. + :param count: Limit results to this value. When the query is for multiple topics, count applies to individual + topics. For example, a query on 2 topics with count=5 will return 5 records for each topic + :param order: How to order the results, either "FIRST_TO_LAST" or "LAST_TO_FIRST" + :type start: datetime + :type end: datetime + :type skip: int + :type count: int + :type order: str + :return: result of the query in the format: + .. code-block:: python + + { + topic_name:[(timestamp1, value1), + (timestamp2:,value2), + ...], + topic_name:[(timestamp1, value1), + (timestamp2:,value2), + ...], + ...} + """ + pass + + @abstractmethod + def create_aggregate_store(self, agg_type, period): + """ + Create the data structure (table or collection) that is going to store the aggregate data for the give + aggregation type and aggregation time period. Table name should be constructed as _ + :param agg_type: The type of aggregation. (avg, sum etc.) + :param period: The time period of aggregation + :return: True if successful, False otherwise + """ + pass + + @abstractmethod + def insert_aggregate_stmt(self, table_name): + """ + The sql statement to insert collected aggregate for a given time period into database + :param table_name: name of the table into which the aggregate data needs to be inserted + :return: sql insert/replace statement to insert aggregate data for a specific time slice + :rtype: str + """ + pass + + def insert_aggregate(self, agg_topic_id, agg_type, period, ts, data, topic_ids): + """ + Insert aggregate data collected for a specific time period into + database. Data is inserted into _ table + :param agg_topic_id: topic id + :param agg_type: type of aggregation + :param period: time period of aggregation + :param ts: end time of aggregation period (not inclusive) + :param data: a float that represents a computed aggregate + :param topic_ids: topic ids or topic ids for which aggregate was computed + :return: True if execution was successful, raises exception in case of connection failures + """ + table_name = agg_type + '_' + period + _log.debug("Inserting aggregate: {} {} {} {} into table {}".format( + ts, agg_topic_id, data, str(topic_ids), table_name)) + self.execute_stmt(self.insert_aggregate_stmt(table_name), + (ts, agg_topic_id, data, str(topic_ids)), commit=True) + return True + + @abstractmethod + def collect_aggregate(self, topic_ids, agg_type, start=None, end=None): + """ + Collect the aggregate data by querying the historian's data store + :param topic_ids: list of topic ids for which aggregation should be performed. + :param agg_type: type of aggregation + :param start: start time for query (inclusive) + :param end: end time for query (exclusive) + :return: a tuple of (aggregated value, count of records over which this aggregation was computed) + """ + pass diff --git a/src/volttron/historian/sql/historian.py b/src/volttron/historian/sql/historian.py new file mode 100644 index 0000000..bd63e2a --- /dev/null +++ b/src/volttron/historian/sql/historian.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} + + +import logging +import sys +import threading + +from volttron.platform.agent import utils +from volttron.platform.agent.base_historian import BaseHistorian +from volttron.platform.dbutils import sqlutils +from volttron.utils.docs import doc_inherit + +__version__ = "4.0.0" + +utils.setup_logging() +_log = logging.getLogger(__name__) + + +class MaskedString(str): + def __repr__(self): + return repr('********') + + +def historian(config_path, **kwargs): + """ + This method is called by the :py:func:`sqlhistorian.historian.main` to + parse the passed config file or configuration dictionary object, validate + the configuration entries, and create an instance of SQLHistorian + :param config_path: could be a path to a configuration file or can be a + dictionary object + :param kwargs: additional keyword arguments if any + :return: an instance of :py:class:`sqlhistorian.historian.SQLHistorian` + """ + if isinstance(config_path, dict): + config_dict = config_path + else: + config_dict = utils.load_config(config_path) + + connection = config_dict.get('connection', None) + + assert connection is not None + database_type = connection.get('type', None) + assert database_type is not None + params = connection.get('params', None) + assert params is not None + + # Avoid printing passwords in the debug message + for key in ['pass', 'passwd', 'password', 'pw']: + try: + params[key] = MaskedString(params[key]) + except KeyError: + pass + + SQLHistorian.__name__ = 'SQLHistorian' + utils.update_kwargs_with_config(kwargs, config_dict) + _log.debug("In sql historian before calling class kwargs is {}".format(kwargs)) + return SQLHistorian(**kwargs) + + +class SQLHistorian(BaseHistorian): + """ + This is a historian agent that writes data to a SQLite or Mysql + database based on the connection parameters in the configuration. + .. seealso:: + - :py:mod:`volttron.platform.dbutils.basedb` + - :py:mod:`volttron.platform.dbutils.mysqlfuncts` + - :py:mod:`volttron.platform.dbutils.sqlitefuncts` + """ + + def __init__(self, connection, tables_def=None, **kwargs): + """Initialise the historian. + + The historian makes two connections to the data store. Both of + these connections are available across the main and processing + thread of the historian. topic_map and topic_meta are used as + cache for the meta data and topic maps. + + :param connection: dictionary that contains necessary information to + establish a connection to the sql database. The dictionary should + contain two entries - + + 1. 'type' - describe the type of database (sqlite or mysql) + 2. 'params' - parameters for connecting to the database. + + :param tables_def: optional parameter. dictionary containing the + names to be used for historian tables. Should contain the following + keys + + 1. "table_prefix": - if specified tables names are prefixed with + this value followed by a underscore + 2."data_table": name of the table that stores historian data, + 3."topics_table": name of the table that stores the list of topics + for which historian contains data data + 4. "meta_table": name of the table that stores the metadata data + for topics + + :param kwargs: additional keyword arguments. + """ + self.connection = connection + self.tables_def, self.table_names = self.parse_table_def(tables_def) + self.topic_id_map = {} + self.topic_name_map = {} + self.topic_meta = {} + self.agg_topic_id_map = {} + # Create two instance so connection is shared within a single thread. + # This is because sqlite only supports sharing of connection within + # a single thread. + # historian_setup and publish_to_historian happens in background thread + # everything else happens in the MainThread + + # One utils class instance( hence one db connection) for main thread + self.main_thread_dbutils = self.get_dbfuncts_object() + # One utils class instance( hence one db connection) for background thread + # this gets initialized in the bg_thread within historian_setup + self.bg_thread_dbutils = None + super(SQLHistorian, self).__init__(**kwargs) + + def manage_db_size(self, history_limit_timestamp, storage_limit_gb): + """ + Optional function to manage database size. + """ + self.bg_thread_dbutils.manage_db_size(history_limit_timestamp, storage_limit_gb) + + @doc_inherit + def version(self): + return __version__ + + @doc_inherit + def publish_to_historian(self, to_publish_list): + try: + published = 0 + with self.bg_thread_dbutils.bulk_insert() as insert_data, \ + self.bg_thread_dbutils.bulk_insert_meta() as insert_meta: + + for x in to_publish_list: + ts = x['timestamp'] + topic = x['topic'] + value = x['value'] + meta = x['meta'] + + # look at the topics that are stored in the database already to see if this topic has a value + lowercase_name = topic.lower() + topic_id = self.topic_id_map.get(lowercase_name, None) + db_topic_name = self.topic_name_map.get(lowercase_name, + None) + old_meta = self.topic_meta.get(topic_id, {}) + update_topic_meta = True + if topic_id is None: + # send metadata data too. If topics table contains metadata column too it will get inserted + topic_id = self.bg_thread_dbutils.insert_topic(topic, metadata=meta) + # user lower case topic name when storing in map for case insensitive comparison + self.topic_name_map[lowercase_name] = topic + self.topic_id_map[lowercase_name] = topic_id + update_topic_meta = False + elif db_topic_name != topic: + if old_meta != meta: + _log.debug(f"META HAS CHANGED TOO. old:{old_meta} new:{meta}") + # pass metadata if metadata is stored in topics table metadata will get updated too + # if not will get ignored + self.bg_thread_dbutils.update_topic(topic, topic_id, metadata=meta) + update_topic_meta = False + else: + self.bg_thread_dbutils.update_topic(topic, topic_id) + self.topic_name_map[lowercase_name] = topic + + if old_meta != meta: + if self.bg_thread_dbutils.topics_table != self.bg_thread_dbutils.meta_table: + # there is a separate metadata table. do bulk insert + _log.debug("meta in separate table") + insert_meta(topic_id, meta) + elif update_topic_meta: + _log.debug(" meta in same table. no topic change only meta changed") + # topic name and metadata are in same table, and metadata has not got into db during insert + # or update of topic so update meta alone in topics table + self.bg_thread_dbutils.update_meta(metadata=meta, topic_id=topic_id) + + # either way update cache + self.topic_meta[topic_id] = meta + + if insert_data(ts, topic_id, value): + published += 1 + + if published: + if self.bg_thread_dbutils.commit(): + _log.debug("Reporting all handled") + self.report_all_handled() + else: + _log.warning('Commit error. Rolling back {} values.'.format(published)) + self.bg_thread_dbutils.rollback() + else: + _log.warning('Unable to publish {}'.format(len(to_publish_list))) + except Exception as e: + # TODO Unable to send alert from here + # if isinstance(e, ConnectionError): + # _log.debug("Sending alert. Exception {}".format(e.args)) + # err_message = "Unable to connect to database. Exception:{}".format(e.args) + # alert_id = DB_CONNECTION_FAILURE + # else: + # err_message = "Unknown exception when publishing data. Exception: {}".format(e.args) + # alert_id = ERROR_PUBLISHING_DATA + # self.vip.health.set_status(STATUS_BAD, err_message) + # status = Status.from_json(self.vip.health.get_status()) + # self.vip.health.send_alert(alert_id, status) + self.bg_thread_dbutils.rollback() + # Raise to the platform so it is logged properly. + raise + + @doc_inherit + def query_topic_list(self): + + _log.debug("query_topic_list Thread is: {}".format(threading.currentThread().getName())) + if len(self.topic_name_map) > 0: + return list(self.topic_name_map.values()) + else: + # No topics present. + return [] + + @doc_inherit + def query_topics_by_pattern(self, topic_pattern): + return self.main_thread_dbutils.query_topics_by_pattern(topic_pattern) + + @doc_inherit + def query_topics_metadata(self, topics): + meta = {} + if isinstance(topics, str): + topic_id = self.topic_id_map.get(topics.lower()) + if topic_id: + meta = {topics: self.topic_meta.get(topic_id)} + elif isinstance(topics, list): + for topic in topics: + topic_id = self.topic_id_map.get(topic.lower()) + if topic_id: + meta[topic] = self.topic_meta.get(topic_id) + return meta + + def query_aggregate_topics(self): + return self.main_thread_dbutils.get_agg_topics() + + @doc_inherit + def query_historian(self, topic, start=None, end=None, agg_type=None, agg_period=None, skip=0, count=None, + order="FIRST_TO_LAST"): + _log.debug("query_historian Thread is: {}".format(threading.currentThread().getName())) + results = dict() + topics_list = [] + if isinstance(topic, str): + topics_list.append(topic) + elif isinstance(topic, list): + topics_list = topic + + multi_topic_query = len(topics_list) > 1 + + topic_ids = [] + id_name_map = {} + for topic in topics_list: + topic_lower = topic.lower() + topic_id = self.topic_id_map.get(topic_lower) + if agg_type: + agg_type = agg_type.lower() + topic_id = self.agg_topic_id_map.get((topic_lower, agg_type, agg_period)) + if topic_id is None: + # load agg topic id again as it might be a newly configured aggregation + agg_map = self.main_thread_dbutils.get_agg_topic_map() + self.agg_topic_id_map.update(agg_map) + _log.debug(" Agg topic map after updating {} ".format(self.agg_topic_id_map)) + topic_id = self.agg_topic_id_map.get((topic_lower, agg_type, agg_period)) + if topic_id: + topic_ids.append(topic_id) + id_name_map[topic_id] = topic + else: + _log.warning('No such topic {}'.format(topic)) + + if not topic_ids: + _log.warning('No topic ids found for topics{}. Returning empty result'.format(topics_list)) + return results + + _log.debug("Querying db reader with topic_ids {} ".format(topic_ids)) + + values = self.main_thread_dbutils.query(topic_ids, id_name_map, start=start, end=end, agg_type=agg_type, + agg_period=agg_period, skip=skip, count=count, order=order) + meta_tid = None + if len(values) > 0: + # If there are results add metadata if it is a query on a single topic + if not multi_topic_query: + values = list(values.values())[0] + if agg_type: + # if aggregation is on single topic find the topic id in the topics table that corresponds to + # agg_topic_id so that we can grab the correct metadata if topic name does not have entry in + # topic_id_map it is a user configured aggregation_topic_name which denotes aggregation across + # multiple points + _log.debug("Single topic aggregate query. Try to get metadata") + meta_tid = self.topic_id_map.get(topic.lower(), None) + else: + # this is a query on raw data, get metadata for topic from topic_meta map + meta_tid = topic_ids[0] + + if values: + metadata = self.topic_meta.get(meta_tid, {}) + results = {'values': values, 'metadata': metadata} + else: + results = dict() + return results + + @doc_inherit + def historian_setup(self): + thread_name = threading.currentThread().getName() + _log.info("historian_setup on Thread: {}".format(thread_name)) + self.bg_thread_dbutils = self.get_dbfuncts_object() + + if not self._readonly: + self.bg_thread_dbutils.setup_historian_tables() + + topic_id_map, topic_name_map = self.bg_thread_dbutils.get_topic_map() + self.topic_id_map.update(topic_id_map) + self.topic_name_map.update(topic_name_map) + self.agg_topic_id_map = self.bg_thread_dbutils.get_agg_topic_map() + topic_meta_map = self.bg_thread_dbutils.get_topic_meta_map() + self.topic_meta.update(topic_meta_map) + _log.debug(f"###DEBUG Loaded topics and metadata on start. Len of topics {len(self.topic_id_map)} " + f"Len of metadata: {len(self.topic_meta)}") + + def get_dbfuncts_object(self): + db_functs_class = sqlutils.get_dbfuncts_class(self.connection['type']) + return db_functs_class(self.connection['params'], self.table_names) + + +def main(argv=sys.argv): + """ + Main entry point for the agent. + """ + + try: + utils.vip_main(historian, version=__version__) + except Exception as e: + print(e) + _log.exception('unhandled exception') + + +if __name__ == '__main__': + # Entry point for script + try: + sys.exit(main()) + except KeyboardInterrupt: + pass diff --git a/src/volttron/historian/sql/sqlutils.py b/src/volttron/historian/sql/sqlutils.py new file mode 100644 index 0000000..0c655d1 --- /dev/null +++ b/src/volttron/historian/sql/sqlutils.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} +import inspect +import logging + +from volttron.platform.agent import utils +from volttron.platform.dbutils.basedb import DbDriver + +utils.setup_logging() +_log = logging.getLogger(__name__) + + +def get_dbfuncts_class(database_type): + mod_name = database_type + "functs" + mod_name_path = "volttron.platform.dbutils.{}".format(mod_name) + loaded_mod = __import__(mod_name_path, fromlist=[mod_name]) + for _, cls in inspect.getmembers(loaded_mod): + # Ensure class is not the root dbdriver + if (inspect.isclass(cls) and issubclass(cls, DbDriver) + and cls is not DbDriver): + break + else: + raise Exception('Invalid module named {}'.format(mod_name_path)) + _log.debug('Historian using module: {}'.format(cls.__name__)) + return cls From 6c4f85aba97f9650cf8eeba3e237654211a41bb0 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Wed, 4 May 2022 18:57:16 -0700 Subject: [PATCH 02/24] initial commit with import fixed --- src/volttron/historian/sql/__init__.py | 42 ++++++++++++++++++++++++- src/volttron/historian/sql/basedb.py | 6 ++-- src/volttron/historian/sql/historian.py | 24 +++++++------- src/volttron/historian/sql/sqlutils.py | 6 ++-- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/volttron/historian/sql/__init__.py b/src/volttron/historian/sql/__init__.py index 69ae4b2..956e251 100644 --- a/src/volttron/historian/sql/__init__.py +++ b/src/volttron/historian/sql/__init__.py @@ -1,3 +1,40 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2022, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} """ VOLTTRON SQL Historian package. @@ -6,5 +43,8 @@ """ from typing import List +from volttron.historian.sql.basedb import DbDriver -__all__: List[str] = [] # noqa: WPS410 (the only __variable__ we use) +__all__: List[str] = [ + "DbDriver", +] # noqa: WPS410 (the only __variable__ we use) diff --git a/src/volttron/historian/sql/basedb.py b/src/volttron/historian/sql/basedb.py index 7fe4f92..4430a4c 100644 --- a/src/volttron/historian/sql/basedb.py +++ b/src/volttron/historian/sql/basedb.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- {{{ # vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: # -# Copyright 2020, Battelle Memorial Institute. +# Copyright 2022, Battelle Memorial Institute. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,8 +46,8 @@ from abc import abstractmethod from gevent.local import local -from volttron.platform.agent import utils -from volttron.platform import jsonapi +from volttron import utils +from volttron.utils import jsonapi utils.setup_logging() _log = logging.getLogger(__name__) diff --git a/src/volttron/historian/sql/historian.py b/src/volttron/historian/sql/historian.py index bd63e2a..9a4608a 100644 --- a/src/volttron/historian/sql/historian.py +++ b/src/volttron/historian/sql/historian.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- {{{ # vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: # -# Copyright 2020, Battelle Memorial Institute. +# Copyright 2022, Battelle Memorial Institute. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,10 +41,10 @@ import sys import threading -from volttron.platform.agent import utils -from volttron.platform.agent.base_historian import BaseHistorian -from volttron.platform.dbutils import sqlutils -from volttron.utils.docs import doc_inherit +from volttron import utils +from volttron.historian.base import BaseHistorian +from volttron.historian.sql import sqlutils +#from volttron.utils. import doc_inherit __version__ = "4.0.0" @@ -157,11 +157,11 @@ def manage_db_size(self, history_limit_timestamp, storage_limit_gb): """ self.bg_thread_dbutils.manage_db_size(history_limit_timestamp, storage_limit_gb) - @doc_inherit + #@doc_inherit def version(self): return __version__ - @doc_inherit + #@doc_inherit def publish_to_historian(self, to_publish_list): try: published = 0 @@ -241,7 +241,7 @@ def publish_to_historian(self, to_publish_list): # Raise to the platform so it is logged properly. raise - @doc_inherit + #@doc_inherit def query_topic_list(self): _log.debug("query_topic_list Thread is: {}".format(threading.currentThread().getName())) @@ -251,11 +251,11 @@ def query_topic_list(self): # No topics present. return [] - @doc_inherit + #@doc_inherit def query_topics_by_pattern(self, topic_pattern): return self.main_thread_dbutils.query_topics_by_pattern(topic_pattern) - @doc_inherit + #@doc_inherit def query_topics_metadata(self, topics): meta = {} if isinstance(topics, str): @@ -272,7 +272,7 @@ def query_topics_metadata(self, topics): def query_aggregate_topics(self): return self.main_thread_dbutils.get_agg_topics() - @doc_inherit + #@doc_inherit def query_historian(self, topic, start=None, end=None, agg_type=None, agg_period=None, skip=0, count=None, order="FIRST_TO_LAST"): _log.debug("query_historian Thread is: {}".format(threading.currentThread().getName())) @@ -336,7 +336,7 @@ def query_historian(self, topic, start=None, end=None, agg_type=None, agg_period results = dict() return results - @doc_inherit + #@doc_inherit def historian_setup(self): thread_name = threading.currentThread().getName() _log.info("historian_setup on Thread: {}".format(thread_name)) diff --git a/src/volttron/historian/sql/sqlutils.py b/src/volttron/historian/sql/sqlutils.py index 0c655d1..08d69e1 100644 --- a/src/volttron/historian/sql/sqlutils.py +++ b/src/volttron/historian/sql/sqlutils.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- {{{ # vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: # -# Copyright 2020, Battelle Memorial Institute. +# Copyright 2022, Battelle Memorial Institute. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,8 +38,8 @@ import inspect import logging -from volttron.platform.agent import utils -from volttron.platform.dbutils.basedb import DbDriver +from volttron import utils +from volttron.historian.sql import DbDriver utils.setup_logging() _log = logging.getLogger(__name__) From 8a4b7a1bc6ddaf15b8afe9275a91ab6bf0b9b60f Mon Sep 17 00:00:00 2001 From: Chandrika Date: Fri, 13 May 2022 17:29:21 -0700 Subject: [PATCH 03/24] path to volttron-core --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 66cc775..4e152ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ packages = [ { include = "volttron", from = "src" } ] python = ">=3.8,<4.0" # TODO: uncomment when volttron 0.1.0 is available # volttron = "^0.1.0" -volttron-base-historian = {path = "../volttron-base-historian", develop = true} [tool.poetry.dev-dependencies] # formatting, quality, tests @@ -29,6 +28,7 @@ mypy = "^0.942" coverage = "^6.3.2" Sphinx = "^4.5.0" sphinx-rtd-theme = "^1.0.0" +volttron-base-historian = {path = "../volttron-base-historian", develop = true} [tool.yapfignore] ignore_patterns = [ From 3a469196927db233f274d06d9e92a0479559206a Mon Sep 17 00:00:00 2001 From: Chandrika Date: Tue, 17 May 2022 13:34:50 -0700 Subject: [PATCH 04/24] removed legacy support from base historian. fixed sqlite module import --- src/volttron/historian/sql/sqlutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/volttron/historian/sql/sqlutils.py b/src/volttron/historian/sql/sqlutils.py index 08d69e1..4f7d83d 100644 --- a/src/volttron/historian/sql/sqlutils.py +++ b/src/volttron/historian/sql/sqlutils.py @@ -47,7 +47,7 @@ def get_dbfuncts_class(database_type): mod_name = database_type + "functs" - mod_name_path = "volttron.platform.dbutils.{}".format(mod_name) + mod_name_path = f"volttron.historian.{database_type}.{mod_name}" loaded_mod = __import__(mod_name_path, fromlist=[mod_name]) for _, cls in inspect.getmembers(loaded_mod): # Ensure class is not the root dbdriver From a5b93cf4fc741084e83cc114e129e37fdcced386 Mon Sep 17 00:00:00 2001 From: Chandrika Date: Fri, 15 Jul 2022 14:35:29 -0700 Subject: [PATCH 05/24] changed name and changed volttron dependency to wheel --- pyproject.toml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e938768..efa55d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,17 @@ [tool.poetry] -name = "volttron-sql-historian" +name = "volttron-lib-sql-historian" version = "0.1.0" description = "None" authors = ["VOLTTRON "] license = "Apache License 2.0" readme = "README.md" -repository = "https://github.com/VOLTTRON/volttron-sql-historian" -homepage = "https://github.com/VOLTTRON/volttron-sql-historian" +repository = "https://github.com/VOLTTRON/volttron-lib-sql-historian" +homepage = "https://github.com/VOLTTRON/volttron-lib-sql-historian" keywords = [] packages = [ { include = "volttron", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -# TODO: uncomment when volttron 0.1.0 is available -# volttron = "^0.1.0" [tool.poetry.dev-dependencies] # formatting, quality, tests @@ -28,8 +26,8 @@ mypy = "^0.942" coverage = "^6.3.2" Sphinx = "^4.5.0" sphinx-rtd-theme = "^1.0.0" -volttron-base-historian = {path = "../volttron-base-historian", develop = true} -volttron-testing = {path = "../volttron-testing", develop = true} +volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} + [tool.yapfignore] ignore_patterns = [ From b8d3c3fa0f2bf051ed290eda8e9bf5a8483f3980 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Tue, 15 Nov 2022 17:43:25 -0800 Subject: [PATCH 06/24] Initial tested version of SQL historian for VOLTTRON modular code base --- .github/workflows/deploy-pre-release.yml | 123 +++++++++++++++++++++++ README.md | 13 ++- pyproject.toml | 12 +-- 3 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/deploy-pre-release.yml diff --git a/.github/workflows/deploy-pre-release.yml b/.github/workflows/deploy-pre-release.yml new file mode 100644 index 0000000..9e3d8ee --- /dev/null +++ b/.github/workflows/deploy-pre-release.yml @@ -0,0 +1,123 @@ +name: Deploy Pre-Release Artifacts + +on: + push: + branches: + - develop + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: '3.8' + POETRY_VERSION: 1.2.2 + +jobs: + + deploy-pre-release: + runs-on: ubuntu-20.04 + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: | + poetry install --no-interaction + + #---------------------------------------------- + # bump version number for patch + #---------------------------------------------- + - name: Bump Version + run: | + # current_tag is the last tagged release in the repository. + # From there we need to remove the 'v' from the beginning of the tag. + if ! $(git tag -l "v*" = ''); then + # uses -V which is version sort to keep it monotonically increasing. + current_tag=$(git tag -l "v*" | sort --reverse -V |sed -n 1p) + else + current_tag=v0.1.0 + fi + + current_tag=${current_tag#?} + + # current_tag is now the version we want to set our poetry version so + # that we can bump the version + poetry version ${current_tag} + poetry version prerelease --no-interaction + + # Finally because we want to be able to use the variable in later + # steps we set a NEW_TAG environmental variable + NEW_TAG=v$(poetry version --short) + echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + + #--------------------------------------------------------------- + # create build artifacts to be included as part of release + #--------------------------------------------------------------- + - name: Create build artifacts + run: | + poetry build -vvv + + - uses: ncipollo/release-action@v1 + with: + artifacts: dist/*.gz,dist/*.whl + artifactErrorsFailBuild: true + generateReleaseNotes: true + commit: ${{ github.ref }} + prerelease: true + tag: ${{ env.NEW_TAG }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish pre-release to pypi + if: github.repository == 'eclipse-volttron/volttron-lib-sql-historian' + run: | + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish \ No newline at end of file diff --git a/README.md b/README.md index 73690f9..4a833e5 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ # VOLTTRON SQL Historian -[![ci](https://github.com/VOLTTRON/volttron-sql-historian/workflows/ci/badge.svg)](https://github.com/VOLTTRON/volttron-sql-historian/actions?query=workflow%3Aci) -[![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://VOLTTRON.github.io/volttron-sql-historian/) -[![pypi version](https://img.shields.io/pypi/v/volttron-sql-historian.svg)](https://pypi.org/project/volttron-sql-historian/) +[![ci](https://github.com/VOLTTRON/volttron-sql-historian/workflows/ci/badge.svg)](https://github.com/eclipse-volttron/volttron-lib-sql-historian/actions?query=workflow%3Aci) +[![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://VOLTTRON.github.io/volttron-lib-sql-historian/) +[![pypi version](https://img.shields.io/pypi/v/volttron-sql-historian.svg)](https://pypi.org/project/volttron-lib-sql-historian/) -None +Generic SQL Historian library that can be used to implement a historian agent with a relational database backend. +This library cannot be installed as a VOLTTRON agent as is. Only a concrete database implementation package such as +[sqlite-historian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) that depends on this library can be +installed as a VOLTTRON agent. ## Prerequisites @@ -114,4 +117,4 @@ The html files will be located in `~/docs/build open /docs/build/html/index.html ``` -This will open the documentation landing page in your default browsert (e.g. Chrome, Firefox). +This will open the documentation landing page in your default browser (e.g. Chrome, Firefox). diff --git a/pyproject.toml b/pyproject.toml index e70ee2b..47f1c8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,17 +5,17 @@ description = "None" authors = ["VOLTTRON "] license = "Apache License 2.0" readme = "README.md" -repository = "https://github.com/VOLTTRON/volttron-lib-sql-historian" -homepage = "https://github.com/VOLTTRON/volttron-lib-sql-historian" +repository = "https://github.com/eclipse-volttron/volttron-lib-sql-historian" +homepage = "https://github.com/eclipse-volttron/volttron-lib-sql-historian" keywords = [] packages = [ { include = "volttron", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" # TODO: uncomment when volttron 0.1.0 is available -# volttron = "^0.1.0" +#volttron-lib-base-historian="^0.1.0" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] # formatting, quality, tests pytest = "^6.2.5" mock = "^4.0.3" @@ -28,7 +28,7 @@ mypy = "^0.942" coverage = "^6.3.2" Sphinx = "^4.5.0" sphinx-rtd-theme = "^1.0.0" -volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} +#volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} [tool.yapfignore] @@ -46,7 +46,7 @@ column_limit = 99 split_before_logical_operator = true [tool.poetry.scripts] -volttron-sql-historian = "volttron.historian.sql.agent:main" +volttron-sql-historian = "volttron.historian.sql.historian:main" [build-system] requires = ["poetry-core>=1.0.0"] From 852f2401b3b092cfbf50650c993ea844e3277fb2 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Wed, 16 Nov 2022 12:10:03 -0800 Subject: [PATCH 07/24] removed unused ci.yml --- .github/workflows/ci.yml | 101 --------------------------------------- 1 file changed, 101 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index e7db949..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: ci - -on: - push: - branches: - - main - pull_request: - branches: - - main - - develop - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYTHONIOENCODING: UTF-8 - - # To fix an error when running Poetry on Windows - # (https://github.com/python-poetry/poetry/issues/2629), - # we set Poetry's cache directory to .poetry_cache in the current directory. - # It makes it easier to later remove the virtualenv when it's broken. - # Absolute path is necessary to avoid this issue: - # https://github.com/python-poetry/poetry/issues/3049 - POETRY_CACHE_DIR: ${{ github.workspace }}/.poetry_cache - -jobs: - - quality: - - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Set up Poetry - run: pip install poetry - - - name: Set up the cache - uses: actions/cache@v1 - with: - path: .poetry_cache - key: quality-poetry-cache - - - name: Set up the project - run: poetry install -vv - - - name: Check if the documentation builds correctly - run: poetry run duty check-docs - - - name: Check the code quality - run: poetry run duty check-code-quality - - - name: Check if the code is correctly typed - run: poetry run duty check-types - - - name: Check for vulnerabilities in dependencies - run: | - pip install safety - poetry run duty check-dependencies - - tests: - - strategy: - matrix: - os: [ubuntu-latest] - python-version: [3.8, 3.9, 3.10.0] - - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up Poetry - run: pip install poetry - - - name: Set up the cache - uses: actions/cache@v1 - with: - path: .poetry_cache - key: tests-poetry-cache-${{ matrix.os }}-py${{ matrix.python-version }} - - - name: Set up the project - run: poetry install -vv || { rm -rf .poetry_cache/virtualenvs/*; poetry install -vv; } - - - name: Run the test suite - run: poetry run duty test From 73397d563e0f5d86f7a269eeaaf5b948463385e8 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Wed, 16 Nov 2022 13:52:16 -0800 Subject: [PATCH 08/24] updated github workflow files --- .github/workflows/auto-assign-project.yml | 22 ++ .../{code_analysis.yml => code-analysis.yml} | 14 +- .github/workflows/create_release.yml | 116 ---------- .github/workflows/deploy-pre-release.yml | 211 +++++++++--------- .../workflows/downstream-test-response.yml | 24 ++ .github/workflows/publish-to-pypi.yml | 40 ++++ .github/workflows/publish_to_pypi.yml | 39 ---- .github/workflows/run-downstream-tests.yml | 88 ++++++++ .github/workflows/run-test.yml | 86 +++++++ 9 files changed, 374 insertions(+), 266 deletions(-) create mode 100644 .github/workflows/auto-assign-project.yml rename .github/workflows/{code_analysis.yml => code-analysis.yml} (90%) delete mode 100644 .github/workflows/create_release.yml create mode 100644 .github/workflows/downstream-test-response.yml create mode 100644 .github/workflows/publish-to-pypi.yml delete mode 100644 .github/workflows/publish_to_pypi.yml create mode 100644 .github/workflows/run-downstream-tests.yml create mode 100644 .github/workflows/run-test.yml diff --git a/.github/workflows/auto-assign-project.yml b/.github/workflows/auto-assign-project.yml new file mode 100644 index 0000000..8427606 --- /dev/null +++ b/.github/workflows/auto-assign-project.yml @@ -0,0 +1,22 @@ +name: Add bugs to bugs project + +on: + issues: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.3.0 + with: + # You can target a repository in a different organization + # to the issue + project-url: https://github.com/orgs/eclipse-volttron/projects/3 + # project-url: https://github.com/orgs//projects/ + # github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + github-token: ${{ secrets.AUTO_PROJECT_PAT }} + # labeled: bug, needs-triage + # label-operator: OR \ No newline at end of file diff --git a/.github/workflows/code_analysis.yml b/.github/workflows/code-analysis.yml similarity index 90% rename from .github/workflows/code_analysis.yml rename to .github/workflows/code-analysis.yml index 8fae426..e64de06 100644 --- a/.github/workflows/code_analysis.yml +++ b/.github/workflows/code-analysis.yml @@ -9,21 +9,21 @@ # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # -name: CodeQL +name: "CodeQL" on: push: - branches: [main, develop, releases] + branches: [ main, develop, releases ] pull_request: # The branches below must be a subset of the branches above - branches: [main, develop, releases] + branches: [ main, develop, releases ] schedule: - - cron: 34 10 * * 4 + - cron: '34 10 * * 4' jobs: analyze: name: Analyze - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest permissions: actions: read contents: read @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - language: [python] + language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support @@ -67,4 +67,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v1 \ No newline at end of file diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml deleted file mode 100644 index c02617c..0000000 --- a/.github/workflows/create_release.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Create Release - -on: - push: - # Sequence of patterns matched against refs/tags - tags: - - v* # Push events to matching v*, i.e. v1.0, v20.15.10 - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYTHON_VERSION: '3.8' - PROJECT_NAME: volttron-core - -jobs: - - autorelease: - - runs-on: ubuntu-20.04 - - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python ${{ env.PYTHON_VERSION }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_VERSION }} - - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2.1.7 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction - - #---------------------------------------------------------------- - # get project version from pyproject.toml; store in env var - #---------------------------------------------------------------- - - name: Add version to environment vars - run: | - PROJECT_VERSION=$(poetry version --short) - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV - - name: Guardrail to check if tag version matches project version - run: | - TAG=$(git describe HEAD --tags --abbrev=0) - echo $TAG - echo $PROJECT_VERSION - if [[ "$TAG" != "v$PROJECT_VERSION" ]]; then exit 1; fi - - #--------------------------------------------------------------- - # create build artifacts to be included as part of release - #--------------------------------------------------------------- - - name: Create build artifacts - run: | - poetry build -vvv - - #---------------------------------------------------------------- - # create release notes; - # the git log command will check all commits since the last tag to HEAD. - # For each commit, appends the commit hash, the commit message - # subject, the author name, and the author email to the release template. - #---------------------------------------------------------------- - - name: Release Notes - run: git log $(git describe HEAD~ --tags --abbrev=0)..HEAD --pretty='format:* %h %s%n * %an <%ae>' --no-merges >> ".github/RELEASE-TEMPLATE.md" - - #---------------------------------------------------------------- - # create release draft which will be visible at https://github.com/VOLTTRON/volttron-core/releases - #---------------------------------------------------------------- - - name: Create Release Draft - uses: softprops/action-gh-release@v1 - with: - body_path: .github/RELEASE-TEMPLATE.md - draft: true - files: | - dist/${{ env.PROJECT_NAME }}-${{env.PROJECT_VERSION}}-py3-none-any.whl - dist/${{ env.PROJECT_NAME }}-${{env.PROJECT_VERSION}}.tar.gz - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-pre-release.yml b/.github/workflows/deploy-pre-release.yml index 9e3d8ee..de45c19 100644 --- a/.github/workflows/deploy-pre-release.yml +++ b/.github/workflows/deploy-pre-release.yml @@ -12,112 +12,115 @@ defaults: env: LANG: en_US.utf-8 LC_ALL: en_US.utf-8 - PYTHON_VERSION: '3.8' - POETRY_VERSION: 1.2.2 + PYTHON_VERSION: '3.10' + PROJECT_NAME: volttron-lib-sql-historian + RUNS_ON: ubuntu-22.04 + POETRY_VERSION: '1.2.2' jobs: deploy-pre-release: - runs-on: ubuntu-20.04 + runs-on: ${{env.RUNS_ON}} steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - id: setup-python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: ${{ env.POETRY_VERSION }} - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: | - poetry install --no-interaction - - #---------------------------------------------- - # bump version number for patch - #---------------------------------------------- - - name: Bump Version - run: | - # current_tag is the last tagged release in the repository. - # From there we need to remove the 'v' from the beginning of the tag. - if ! $(git tag -l "v*" = ''); then - # uses -V which is version sort to keep it monotonically increasing. - current_tag=$(git tag -l "v*" | sort --reverse -V |sed -n 1p) - else - current_tag=v0.1.0 - fi - - current_tag=${current_tag#?} - - # current_tag is now the version we want to set our poetry version so - # that we can bump the version - poetry version ${current_tag} - poetry version prerelease --no-interaction - - # Finally because we want to be able to use the variable in later - # steps we set a NEW_TAG environmental variable - NEW_TAG=v$(poetry version --short) - echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV - - #--------------------------------------------------------------- - # create build artifacts to be included as part of release - #--------------------------------------------------------------- - - name: Create build artifacts - run: | - poetry build -vvv - - - uses: ncipollo/release-action@v1 - with: - artifacts: dist/*.gz,dist/*.whl - artifactErrorsFailBuild: true - generateReleaseNotes: true - commit: ${{ github.ref }} - prerelease: true - tag: ${{ env.NEW_TAG }} - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Publish pre-release to pypi - if: github.repository == 'eclipse-volttron/volttron-lib-sql-historian' - run: | - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - poetry publish \ No newline at end of file + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: | + poetry install --no-interaction + + #---------------------------------------------- + # bump version number for patch + #---------------------------------------------- + - name: Bump Version + run: | + # current_tag is the last tagged release in the repository. From there + # we need to remove the v from the beginning of the tag. + if ! $(git tag -l "v*" = ''); then + # uses -V which is version sort to keep it monotonically increasing. + current_tag=$(git tag -l "v*" | grep --invert-match '-' | sort --reverse -V | sed -n 1p) + else + current_tag=v10.0.0 + fi + + current_tag=${current_tag#?} + + # current_tag is now the version we want to set our poetry version so + # that we can bump the version + poetry version ${current_tag} + poetry version prerelease --no-interaction + + NEW_TAG=v$(poetry version --short) + + # Finally because we want to be able to use the variable in later + # steps we set a NEW_TAG environmental variable + echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + + #--------------------------------------------------------------- + # create build artifacts to be included as part of release + #--------------------------------------------------------------- + - name: Create build artifacts + run: | + poetry build -vvv + + - uses: ncipollo/release-action@v1 + with: + artifacts: "dist/*.gz,dist/*.whl" + artifactErrorsFailBuild: true + generateReleaseNotes: true + commit: ${{ github.ref }} + prerelease: true + tag: ${{ env.NEW_TAG }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish pre-release to pypi + if: github.repository == "eclipse-volttron/${{env.PROJECT_NAME}}" + run: | + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish diff --git a/.github/workflows/downstream-test-response.yml b/.github/workflows/downstream-test-response.yml new file mode 100644 index 0000000..8ad1116 --- /dev/null +++ b/.github/workflows/downstream-test-response.yml @@ -0,0 +1,24 @@ +name: Respond to downstream test runs as part of downstream testing + + +on: + repository_dispatch: + # To add more test responses from volttron modular repos, + # put the name of the event type in the list below. + # The name of the event type should come from Workflow + # that is triggering this workflow. + # For example, 'VOLTTRON/volttron-listener-agent` repo has a workflow + # called downstream-testing-dispatch.yml that will send a repository + # dispatch to this repo using with an event type called "listener-downstream-testing-response" + types: [listener-downstream-testing-response] + +jobs: + test-response: + runs-on: ubuntu-22.04 + + steps: + - name: Repository Dispatch Triggered + run: | + echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + echo "Event '${{ github.event.action }}' received from '${{ github.event.client_payload.repository }}'" + echo "Payload from downstream workflow: '${{ toJson(github.event.client_payload) }}'" \ No newline at end of file diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 0000000..b378626 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,40 @@ +--- +# Documentation located +# https://github.com/marketplace/actions/publish-python-poetry-package +name: Publish to PyPi + +on: + release: + types: [published] + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + +jobs: + + publish_to_pypi: + + runs-on: ubuntu-22.04 + + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build and publish to pypi + uses: JRubics/poetry-publish@v1.7 + with: + # These are only needed when using test.pypi + #repository_name: testpypi + #repository_url: https://test.pypi.org/legacy/ + pypi_token: ${{ secrets.PYPI_TOKEN }} + ignore_dev_requirements: "yes" diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml deleted file mode 100644 index 85963cc..0000000 --- a/.github/workflows/publish_to_pypi.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Documentation located -# https://github.com/marketplace/actions/publish-python-poetry-package -name: Publish to PyPi - -on: - release: - types: [published] - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - -jobs: - - publish_to_pypi: - - runs-on: ubuntu-20.04 - - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Build and publish to pypi - uses: JRubics/poetry-publish@v1.7 - with: - # These are only needed when using test.pypi - #repository_name: testpypi - #repository_url: https://test.pypi.org/legacy/ - pypi_token: ${{ secrets.PYPI_TOKEN }} - ignore_dev_requirements: yes diff --git a/.github/workflows/run-downstream-tests.yml b/.github/workflows/run-downstream-tests.yml new file mode 100644 index 0000000..dddaa2f --- /dev/null +++ b/.github/workflows/run-downstream-tests.yml @@ -0,0 +1,88 @@ +name: Start downstream tests + +on: + push: + branches: + - main + pull_request: + branches: + - develop + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: '3.10' + EVENT_TYPE: downstream-testing + +jobs: + initiate_downstream_testing: + strategy: + matrix: + # To add more Volttron modular repos that need to be tested against a version of volttron-core, + # add the name of repo to the list below. + # The name of the repo takes the following form: / + # For example, to add the OpenADR agent repo, use the following name: + # VOLTTRON/volttron-openadr-ven + repo: [ eclipse-volttron/volttron-sqlite-historian ] + # , 'VOLTTRON/volttron-openadr-ven' ] + + runs-on: ubuntu-22.04 + + steps: + - run: env + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: poetry install --no-interaction + + - name: Create build artifacts + run: | + poetry build -vvv + + - name: Check if wheels were built + run: | + ls -lh dist + + - name: Upload wheels as artifact + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist + + - run: echo "Workflow run_id ${{ github.run_id }}" + + - name: Repository Dispatch + uses: peter-evans/repository-dispatch@v1.1.3 + with: + token: ${{ secrets.ACTION_HOOK_TOKEN }} + repository: ${{ matrix.repo }} + event-type: ${{ env.EVENT_TYPE }} + client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "repository": "${{ github.repository }}", "api_url": "${{ github.api_url }}", "run_id": "${{ github.run_id }}" }' \ No newline at end of file diff --git a/.github/workflows/run-test.yml b/.github/workflows/run-test.yml new file mode 100644 index 0000000..82c8a13 --- /dev/null +++ b/.github/workflows/run-test.yml @@ -0,0 +1,86 @@ +name: Run Pytests + +on: + push: + paths: + - '**.py' + - 'poetry.yml' + pull_request: + types: + - opened + - synchronize + - reopened + paths: + - '**.py' + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: '3.10' + PROJECT_NAME: volttron-lib-base-historian + +jobs: + + run-tests: + strategy: + matrix: + os: ["ubuntu-20.04", "ubuntu-22.04"] + python: ["3.8", "3.9", "3.10"] + + runs-on: ${{ matrix.os }} + + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python }} + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.2.2 + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + +# #---------------------------------------------- +# # load cached venv if cache exists +# #---------------------------------------------- +# - name: Load cached venv +# id: cached-poetry-dependencies +# uses: actions/cache@v2.1.7 +# with: +# path: .venv +# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} +# #---------------------------------------------- +# # install dependencies if cache does not exist +# #---------------------------------------------- +# - name: Install dependencies +# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' +# run: poetry install --no-interaction --no-root + + #---------------------------------------------- + # install your root project, and run tests. + #---------------------------------------------- + - name: Install library and run tests + run: | + poetry install --no-interaction + poetry run pytest From 1452b2e4692f326f7d2a264c71994bc06e7234d1 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Wed, 16 Nov 2022 15:08:10 -0800 Subject: [PATCH 09/24] updated readme --- README.md | 145 ++++++++++++++++++++++++++---------------------------- 1 file changed, 70 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 4a833e5..2a2fd14 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -# VOLTTRON SQL Historian [![ci](https://github.com/VOLTTRON/volttron-sql-historian/workflows/ci/badge.svg)](https://github.com/eclipse-volttron/volttron-lib-sql-historian/actions?query=workflow%3Aci) [![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://VOLTTRON.github.io/volttron-lib-sql-historian/) @@ -10,111 +9,107 @@ This library cannot be installed as a VOLTTRON agent as is. Only a concrete data [sqlite-historian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) that depends on this library can be installed as a VOLTTRON agent. -## Prerequisites -* Python 3.8 -* Poetry +## Requirements -### Python -VOLTTRON SQL Historian requires Python 3.8 or above. + - Python >= 3.8 +## Installation -To install Python 3.8, we recommend using [pyenv](https://github.com/pyenv/pyenv). +Create and activate a virtual environment. -```bash -# install pyenv -git clone https://github.com/pyenv/pyenv ~/.pyenv - -# setup pyenv (you should also put these three lines in .bashrc or similar) -export PATH="${HOME}/.pyenv/bin:${PATH}" -export PYENV_ROOT="${HOME}/.pyenv" -eval "$(pyenv init -)" - -# install Python 3.8 -pyenv install 3.8.10 - -# make it available globally -pyenv global system 3.8.10 +```shell +python -m venv env +source env/bin/activate ``` -### Poetry - -This project uses `poetry` to install and manage dependencies. To install poetry, -follow these [instructions](https://python-poetry.org/docs/master/#installation). - - - -## Installation and Virtual Environment Setup - -If you want to install all your dependencies, including dependencies to help with developing your agent, run this command: - -```poetry install``` - -If you want to install only the dependencies needed to run your agent, run this command: - -```poetry install --no-dev``` - -Set the environment to be in your project directory: - -```poetry config virtualenvs.in-project true``` - -Activate the virtual environment: - -```poetry shell``` +Installing volttron-listener requires a running volttron instance. +```shell +pip install volttron -## Git Setup +# Start platform with output going to volttron.log +volttron -vv -l volttron.log & +``` -1. To use git to manage version control, create a new git repository in your local agent project. +Install and start the volttron-listener. -``` -git init +```shell +vctl install volttron-listener --start ``` -2. Then create a new repo in your Github or Gitlab account. Copy the URL that points to that new repo in -your Github or Gitlab account. This will be known as our 'remote'. +View the status of the installed agent -3. Add the remote (i.e. the new repo URL from your Github or Gitlab account) to your local repository. Run the following command: +```shell +vctl status +``` -```git remote add origin ``` +## Development -When you push to your repo, note that the default branch is called 'main'. +Developing on this agent requires poetry 1.2.2 or greater be used. One can install it from https://python-poetry.org/docs/#installation. The VOLTTRON team prefers to have the python environments created within the project directory. Execute +this command to make that behavior the default. +```shell +poetry config virtualenvs.in-project true +``` -## Optional Configurations +Clone the repository. -## Precommit +```shell +git clone https://github.com/eclipse-volttron/volttron-listener +``` -Install pre-commit hooks: +Change to the repository directory and use poetry install to setup the environment. -```pre-commit install``` +```shell +cd volttron-listener +poetry install +``` -To run pre-commit on all your files, run this command: +### Building Wheel -```pre-commit run --all-files``` +To build a wheel from this project execute the following: -If you have precommit installed and you want to ignore running the commit hooks -every time you run a commit, include the `--no-verify` flag in your commit. The following -is an example: +```shell +poetry build +``` -```git commit -m "Some message" --no-verify``` +The wheel and source distribution will be located in the ```./dist/``` directory. -# Documentation +### Bumping version number of project -To build the docs, navigate to the 'docs' directory and build the documentation: +To bump the version number of the project execute one of the following. ```shell -cd docs -make html -``` +# patch, minor, major, prepatch, preminor, premajor, prerelease -After the documentation is built, view the documentation in html form in your browser. -The html files will be located in `~/docs/build/html`. +# use patch +user@path$ poetry patch -**PROTIP: To open the landing page of your documentation directly from the command line, run the following command:** +# output +Bumping version from 0.2.0-alpha.0 to 0.2.0 -```shell -open /docs/build/html/index.html +# use prepatch +user@path$ poetry version prepatch + +# output +Bumping version from 0.2.0 to 0.2.1-alpha.0 ``` -This will open the documentation landing page in your default browser (e.g. Chrome, Firefox). +# Disclaimer Notice + +This material was prepared as an account of work sponsored by an agency of the +United States Government. Neither the United States Government nor the United +States Department of Energy, nor Battelle, nor any of their employees, nor any +jurisdiction or organization that has cooperated in the development of these +materials, makes any warranty, express or implied, or assumes any legal +liability or responsibility for the accuracy, completeness, or usefulness or any +information, apparatus, product, software, or process disclosed, or represents +that its use would not infringe privately owned rights. + +Reference herein to any specific commercial product, process, or service by +trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the United +States Government or any agency thereof, or Battelle Memorial Institute. The +views and opinions of authors expressed herein do not necessarily state or +reflect those of the United States Government or any agency thereof. From 6beeab92125fec610d54a088b5306940dc8dfe63 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Thu, 17 Nov 2022 14:30:53 -0800 Subject: [PATCH 10/24] fixed version number in pyproject.toml and github workflow. Updated README.md --- .github/workflows/deploy-pre-release.yml | 5 +-- README.md | 54 ++---------------------- pyproject.toml | 3 +- 3 files changed, 7 insertions(+), 55 deletions(-) diff --git a/.github/workflows/deploy-pre-release.yml b/.github/workflows/deploy-pre-release.yml index de45c19..518fbe1 100644 --- a/.github/workflows/deploy-pre-release.yml +++ b/.github/workflows/deploy-pre-release.yml @@ -14,13 +14,12 @@ env: LC_ALL: en_US.utf-8 PYTHON_VERSION: '3.10' PROJECT_NAME: volttron-lib-sql-historian - RUNS_ON: ubuntu-22.04 POETRY_VERSION: '1.2.2' jobs: deploy-pre-release: - runs-on: ${{env.RUNS_ON}} + runs-on: ubuntu-22.04 steps: - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" @@ -86,7 +85,7 @@ jobs: # uses -V which is version sort to keep it monotonically increasing. current_tag=$(git tag -l "v*" | grep --invert-match '-' | sort --reverse -V | sed -n 1p) else - current_tag=v10.0.0 + current_tag=v0.1.0 fi current_tag=${current_tag#?} diff --git a/README.md b/README.md index 2a2fd14..16eeb88 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ [![ci](https://github.com/VOLTTRON/volttron-sql-historian/workflows/ci/badge.svg)](https://github.com/eclipse-volttron/volttron-lib-sql-historian/actions?query=workflow%3Aci) -[![documentation](https://img.shields.io/badge/docs-mkdocs%20material-blue.svg?style=flat)](https://VOLTTRON.github.io/volttron-lib-sql-historian/) [![pypi version](https://img.shields.io/pypi/v/volttron-sql-historian.svg)](https://pypi.org/project/volttron-lib-sql-historian/) @@ -16,55 +15,10 @@ installed as a VOLTTRON agent. ## Installation -Create and activate a virtual environment. - -```shell -python -m venv env -source env/bin/activate -``` - -Installing volttron-listener requires a running volttron instance. - -```shell -pip install volttron - -# Start platform with output going to volttron.log -volttron -vv -l volttron.log & -``` - -Install and start the volttron-listener. - -```shell -vctl install volttron-listener --start -``` - -View the status of the installed agent - -```shell -vctl status -``` - -## Development - -Developing on this agent requires poetry 1.2.2 or greater be used. One can install it from https://python-poetry.org/docs/#installation. The VOLTTRON team prefers to have the python environments created within the project directory. Execute -this command to make that behavior the default. - -```shell -poetry config virtualenvs.in-project true -``` - -Clone the repository. - -```shell -git clone https://github.com/eclipse-volttron/volttron-listener -``` - -Change to the repository directory and use poetry install to setup the environment. - -```shell -cd volttron-listener -poetry install -``` +This library can be installed using ```pip install volttron-lib-sql-historian```. However this is not necessary. Any +historian agent that uses this library will automatically install it as part of its installation. For example, +installing [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) will automatically install +volttron-lib-sql-historian ### Building Wheel diff --git a/pyproject.toml b/pyproject.toml index 47f1c8b..5afd1ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,8 +12,7 @@ packages = [ { include = "volttron", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -# TODO: uncomment when volttron 0.1.0 is available -#volttron-lib-base-historian="^0.1.0" +volttron-lib-base-historian="^0.1.1a1" [tool.poetry.group.dev.dependencies] # formatting, quality, tests From e0c067cb20ba99eae1f45e7bec2df4cb8691c195 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Thu, 17 Nov 2022 14:37:12 -0800 Subject: [PATCH 11/24] added dev steps back into README.md --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 16eeb88..3ec6b18 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,29 @@ historian agent that uses this library will automatically install it as part of installing [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) will automatically install volttron-lib-sql-historian +## Development + +Development requires poetry 1.2.2 or greater be used. +One can install it from https://python-poetry.org/docs/#installation. The VOLTTRON team prefers to have the python +environments created within the project directory. Execute this command to make that behavior the default. + +```shell +poetry config virtualenvs.in-project true +``` + +Clone the repository. + +```shell +git clone https://github.com/eclipse-volttron/volttron-lib-sql-historian +``` + +Change to the repository directory and use poetry install to setup the environment. + +```shell +cd volttron-lib-sql-historian +poetry install +``` + ### Building Wheel To build a wheel from this project execute the following: From dffc2b84ee10f6dddb1f390bab280ce714df96a7 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Thu, 17 Nov 2022 15:03:14 -0800 Subject: [PATCH 12/24] cleanup --- config | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 config diff --git a/config b/config deleted file mode 100644 index 0966014..0000000 --- a/config +++ /dev/null @@ -1,10 +0,0 @@ -{ - # VOLTTRON config files are JSON with support for python style comments. - "setting1": 2, # Integers - "setting2": "some/random/topic2", #Strings - "setting3": true, # Booleans: remember that in JSON true and false are not capitalized. - "setting4": false, - "setting5": 5.1, # Floating point numbers. - "setting6": [1,2,3,4], #Lists - "setting7": {"setting7a": "a", "setting7b": "b"} #Objects -} From 93636f7e704a93b0ca850878e86b18857ba9da64 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Thu, 17 Nov 2022 16:40:28 -0800 Subject: [PATCH 13/24] running integration tests explicitly --- .github/workflows/run-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-test.yml b/.github/workflows/run-test.yml index 82c8a13..0fa590e 100644 --- a/.github/workflows/run-test.yml +++ b/.github/workflows/run-test.yml @@ -84,3 +84,4 @@ jobs: run: | poetry install --no-interaction poetry run pytest + poetry run pytest tests/test_sqlite_historian_integration.py From 4f01876c93ee49281057f5ab3a2980c58354f866 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Fri, 18 Nov 2022 12:30:16 -0800 Subject: [PATCH 14/24] top level package name change to historian. readme updates --- README.md | 5 +++++ pyproject.toml | 9 +++------ src/{volttron => }/historian/sql/__init__.py | 2 +- src/{volttron => }/historian/sql/basedb.py | 0 src/{volttron => }/historian/sql/historian.py | 4 ++-- src/{volttron => }/historian/sql/sqlutils.py | 4 ++-- 6 files changed, 13 insertions(+), 11 deletions(-) rename src/{volttron => }/historian/sql/__init__.py (97%) rename src/{volttron => }/historian/sql/basedb.py (100%) rename src/{volttron => }/historian/sql/historian.py (99%) rename src/{volttron => }/historian/sql/sqlutils.py (95%) diff --git a/README.md b/README.md index 3ec6b18..3991805 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,11 @@ Change to the repository directory and use poetry install to setup the environme cd volttron-lib-sql-historian poetry install ``` +To create a new relational database based historian by extending this library, subclass +[DBDriver](https://github.com/eclipse-volttron/volttron-lib-sql-historian/blob/develop/src/historian/sql/basedb.py#L79). +The subclass should be in a module historian..functs.py for it to be dynamically loaded +by the base DBDriver. Please refer to [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) as +an example ### Building Wheel diff --git a/pyproject.toml b/pyproject.toml index 5afd1ba..0e7b717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,11 @@ readme = "README.md" repository = "https://github.com/eclipse-volttron/volttron-lib-sql-historian" homepage = "https://github.com/eclipse-volttron/volttron-lib-sql-historian" keywords = [] -packages = [ { include = "volttron", from = "src" } ] +packages = [ { include = "historian", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -volttron-lib-base-historian="^0.1.1a1" +#volttron-lib-base-historian="^0.1.1a1" [tool.poetry.group.dev.dependencies] # formatting, quality, tests @@ -27,7 +27,7 @@ mypy = "^0.942" coverage = "^6.3.2" Sphinx = "^4.5.0" sphinx-rtd-theme = "^1.0.0" -#volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} +volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} [tool.yapfignore] @@ -44,9 +44,6 @@ spaces_before_comment = 4 column_limit = 99 split_before_logical_operator = true -[tool.poetry.scripts] -volttron-sql-historian = "volttron.historian.sql.historian:main" - [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/src/volttron/historian/sql/__init__.py b/src/historian/sql/__init__.py similarity index 97% rename from src/volttron/historian/sql/__init__.py rename to src/historian/sql/__init__.py index 956e251..38cea9e 100644 --- a/src/volttron/historian/sql/__init__.py +++ b/src/historian/sql/__init__.py @@ -43,7 +43,7 @@ """ from typing import List -from volttron.historian.sql.basedb import DbDriver +from historian.sql.basedb import DbDriver __all__: List[str] = [ "DbDriver", diff --git a/src/volttron/historian/sql/basedb.py b/src/historian/sql/basedb.py similarity index 100% rename from src/volttron/historian/sql/basedb.py rename to src/historian/sql/basedb.py diff --git a/src/volttron/historian/sql/historian.py b/src/historian/sql/historian.py similarity index 99% rename from src/volttron/historian/sql/historian.py rename to src/historian/sql/historian.py index 9a4608a..479d513 100644 --- a/src/volttron/historian/sql/historian.py +++ b/src/historian/sql/historian.py @@ -42,8 +42,8 @@ import threading from volttron import utils -from volttron.historian.base import BaseHistorian -from volttron.historian.sql import sqlutils +from historian.base import BaseHistorian +from historian.sql import sqlutils #from volttron.utils. import doc_inherit __version__ = "4.0.0" diff --git a/src/volttron/historian/sql/sqlutils.py b/src/historian/sql/sqlutils.py similarity index 95% rename from src/volttron/historian/sql/sqlutils.py rename to src/historian/sql/sqlutils.py index 4f7d83d..01c5107 100644 --- a/src/volttron/historian/sql/sqlutils.py +++ b/src/historian/sql/sqlutils.py @@ -39,7 +39,7 @@ import logging from volttron import utils -from volttron.historian.sql import DbDriver +from historian.sql import DbDriver utils.setup_logging() _log = logging.getLogger(__name__) @@ -47,7 +47,7 @@ def get_dbfuncts_class(database_type): mod_name = database_type + "functs" - mod_name_path = f"volttron.historian.{database_type}.{mod_name}" + mod_name_path = f"historian.{database_type}.{mod_name}" loaded_mod = __import__(mod_name_path, fromlist=[mod_name]) for _, cls in inspect.getmembers(loaded_mod): # Ensure class is not the root dbdriver From 1d79cfc39c87920976eab71e43b7d6b5787574f7 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Fri, 18 Nov 2022 14:39:36 -0800 Subject: [PATCH 15/24] referring to common document for agent development. updated pyproject.toml to depend on latest base historian version --- .github/workflows/run-test.yml | 2 +- README.md | 54 ++-------------------------------- pyproject.toml | 4 +-- 3 files changed, 6 insertions(+), 54 deletions(-) diff --git a/.github/workflows/run-test.yml b/.github/workflows/run-test.yml index 0fa590e..41eb2dc 100644 --- a/.github/workflows/run-test.yml +++ b/.github/workflows/run-test.yml @@ -84,4 +84,4 @@ jobs: run: | poetry install --no-interaction poetry run pytest - poetry run pytest tests/test_sqlite_historian_integration.py + diff --git a/README.md b/README.md index 3991805..ba3cb1a 100644 --- a/README.md +++ b/README.md @@ -2,82 +2,34 @@ [![ci](https://github.com/VOLTTRON/volttron-sql-historian/workflows/ci/badge.svg)](https://github.com/eclipse-volttron/volttron-lib-sql-historian/actions?query=workflow%3Aci) [![pypi version](https://img.shields.io/pypi/v/volttron-sql-historian.svg)](https://pypi.org/project/volttron-lib-sql-historian/) - Generic SQL Historian library that can be used to implement a historian agent with a relational database backend. This library cannot be installed as a VOLTTRON agent as is. Only a concrete database implementation package such as [sqlite-historian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) that depends on this library can be installed as a VOLTTRON agent. - ## Requirements - Python >= 3.8 ## Installation -This library can be installed using ```pip install volttron-lib-sql-historian```. However this is not necessary. Any +This library can be installed using ```pip install volttron-lib-sql-historian```. However, this is not necessary. Any historian agent that uses this library will automatically install it as part of its installation. For example, installing [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) will automatically install volttron-lib-sql-historian ## Development -Development requires poetry 1.2.2 or greater be used. -One can install it from https://python-poetry.org/docs/#installation. The VOLTTRON team prefers to have the python -environments created within the project directory. Execute this command to make that behavior the default. - -```shell -poetry config virtualenvs.in-project true -``` - -Clone the repository. +Please see the following for contributing guidelines [contributing](https://github.com/eclipse-volttron/volttron-core/blob/develop/CONTRIBUTING.md). -```shell -git clone https://github.com/eclipse-volttron/volttron-lib-sql-historian -``` +Please see the following helpful guide about [developing modular VOLTTRON agents](https://github.com/eclipse-volttron/volttron-core/blob/develop/DEVELOPING_ON_MODULAR.md) -Change to the repository directory and use poetry install to setup the environment. - -```shell -cd volttron-lib-sql-historian -poetry install -``` To create a new relational database based historian by extending this library, subclass [DBDriver](https://github.com/eclipse-volttron/volttron-lib-sql-historian/blob/develop/src/historian/sql/basedb.py#L79). The subclass should be in a module historian..functs.py for it to be dynamically loaded by the base DBDriver. Please refer to [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) as an example -### Building Wheel - -To build a wheel from this project execute the following: - -```shell -poetry build -``` - -The wheel and source distribution will be located in the ```./dist/``` directory. - -### Bumping version number of project - -To bump the version number of the project execute one of the following. - -```shell -# patch, minor, major, prepatch, preminor, premajor, prerelease - -# use patch -user@path$ poetry patch - -# output -Bumping version from 0.2.0-alpha.0 to 0.2.0 - -# use prepatch -user@path$ poetry version prepatch - -# output -Bumping version from 0.2.0 to 0.2.1-alpha.0 -``` - # Disclaimer Notice This material was prepared as an account of work sponsored by an agency of the diff --git a/pyproject.toml b/pyproject.toml index 0e7b717..a433f19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = [ { include = "historian", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -#volttron-lib-base-historian="^0.1.1a1" +volttron-lib-base-historian="^0.1.1a2" [tool.poetry.group.dev.dependencies] # formatting, quality, tests @@ -27,7 +27,7 @@ mypy = "^0.942" coverage = "^6.3.2" Sphinx = "^4.5.0" sphinx-rtd-theme = "^1.0.0" -volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} +#volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} [tool.yapfignore] From e095765afb8b9b28bada3bab56d593c3db546071 Mon Sep 17 00:00:00 2001 From: Shwetha Niddodi Date: Mon, 21 Nov 2022 11:51:50 -0800 Subject: [PATCH 16/24] Update pyproject.toml Bumped up version for volttron-lib-base-historian --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a433f19..0e45362 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = [ { include = "historian", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -volttron-lib-base-historian="^0.1.1a2" +volttron-lib-base-historian="^0.1.1a3" [tool.poetry.group.dev.dependencies] # formatting, quality, tests From 4d38436c789a088addec239666f0378312ceeee8 Mon Sep 17 00:00:00 2001 From: Shwetha Niddodi Date: Tue, 22 Nov 2022 11:48:12 -0800 Subject: [PATCH 17/24] Update pyproject.toml Bumped volttron-lib-base-historian version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0e45362..748d46d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = [ { include = "historian", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -volttron-lib-base-historian="^0.1.1a3" +volttron-lib-base-historian="^0.1.1a4" [tool.poetry.group.dev.dependencies] # formatting, quality, tests From 6cdf2ab4665a7400c38563b11ce0d2872b1436cb Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Wed, 23 Nov 2022 22:00:37 -0800 Subject: [PATCH 18/24] Updated copyright. --- docs/source/conf.py | 24 ++++++++++++++++++ src/historian/sql/__init__.py | 45 ++++++++++++---------------------- src/historian/sql/basedb.py | 45 ++++++++++++---------------------- src/historian/sql/historian.py | 45 ++++++++++++---------------------- src/historian/sql/sqlutils.py | 45 ++++++++++++---------------------- 5 files changed, 86 insertions(+), 118 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 59a00f8..bc4b17e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,3 +1,27 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Installable Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2022 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + # Configuration file for the Sphinx documentation builder. # -- Project information diff --git a/src/historian/sql/__init__.py b/src/historian/sql/__init__.py index 38cea9e..f7b8e51 100644 --- a/src/historian/sql/__init__.py +++ b/src/historian/sql/__init__.py @@ -1,40 +1,27 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2022, Battelle Memorial Institute. +# Installable Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2022 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} + """ VOLTTRON SQL Historian package. diff --git a/src/historian/sql/basedb.py b/src/historian/sql/basedb.py index 4430a4c..9418352 100644 --- a/src/historian/sql/basedb.py +++ b/src/historian/sql/basedb.py @@ -1,42 +1,27 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2022, Battelle Memorial Institute. +# Installable Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2022 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} - import contextlib import importlib import logging diff --git a/src/historian/sql/historian.py b/src/historian/sql/historian.py index 479d513..360a18d 100644 --- a/src/historian/sql/historian.py +++ b/src/historian/sql/historian.py @@ -1,42 +1,27 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2022, Battelle Memorial Institute. +# Installable Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2022 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} - import logging import sys import threading diff --git a/src/historian/sql/sqlutils.py b/src/historian/sql/sqlutils.py index 01c5107..ffe8379 100644 --- a/src/historian/sql/sqlutils.py +++ b/src/historian/sql/sqlutils.py @@ -1,40 +1,27 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2022, Battelle Memorial Institute. +# Installable Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2022 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} + import inspect import logging From bd3ca9d1e9849344ed2573c69ce74e9f7980a890 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Wed, 23 Nov 2022 22:34:11 -0800 Subject: [PATCH 19/24] Added license.md file. --- LICENSE.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..214344d --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,9 @@ +Copyright 2022, Battelle Memorial Institute. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +The patent license grant shall only be applicable to the following patent and patent application (Battelle IPID 17008-E), as assigned to the Battelle Memorial Institute, as used in conjunction with this Work: US Patent No. 9,094,385, issued 7/28/15 USPTO Patent App. No. 14/746,577, filed 6/22/15, published as US 2016-0006569. + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. From 932381b439345e3be9e5f4776043185e275dd92e Mon Sep 17 00:00:00 2001 From: Shwetha Niddodi Date: Mon, 28 Nov 2022 10:03:18 -0800 Subject: [PATCH 20/24] Adding correct LICENSE --- LICENSE | 216 +++++++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE.md | 9 --- 2 files changed, 216 insertions(+), 9 deletions(-) create mode 100644 LICENSE delete mode 100644 LICENSE.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9464da7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,216 @@ +Copyright 2022 Battelle Memorial Institute + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy +of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 214344d..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,9 +0,0 @@ -Copyright 2022, Battelle Memorial Institute. - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -The patent license grant shall only be applicable to the following patent and patent application (Battelle IPID 17008-E), as assigned to the Battelle Memorial Institute, as used in conjunction with this Work: US Patent No. 9,094,385, issued 7/28/15 USPTO Patent App. No. 14/746,577, filed 6/22/15, published as US 2016-0006569. - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. From a4c81862dce04108fa187c5b566c33ce29cad9fc Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Mon, 28 Nov 2022 15:47:55 -0800 Subject: [PATCH 21/24] Update run-test workflow; add coverage --- .../workflows/{run-test.yml => run-tests.yml} | 67 +++++---- .gitignore | 138 ++++++++++++++++++ .pre-commit-config.yaml | 50 +++++++ README.md | 21 ++- pyproject.toml | 34 ++--- 5 files changed, 251 insertions(+), 59 deletions(-) rename .github/workflows/{run-test.yml => run-tests.yml} (53%) create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/run-test.yml b/.github/workflows/run-tests.yml similarity index 53% rename from .github/workflows/run-test.yml rename to .github/workflows/run-tests.yml index 41eb2dc..444eb91 100644 --- a/.github/workflows/run-test.yml +++ b/.github/workflows/run-tests.yml @@ -3,15 +3,17 @@ name: Run Pytests on: push: paths: - - '**.py' - - 'poetry.yml' + - '**.py' + - '**.yaml' + - '**.yml' + - '**.toml' pull_request: types: - - opened - - synchronize - - reopened + - opened + - synchronize + - reopened paths: - - '**.py' + - '**.py' defaults: run: @@ -28,38 +30,38 @@ jobs: run-tests: strategy: matrix: - os: ["ubuntu-20.04", "ubuntu-22.04"] - python: ["3.8", "3.9", "3.10"] + os: [ubuntu-20.04, ubuntu-22.04] + python: ['3.8', '3.9', '3.10'] runs-on: ${{ matrix.os }} steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." #---------------------------------------------- # check-out repo and set-up python #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v3 + - name: Checkout code + uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} + - name: Set up Python ${{ matrix.python }} + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.2.2 - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.2.2 + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true # #---------------------------------------------- # # load cached venv if cache exists @@ -80,8 +82,15 @@ jobs: #---------------------------------------------- # install your root project, and run tests. #---------------------------------------------- - - name: Install library and run tests - run: | - poetry install --no-interaction - poetry run pytest + - name: Check file existence + id: check_files + uses: andstor/file-existence-action@v2 + with: + files: tests + - name: Install library and run tests + if: steps.check_files.outputs.files_exists == 'true' + run: | + poetry install --no-interaction + poetry add --group dev pytest-github-actions-annotate-failures + poetry run pytest --cov=src tests/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..00ac7c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.idea/ + +*secret +config_*.json +.DS_Store +AUTHORS +ChangeLog +poetry.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4140453 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-xml + - id: forbid-new-submodules + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-merge-conflict + - id: no-commit-to-branch # blocks main commits. To bypass do git commit --allow-empty + - id: pretty-format-json + +# For more information about mypy, see https://github.com/pre-commit/mirrors-mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910-1 + hooks: + - id: mypy + exclude: ^(docs/|example-plugin/|tests/fixtures|tests/conftest.py) + files: ^src/ + +- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.2.0 + hooks: + - id: pretty-format-toml + args: [--autofix] + - id: pretty-format-yaml + args: [--autofix] + exclude: .copier-answer.yml + +# For more information about YAPF, see https://github.com/google/yapf +- repo: https://github.com/craig8/mirrors-yapf + rev: b84f670025671a341d0afd2b06b877b195d65c0f # Use the sha / tag you want to point at + hooks: + - id: yapf + name: yapf + description: A formatter for Python files. + entry: yapf + language: python + types: [python] + +- repo: https://github.com/python-poetry/poetry + rev: 1.2.2 + hooks: + - id: poetry-check diff --git a/README.md b/README.md index ba3cb1a..94ca0fe 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ - -[![ci](https://github.com/VOLTTRON/volttron-sql-historian/workflows/ci/badge.svg)](https://github.com/eclipse-volttron/volttron-lib-sql-historian/actions?query=workflow%3Aci) [![pypi version](https://img.shields.io/pypi/v/volttron-sql-historian.svg)](https://pypi.org/project/volttron-lib-sql-historian/) +![Passing?](https://github.com/VOLTTRON/volttron-sql-historian/actions/workflows/run-tests.yml/badge.svg) -Generic SQL Historian library that can be used to implement a historian agent with a relational database backend. -This library cannot be installed as a VOLTTRON agent as is. Only a concrete database implementation package such as -[sqlite-historian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) that depends on this library can be +Generic SQL Historian library that can be used to implement a historian agent with a relational database backend. +This library cannot be installed as a VOLTTRON agent as is. Only a concrete database implementation package such as +[sqlite-historian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) that depends on this library can be installed as a VOLTTRON agent. ## Requirements @@ -13,9 +12,9 @@ installed as a VOLTTRON agent. ## Installation -This library can be installed using ```pip install volttron-lib-sql-historian```. However, this is not necessary. Any -historian agent that uses this library will automatically install it as part of its installation. For example, -installing [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) will automatically install +This library can be installed using ```pip install volttron-lib-sql-historian```. However, this is not necessary. Any +historian agent that uses this library will automatically install it as part of its installation. For example, +installing [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) will automatically install volttron-lib-sql-historian ## Development @@ -24,10 +23,10 @@ Please see the following for contributing guidelines [contributing](https://gith Please see the following helpful guide about [developing modular VOLTTRON agents](https://github.com/eclipse-volttron/volttron-core/blob/develop/DEVELOPING_ON_MODULAR.md) -To create a new relational database based historian by extending this library, subclass +To create a new relational database based historian by extending this library, subclass [DBDriver](https://github.com/eclipse-volttron/volttron-lib-sql-historian/blob/develop/src/historian/sql/basedb.py#L79). -The subclass should be in a module historian..functs.py for it to be dynamically loaded -by the base DBDriver. Please refer to [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) as +The subclass should be in a module historian..functs.py for it to be dynamically loaded +by the base DBDriver. Please refer to [SQLiteHistorian](https://github.com/eclipse-volttron/volttron-sqlitehistorian) as an example # Disclaimer Notice diff --git a/pyproject.toml b/pyproject.toml index 748d46d..812b94f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,11 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +git-changelog = ">=0.5.0" +httpx = ">=0.16.1" +jinja2-cli = ">=0.7.0" +toml = ">=0.10.2" + [tool.poetry] name = "volttron-lib-sql-historian" version = "0.1.0" @@ -12,10 +20,9 @@ packages = [ { include = "historian", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -volttron-lib-base-historian="^0.1.1a4" +volttron-lib-base-historian = "^0.1.1a4" [tool.poetry.group.dev.dependencies] -# formatting, quality, tests pytest = "^6.2.5" mock = "^4.0.3" pre-commit = "^2.17.0" @@ -25,10 +32,15 @@ isort = "^5.10.1" safety = "^1.10.3" mypy = "^0.942" coverage = "^6.3.2" +pytest-cov = "^3.0.0" Sphinx = "^4.5.0" sphinx-rtd-theme = "^1.0.0" -#volttron-lib-base-historian = {path = "../volttron-lib-base-historian", develop = true} +[tool.yapf] +based_on_style = "pep8" +spaces_before_comment = 4 +column_limit = 99 +split_before_logical_operator = true [tool.yapfignore] ignore_patterns = [ @@ -37,19 +49,3 @@ ignore_patterns = [ "dist/**", "docs/**" ] - -[tool.yapf] -based_on_style = "pep8" -spaces_before_comment = 4 -column_limit = 99 -split_before_logical_operator = true - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - -# tasks -git-changelog = ">=0.5.0" -httpx = ">=0.16.1" -jinja2-cli = ">=0.7.0" -toml = ">=0.10.2" From b38843ffeaf2f897137ca95b89774e03d22e724a Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:42:15 -0800 Subject: [PATCH 22/24] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 94ca0fe..76f8011 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![pypi version](https://img.shields.io/pypi/v/volttron-sql-historian.svg)](https://pypi.org/project/volttron-lib-sql-historian/) -![Passing?](https://github.com/VOLTTRON/volttron-sql-historian/actions/workflows/run-tests.yml/badge.svg) +[![pypi version](https://img.shields.io/pypi/v/volttron-lib-sql-historian.svg)](https://pypi.org/project/volttron-lib-sql-historian/) +![Passing?](https://github.com/VOLTTRON/volttron-lib-sql-historian/actions/workflows/run-tests.yml/badge.svg) Generic SQL Historian library that can be used to implement a historian agent with a relational database backend. This library cannot be installed as a VOLTTRON agent as is. Only a concrete database implementation package such as From a6f4919c1e5aded8b5e643837e385948dc3e76be Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Tue, 29 Nov 2022 12:41:57 -0800 Subject: [PATCH 23/24] Prepare for 10.0.0-rc --- .copier-answers.yml | 20 -------------------- pyproject.toml | 6 +++--- 2 files changed, 3 insertions(+), 23 deletions(-) delete mode 100644 .copier-answers.yml diff --git a/.copier-answers.yml b/.copier-answers.yml deleted file mode 100644 index ad4f45e..0000000 --- a/.copier-answers.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Changes here will be overwritten by Copier -_commit: v0.3.0-beta-2-g3b3bf71 -_src_path: gh:VOLTTRON/copier-poetry-volttron-agent -author_email: volttron@pnnl.gov -author_fullname: VOLTTRON -author_username: VOLTTRON -copyright_date: '2022' -copyright_holder: PNNL -copyright_holder_email: volttron@pnnl.gov -copyright_license: Apache License 2.0 -project_description: null -project_name: VOLTTRON SQL Historian -python_agent_class_name: Volttron_sql_historian -python_package_command_line_name: volttron-sql-historian -python_package_distribution_name: volttron-sql-historian -python_package_import_name: volttron.historian.sql -repository_name: volttron-sql-historian -repository_namespace: VOLTTRON -repository_provider: github.com -use_precommit: true diff --git a/pyproject.toml b/pyproject.toml index 812b94f..5a18c88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,9 +8,9 @@ toml = ">=0.10.2" [tool.poetry] name = "volttron-lib-sql-historian" -version = "0.1.0" -description = "None" -authors = ["VOLTTRON "] +version = "0.2.0-rc" +description = "A library for supporting sql based historians." +authors = ["VOLTTRON Team "] license = "Apache License 2.0" readme = "README.md" repository = "https://github.com/eclipse-volttron/volttron-lib-sql-historian" From 3fc5aa78cab2881cb89fab2a6f7afde3542a730d Mon Sep 17 00:00:00 2001 From: Shwetha Niddodi Date: Tue, 29 Nov 2022 14:03:09 -0800 Subject: [PATCH 24/24] Prepare for 10.0.0rc release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5a18c88..b7fcd4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ packages = [ { include = "historian", from = "src" } ] [tool.poetry.dependencies] python = ">=3.8,<4.0" -volttron-lib-base-historian = "^0.1.1a4" +volttron-lib-base-historian = "^0.2.0rc0" [tool.poetry.group.dev.dependencies] pytest = "^6.2.5"