From 20318b9420833143d3dd5116f2f61392ca74fae9 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 22 Dec 2023 13:11:53 -0800 Subject: [PATCH 01/50] updated readme --- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 970d955..373de58 100644 --- a/README.md +++ b/README.md @@ -1 +1,45 @@ -# volttron-log-statistics \ No newline at end of file +# volttron-log-statistics + +The Log Statistics agent periodically reads the "volttron.log" file based on the configured interval, computes the size +delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard +deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system +which may be an indication of some sort of failure or breach. + + +### Configuration + +The Log Statistics agent has 4 required configuration values: + +- `file_path`: This should be the path to the "volttron.log" file +- `analysis_interval_secs`: The interval in seconds between publishing the size delta statistic to the message bus +- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the + historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") +- `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the + historian framework ("datalogger", "record", "analysis", "devices") + +The following is an example configuration file: + +```json +{ + "file_path" : "~/volttron/volttron.log", + "analysis_interval_min" : 60, + "publish_topic" : "platform/log_statistics", + "historian_topic" : "record/log_statistics" +} +``` + + +### Periodic Publish + +The Log Statistics agent will run statistics publishes automatically based on the configured intervals. + +The following is an example of a periodic size delta publish: + +``` +Peer: pubsub +Sender: platform.logstatisticsagent1 +Bus: +Topic: platform/log_statistics +Headers: {'min_compatible_version': '3.0', 'max_compatible_version': ''} +Message: {'log_size_delta': 902, 'timestamp': '2021-01-25T22:48:16.924135Z'} +``` \ No newline at end of file From 877dec39836c562657bb60c68d05e8ae8f1a9b8f Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 22 Dec 2023 13:18:20 -0800 Subject: [PATCH 02/50] added src and agent --- src/log_statistics/agent.py | 154 ++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 src/log_statistics/agent.py diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py new file mode 100644 index 0000000..21be72b --- /dev/null +++ b/src/log_statistics/agent.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Installable Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2022 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +import datetime +import logging +import os +import sys +import statistics + +from volttron.platform.vip.agent import Agent, RPC, Core +from volttron.platform.agent import utils +from volttron.platform.agent.utils import get_aware_utc_now + +utils.setup_logging() +_log = logging.getLogger(__name__) +__version__ = '1.0' + + +def log_statistics(config_path, **kwargs): + """ + Load the LogStatisticsAgent agent configuration and returns and instance + of the agent created using that configuration. + :param config_path: Path to a configuration file. + :type config_path: str + :returns: LogStatisticsAgent agent instance + :rtype: LogStatisticsAgent agent + """ + config = utils.load_config(config_path) + return LogStatisticsAgent(config, **kwargs) + + +class LogStatisticsAgent(Agent): + """ + LogStatisticsAgent reads volttron.log file size every hour, compute the size delta from previous hour and publish + the difference with timestamp. It also publishes standard deviation every 24 hours. + :param config: Configuration dict + :type config: dict + Example configuration: + .. code-block:: python + { + "file_path" : "/home/volttron/volttron.log", + "analysis_interval_sec" : 60, + "publish_topic" : "platform/log_statistics", + "historian_topic" : "analysis/log_statistics" + } + """ + + def __init__(self, config, **kwargs): + super(LogStatisticsAgent, self).__init__(**kwargs) + self.analysis_interval_sec = config["analysis_interval_sec"] + self.file_path = config["file_path"] + self.publish_topic = config["publish_topic"] + self.historian_topic = config["historian_topic"] + self.size_delta_list = [] + self.file_start_size = None + self.prev_file_size = None + self._scheduled_event = None + + @Core.receiver('onstart') + def starting(self, sender, **kwargs): + _log.info("Starting " + self.__class__.__name__ + " agent") + self.publish_analysis() + + def publish_analysis(self): + """ + Publishes file's size increment in previous time interval (60 minutes) with timestamp. + Also publishes standard deviation of file's hourly size differences every 24 hour. + """ + if self._scheduled_event is not None: + self._scheduled_event.cancel() + + if self.prev_file_size is None: + self.prev_file_size = self.get_file_size() + _log.debug("init_file_size = {}".format(self.prev_file_size)) + else: + # read file size + curr_file_size = self.get_file_size() + + # calculate size delta + size_delta = curr_file_size - self.prev_file_size + self.prev_file_size = curr_file_size + + self.size_delta_list.append(size_delta) + + headers = {'Date': datetime.datetime.utcnow().isoformat() + 'Z'} + + publish_message = {'timestamp': datetime.datetime.utcnow().isoformat() + 'Z', + 'log_size_delta': size_delta} + historian_message = [{"log_size_delta ": size_delta}, + {"log_size_delta ": {'units': 'bytes', 'tz': 'UTC', 'type': 'float'}}] + + if len(self.size_delta_list) == 24: + standard_deviation = statistics.stdev(self.size_delta_list) + publish_message['log_std_dev'] = standard_deviation + historian_message[0]['log_std_dev'] = standard_deviation + historian_message[1]['log_std_dev'] = {'units': 'bytes', 'tz': 'UTC', 'type': 'float'} + + _log.debug('publishing message {} with header {} on historian topic {}' + .format(historian_message, headers, self.historian_topic)) + self.vip.pubsub.publish(peer="pubsub", topic=self.historian_topic, headers=headers, + message=historian_message) + + self.size_delta_list = [] + + _log.debug('publishing message {} on topic {}'.format(publish_message, self.publish_topic)) + self.vip.pubsub.publish(peer="pubsub", topic=self.publish_topic, message=publish_message) + + _log.debug('Scheduling next periodic call') + now = get_aware_utc_now() + next_update_time = now + datetime.timedelta(seconds=self.analysis_interval_sec) + + self._scheduled_event = self.core.schedule(next_update_time, self.publish_analysis) + + def get_file_size(self): + try: + return os.path.getsize(self.file_path) + except OSError as e: + _log.error(e) + + +def main(argv=sys.argv): + """ + Main method called by the platform. + """ + utils.vip_main(log_statistics, identity='platform.logstatisticsagent') + + +if __name__ == '__main__': + # Entry point for script + try: + sys.exit(main()) + except KeyboardInterrupt: + pass \ No newline at end of file From 1d865088f472265d54853c700f284a0995af0891 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 22 Dec 2023 13:37:54 -0800 Subject: [PATCH 03/50] added test and adjusted imports --- tests/test_log_statistics.py | 104 +++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/test_log_statistics.py diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py new file mode 100644 index 0000000..7f7dfd7 --- /dev/null +++ b/tests/test_log_statistics.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Installable Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2022 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +import os +import json +import pytest +from mock import MagicMock + +from volttron.client.messaging.health import STATUS_GOOD +from volttron.client.vip.agent import Agent +from volttron.client import get_ops, get_home + +test_config = { + "analysis_interval_sec": 2, + "publish_topic": "platform/log_statistics", + "historian_topic": "analysis/log_statistics" +} + +@pytest.fixture(scope="module") +def publish_agent(request, volttron_instance): + test_config = { + "file_path": os.path.join(volttron_instance.volttron_home, "volttron.log"), + "analysis_interval_sec": 2, + "publish_topic": "platform/log_statistics", + "historian_topic": "analysis/log_statistics" + } + # 1: Start a fake agent to publish to message bus + agent = volttron_instance.build_agent() + + agent.callback = MagicMock(name="callback") + agent.callback.reset_mock() + + agent.vip.pubsub.subscribe(peer='pubsub', + prefix=test_config.get("publish_topic"), + callback=agent.callback).get() + + def stop_agent(): + print("In teardown method of publish_agent") + if isinstance(agent, Agent): + agent.core.stop() + + request.addfinalizer(stop_agent) + return agent + + +def test_default_config(volttron_instance, publish_agent): + """ + Test the default configuration file included with the agent + """ + config_path = os.path.join(get_ops("LogStatisticsAgent"), "logstatisticsagent.config") + with open(config_path, "r") as config_file: + config_json = json.load(config_file) + assert isinstance(config_json, dict) + stats_uuid = volttron_instance.install_agent( + agent_dir=get_ops("LogStatisticsAgent"), + config_file=config_json, + start=True, + vip_identity="health_test") + assert publish_agent.vip.rpc.call("health_test", "health.get_status").get(timeout=10).get('status') == STATUS_GOOD + volttron_instance.remove_agent(stats_uuid) + + +def test_log_stats(volttron_instance, publish_agent): + test_config["file_path"] = volttron_instance.log_path + print(f"File path: {test_config['file_path']}") + + stats_uuid = volttron_instance.install_agent( + agent_dir=get_ops("LogStatisticsAgent"), + config_file=test_config, + start=True, + vip_identity="health_test") + + import gevent + gevent.sleep(1) + + # building another agent should populate the logs + volttron_instance.build_agent(identity="log_populate") + + gevent.sleep(1) + + assert publish_agent.callback.call_count >= 1 + + volttron_instance.remove_agent(stats_uuid) \ No newline at end of file From 8d2b96fe254a5b0eddf707c41b92ad7e4b64cbb4 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 22 Dec 2023 13:51:55 -0800 Subject: [PATCH 04/50] added pyproject.toml --- pyproject.toml | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ed677cc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,65 @@ +[build-system] +requires = ["poetry-core>=1.2.2"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +volttron-log-statistics = "log-statistics.agent:main" + +[tool.isort] +profile = "black" + +[tool.poetry] +include = ["src/log_statistics", "config"] + +name = "volttron-log-statistics" +version = "0.1.0" +description = "" +authors = ["VOLTTRON Team "] +license = "Apache License 2.0" +readme = "README.md" +repository = "https://github.com/eclipse-volttron/volttron-log-statistics" +homepage = "https://github.com/eclipse-volttron/volttron-log-statistics" +keywords = [] +packages = [ { include = "log_statistics", from = "src" } ] +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Intended Audience :: Science/Research", + "Intended Audience :: Information Technology", + "Intended Audience :: Developers", + "Intended Audience :: Other Audience", + "License :: OSI Approved :: Apache Software License" +] + +[tool.poetry.dependencies] +python = ">=3.10,<4.0" +volttron = "^10.0.4rc1" + +[tool.poetry.group.dev.dependencies] +volttron-testing = "^0.4.0rc0" +pytest = "^6.2.5" +pytest-cov = "^3.0.0" +mock = "^4.0.3" +pre-commit = "^2.17.0" +yapf = "^0.32.0" +toml = "^0.10.2" +mypy = "^0.942" +coverage = "^6.3.2" +isort = "^5.10.1" + +[tool.poetry.dev-dependencies] +Sphinx = "^4.5.0" +sphinx-rtd-theme = "^1.0.0" + +[tool.yapf] +based_on_style = "pep8" +spaces_before_comment = 4 +column_limit = 99 +split_before_logical_operator = true + +[tool.yapfignore] +ignore_patterns = [ + ".venv/**", + ".pytest_cache/**", + "dist/**", + "docs/**" +] \ No newline at end of file From 4e61fa9de4410caf484e81a585f42e3e88fee262 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 22 Dec 2023 15:14:49 -0800 Subject: [PATCH 05/50] added pyproject --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ed677cc..af3e054 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["poetry-core>=1.2.2"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -volttron-log-statistics = "log-statistics.agent:main" +volttron-log-statistics = "log_statistics.agent:main" [tool.isort] profile = "black" From 6b528e4f129a1bffbaec80bb1bf7338c9b550439 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 22 Dec 2023 15:15:15 -0800 Subject: [PATCH 06/50] temp path --- src/log_statistics/agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 21be72b..315a35a 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -28,9 +28,9 @@ import sys import statistics -from volttron.platform.vip.agent import Agent, RPC, Core -from volttron.platform.agent import utils -from volttron.platform.agent.utils import get_aware_utc_now +from volttron.client.vip.agent import Agent, RPC, Core +from volttron import utils +from volttron.utils.time import get_aware_utc_now utils.setup_logging() _log = logging.getLogger(__name__) @@ -143,7 +143,7 @@ def main(argv=sys.argv): """ Main method called by the platform. """ - utils.vip_main(log_statistics, identity='platform.logstatisticsagent') + utils.vip_main(log_statistics, identity='platform.log_statistics') if __name__ == '__main__': From d2169a944274545282f0854b91f4e3f2f1cccc4d Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 13:12:16 -0800 Subject: [PATCH 07/50] updated publish --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 373de58..9cb7996 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The following is an example configuration file: ```json { "file_path" : "~/volttron/volttron.log", - "analysis_interval_min" : 60, + "analysis_interval_sec" : 60, "publish_topic" : "platform/log_statistics", "historian_topic" : "record/log_statistics" } @@ -37,9 +37,9 @@ The following is an example of a periodic size delta publish: ``` Peer: pubsub -Sender: platform.logstatisticsagent1 +Sender: platform.log_statistics Bus: -Topic: platform/log_statistics -Headers: {'min_compatible_version': '3.0', 'max_compatible_version': ''} -Message: {'log_size_delta': 902, 'timestamp': '2021-01-25T22:48:16.924135Z'} +Topic: heartbeat/platform.log_statistics +Headers: {'TimeStamp': '2023-12-26T21:05:06.675246+00:00', 'min_compatible_version': '3.0', 'max_compatible_version': ''} +Message: {'log_size_delta': 6059, 'timestamp': '2023-12-26T21:05:10.627008Z'} ``` \ No newline at end of file From a539bd18b86bb89715677ed61184f0b919d48a37 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 13:24:00 -0800 Subject: [PATCH 08/50] fixed readme --- README.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9cb7996..537f21c 100644 --- a/README.md +++ b/README.md @@ -5,19 +5,58 @@ delta from the previous hour and publishes the difference in bytes with a timest deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. +# Prerequisites -### Configuration +* Python 3.8 -The Log Statistics agent has 4 required configuration values: +## Python -- `file_path`: This should be the path to the "volttron.log" file -- `analysis_interval_secs`: The interval in seconds between publishing the size delta statistic to the message bus -- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the - historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") -- `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the - historian framework ("datalogger", "record", "analysis", "devices") +
+To install Python 3.8, we recommend using pyenv. + +```bash +# install pyenv +git clone https://github.com/pyenv/pyenv ~/.pyenv + +# setup pyenv (you should also put these three lines in .bashrc or similar) +export PATH="${HOME}/.pyenv/bin:${PATH}" +export PYENV_ROOT="${HOME}/.pyenv" +eval "$(pyenv init -)" -The following is an example configuration file: +# install Python 3.8 +pyenv install 3.8.10 + +# make it available globally +pyenv global system 3.8.10 +``` +
+ +# Installation + +1. Create and activate a virtual environment. + +```shell +python -m venv env +source env/bin/activate +``` + +2. Install volttron and start the platform. + +```shell +pip install volttron + +# Start platform with output going to volttron.log +volttron -vv -l volttron.log & +``` +3. Create a config directory and navigate to it: + +```shell +mkdir config +cd config +``` +### Configuration + +4. Navigate to the config directory and create a file called `log_stat_config.json` and add the following JSON to it: ```json { @@ -28,6 +67,22 @@ The following is an example configuration file: } ``` +5. Install and start the log statistics agent +```bash +vctl install volttron-log-statistics --agent-config log_stat_config.json --json --vip-identity platform.log_statistics --start --force +``` + +The Log Statistics agent has 4 required configuration values: + +- `file_path`: This should be the path to the "volttron.log" file +- `analysis_interval_secs`: The interval in seconds between publishing the size delta statistic to the message bus +- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the + historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") +- `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the + historian framework ("datalogger", "record", "analysis", "devices") + + + ### Periodic Publish From 66f9dd819a382d9ee15fbe63025d0141af4cbb49 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 13:24:57 -0800 Subject: [PATCH 09/50] fixed readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 537f21c..e0b38c4 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ The Log Statistics agent has 4 required configuration values: ### Periodic Publish + The Log Statistics agent will run statistics publishes automatically based on the configured intervals. The following is an example of a periodic size delta publish: From f5c468a32aacd7d70784f853084e93830b0e9fc0 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 13:28:14 -0800 Subject: [PATCH 10/50] changed config name --- log_stat_config.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 log_stat_config.json diff --git a/log_stat_config.json b/log_stat_config.json new file mode 100644 index 0000000..2e39e4d --- /dev/null +++ b/log_stat_config.json @@ -0,0 +1,6 @@ +{ + "file_path" : "/home/riley/env/volttron_home/volttron.log", + "analysis_interval_sec" : 60, + "publish_topic" : "platform/log_statistics", + "historian_topic" : "record/log_statistics" +} \ No newline at end of file From 09c722a04680a1ebf15608f63719e3707cb41a6d Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 13:33:36 -0800 Subject: [PATCH 11/50] added notice --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e0b38c4..be40957 100644 --- a/README.md +++ b/README.md @@ -98,4 +98,22 @@ Bus: Topic: heartbeat/platform.log_statistics Headers: {'TimeStamp': '2023-12-26T21:05:06.675246+00:00', 'min_compatible_version': '3.0', 'max_compatible_version': ''} Message: {'log_size_delta': 6059, 'timestamp': '2023-12-26T21:05:10.627008Z'} -``` \ No newline at end of file +``` + +# Disclaimer Notice + +This material was prepared as an account of work sponsored by an agency of the +United States Government. Neither the United States Government nor the United +States Department of Energy, nor Battelle, nor any of their employees, nor any +jurisdiction or organization that has cooperated in the development of these +materials, makes any warranty, express or implied, or assumes any legal +liability or responsibility for the accuracy, completeness, or usefulness or any +information, apparatus, product, software, or process disclosed, or represents +that its use would not infringe privately owned rights. + +Reference herein to any specific commercial product, process, or service by +trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the United +States Government or any agency thereof, or Battelle Memorial Institute. The +views and opinions of authors expressed herein do not necessarily state or +reflect those of the United States Government or any agency thereof. \ No newline at end of file From a0a4d3e0a43feb949d8f7a6eb05268fe62a634ea Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 13:43:00 -0800 Subject: [PATCH 12/50] added license --- LICENSE | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..66b409e --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2023 Battelle Memorial Institute + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From 41b3248942db9773393e6d1b5b14fdaf2a4d88ec Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 15:38:00 -0800 Subject: [PATCH 13/50] updated import --- tests/test_log_statistics.py | 39 +++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py index 7f7dfd7..e536233 100644 --- a/tests/test_log_statistics.py +++ b/tests/test_log_statistics.py @@ -29,13 +29,20 @@ from volttron.client.messaging.health import STATUS_GOOD from volttron.client.vip.agent import Agent -from volttron.client import get_ops, get_home +from volttrontesting.fixtures.volttron_platform_fixtures import volttron_instance +#from volttron.client import get_ops, get_home test_config = { "analysis_interval_sec": 2, "publish_topic": "platform/log_statistics", "historian_topic": "analysis/log_statistics" } +default_config = { + "file_path" : "/home/riley/env/volttron_home/volttron.log", + "analysis_interval_sec" : 60, + "publish_topic" : "platform/log_statistics", + "historian_topic" : "record/log_statistics" +} @pytest.fixture(scope="module") def publish_agent(request, volttron_instance): @@ -68,16 +75,28 @@ def test_default_config(volttron_instance, publish_agent): """ Test the default configuration file included with the agent """ - config_path = os.path.join(get_ops("LogStatisticsAgent"), "logstatisticsagent.config") - with open(config_path, "r") as config_file: - config_json = json.load(config_file) - assert isinstance(config_json, dict) + + # config_path = "/home/riley/volttron-log-statistics/log_stat_config.json" + # with open(config_path, "r") as config_file: + # config_json = json.load(config_file) + # assert isinstance(config_json, dict) + stats_uuid = volttron_instance.install_agent( - agent_dir=get_ops("LogStatisticsAgent"), - config_file=config_json, + agent_dir="volttron-log-statistics", + config_file=default_config, start=True, vip_identity="health_test") - assert publish_agent.vip.rpc.call("health_test", "health.get_status").get(timeout=10).get('status') == STATUS_GOOD + + import gevent + gevent.sleep(1) + + # building another agent should populate the logs + volttron_instance.build_agent(identity="log_populate") + + gevent.sleep(1) + + assert publish_agent.callback.call_count >= 1 + volttron_instance.remove_agent(stats_uuid) @@ -86,7 +105,7 @@ def test_log_stats(volttron_instance, publish_agent): print(f"File path: {test_config['file_path']}") stats_uuid = volttron_instance.install_agent( - agent_dir=get_ops("LogStatisticsAgent"), + agent_dir="volttron-log-statistics", config_file=test_config, start=True, vip_identity="health_test") @@ -101,4 +120,4 @@ def test_log_stats(volttron_instance, publish_agent): assert publish_agent.callback.call_count >= 1 - volttron_instance.remove_agent(stats_uuid) \ No newline at end of file + volttron_instance.remove_agent(stats_uuid) From dd18f185b0f1ca3fd9ad586e398d0bebe7a0436a Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 15:39:23 -0800 Subject: [PATCH 14/50] updated config file name --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af3e054..3c0b9be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ volttron-log-statistics = "log_statistics.agent:main" profile = "black" [tool.poetry] -include = ["src/log_statistics", "config"] +include = ["src/log_statistics", "log_stat_config.json"] name = "volttron-log-statistics" version = "0.1.0" @@ -62,4 +62,4 @@ ignore_patterns = [ ".pytest_cache/**", "dist/**", "docs/**" -] \ No newline at end of file +] From b0f0f8e66809a1e4037f453731f6179240bcd466 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Dec 2023 17:15:38 -0800 Subject: [PATCH 15/50] fixed test --- tests/test_log_statistics.py | 50 ++++++++++++------------------------ 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py index e536233..02ee424 100644 --- a/tests/test_log_statistics.py +++ b/tests/test_log_statistics.py @@ -37,12 +37,6 @@ "publish_topic": "platform/log_statistics", "historian_topic": "analysis/log_statistics" } -default_config = { - "file_path" : "/home/riley/env/volttron_home/volttron.log", - "analysis_interval_sec" : 60, - "publish_topic" : "platform/log_statistics", - "historian_topic" : "record/log_statistics" -} @pytest.fixture(scope="module") def publish_agent(request, volttron_instance): @@ -70,34 +64,22 @@ def stop_agent(): request.addfinalizer(stop_agent) return agent - -def test_default_config(volttron_instance, publish_agent): - """ - Test the default configuration file included with the agent - """ - - # config_path = "/home/riley/volttron-log-statistics/log_stat_config.json" - # with open(config_path, "r") as config_file: - # config_json = json.load(config_file) - # assert isinstance(config_json, dict) - - stats_uuid = volttron_instance.install_agent( - agent_dir="volttron-log-statistics", - config_file=default_config, - start=True, - vip_identity="health_test") - - import gevent - gevent.sleep(1) - - # building another agent should populate the logs - volttron_instance.build_agent(identity="log_populate") - - gevent.sleep(1) - - assert publish_agent.callback.call_count >= 1 - - volttron_instance.remove_agent(stats_uuid) +# TODO get path to defualt config with python package. +# def test_default_config(volttron_instance, publish_agent): +# """ +# Test the default configuration file included with the agent +# """ +# # config_path = os.path.join(get_ops("LogStatisticsAgent"), "logstatisticsagent.config") +# # with open(config_path, "r") as config_file: +# # config_json = json.load(config_file) +# assert isinstance(test_config, dict) +# stats_uuid = volttron_instance.install_agent( +# agent_dir="volttron-log-statistics", +# config_file=test_config, +# start=True, +# vip_identity="health_test") +# assert publish_agent.vip.rpc.call("health_test", "health.get_status").get(timeout=10).get('status') == STATUS_GOOD +# volttron_instance.remove_agent(stats_uuid) def test_log_stats(volttron_instance, publish_agent): From 3e06677db119b16099375511f17aea4a4c38d07f Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 27 Dec 2023 08:10:31 -0800 Subject: [PATCH 16/50] added pre commit --- .pre-commit-config.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4b6ddac --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-xml + - id: forbid-new-submodules + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-merge-conflict + - id: no-commit-to-branch # blocks main commits. To bypass do git commit --allow-empty + - id: pretty-format-json From f00a6548d0e4d4c576fe8326ef9d139e17875ccb Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 2 Jan 2024 12:52:46 -0800 Subject: [PATCH 17/50] removed default config test --- tests/test_log_statistics.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py index 02ee424..7334d44 100644 --- a/tests/test_log_statistics.py +++ b/tests/test_log_statistics.py @@ -64,24 +64,6 @@ def stop_agent(): request.addfinalizer(stop_agent) return agent -# TODO get path to defualt config with python package. -# def test_default_config(volttron_instance, publish_agent): -# """ -# Test the default configuration file included with the agent -# """ -# # config_path = os.path.join(get_ops("LogStatisticsAgent"), "logstatisticsagent.config") -# # with open(config_path, "r") as config_file: -# # config_json = json.load(config_file) -# assert isinstance(test_config, dict) -# stats_uuid = volttron_instance.install_agent( -# agent_dir="volttron-log-statistics", -# config_file=test_config, -# start=True, -# vip_identity="health_test") -# assert publish_agent.vip.rpc.call("health_test", "health.get_status").get(timeout=10).get('status') == STATUS_GOOD -# volttron_instance.remove_agent(stats_uuid) - - def test_log_stats(volttron_instance, publish_agent): test_config["file_path"] = volttron_instance.log_path print(f"File path: {test_config['file_path']}") From 2b8b6f5ebb84459db4af5d77ddc2649595b3b286 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 5 Jan 2024 12:37:32 -0800 Subject: [PATCH 18/50] adjusted output, general flow --- README.md | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index be40957..844c067 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,16 @@ cd config ``` ### Configuration -4. Navigate to the config directory and create a file called `log_stat_config.json` and add the following JSON to it: +4. Navigate to the config directory and create a file called `log_stat_config.json` and adjust the 4 required configuration values to fit your needs. An example is shown below. + +The Log Statistics agent has 4 required configuration values: + +- `file_path`: This should be the path to the "volttron.log" file +- `analysis_interval_secs`: The interval in seconds between publishing the size delta statistic to the message bus +- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the + historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") +- `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the + historian framework ("datalogger", "record", "analysis", "devices") ```json { @@ -72,18 +81,6 @@ cd config vctl install volttron-log-statistics --agent-config log_stat_config.json --json --vip-identity platform.log_statistics --start --force ``` -The Log Statistics agent has 4 required configuration values: - -- `file_path`: This should be the path to the "volttron.log" file -- `analysis_interval_secs`: The interval in seconds between publishing the size delta statistic to the message bus -- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the - historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") -- `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the - historian framework ("datalogger", "record", "analysis", "devices") - - - - ### Periodic Publish @@ -92,12 +89,7 @@ The Log Statistics agent will run statistics publishes automatically based on th The following is an example of a periodic size delta publish: ``` -Peer: pubsub -Sender: platform.log_statistics -Bus: -Topic: heartbeat/platform.log_statistics -Headers: {'TimeStamp': '2023-12-26T21:05:06.675246+00:00', 'min_compatible_version': '3.0', 'max_compatible_version': ''} -Message: {'log_size_delta': 6059, 'timestamp': '2023-12-26T21:05:10.627008Z'} +2024-01-05 12:30:21,952 (volttron-log-statistics-0.1.0 6584) log_statistics.agent(126) DEBUG: publishing message {'timestamp': '2024-01-05T20:30:21.952503Z', 'log_size_delta': 338} on topic platform/log_statistics ``` # Disclaimer Notice From 81d7beee57f64644248e1544a92fd9a08ae5f727 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 5 Jan 2024 12:48:41 -0800 Subject: [PATCH 19/50] removed import --- tests/test_log_statistics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py index 7334d44..1d5546d 100644 --- a/tests/test_log_statistics.py +++ b/tests/test_log_statistics.py @@ -30,7 +30,6 @@ from volttron.client.messaging.health import STATUS_GOOD from volttron.client.vip.agent import Agent from volttrontesting.fixtures.volttron_platform_fixtures import volttron_instance -#from volttron.client import get_ops, get_home test_config = { "analysis_interval_sec": 2, From db8d491b0283e1163fcb252a14c4b4dc7201776c Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 5 Jan 2024 13:02:34 -0800 Subject: [PATCH 20/50] added type hints --- src/log_statistics/agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 315a35a..e4a583c 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -37,7 +37,7 @@ __version__ = '1.0' -def log_statistics(config_path, **kwargs): +def log_statistics(config_path: str, **kwargs): """ Load the LogStatisticsAgent agent configuration and returns and instance of the agent created using that configuration. @@ -66,7 +66,7 @@ class LogStatisticsAgent(Agent): } """ - def __init__(self, config, **kwargs): + def __init__(self, config: dict, **kwargs): super(LogStatisticsAgent, self).__init__(**kwargs) self.analysis_interval_sec = config["analysis_interval_sec"] self.file_path = config["file_path"] From 5492b70e883ddf2025a9517ca349e933ff331e0f Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 19 Jan 2024 11:13:28 -0800 Subject: [PATCH 21/50] added .gitignore --- .gitignore | 168 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6dfb9ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,168 @@ +.coverage* +pip-wheel-metadata/ +site/ +.venv/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# pycharm project settings +.idea/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ From c098043589afa0b11919c425058bfcd99fd4b837 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 19 Jan 2024 11:16:16 -0800 Subject: [PATCH 22/50] cleaned with linter --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 844c067..53e1e4b 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,15 @@ # volttron-log-statistics -The Log Statistics agent periodically reads the "volttron.log" file based on the configured interval, computes the size -delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard -deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system -which may be an indication of some sort of failure or breach. +The Log Statistics agent periodically reads the "volttron.log" file based on the configured interval, computes the size delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. -# Prerequisites +## Prerequisites * Python 3.8 ## Python
+ To install Python 3.8, we recommend using pyenv. ```bash @@ -29,9 +27,10 @@ pyenv install 3.8.10 # make it available globally pyenv global system 3.8.10 ``` +
-# Installation +## Installation 1. Create and activate a virtual environment. @@ -48,12 +47,14 @@ pip install volttron # Start platform with output going to volttron.log volttron -vv -l volttron.log & ``` + 3. Create a config directory and navigate to it: ```shell mkdir config cd config ``` + ### Configuration 4. Navigate to the config directory and create a file called `log_stat_config.json` and adjust the 4 required configuration values to fit your needs. An example is shown below. @@ -77,22 +78,22 @@ The Log Statistics agent has 4 required configuration values: ``` 5. Install and start the log statistics agent + ```bash vctl install volttron-log-statistics --agent-config log_stat_config.json --json --vip-identity platform.log_statistics --start --force ``` ### Periodic Publish - The Log Statistics agent will run statistics publishes automatically based on the configured intervals. The following is an example of a periodic size delta publish: -``` +```log 2024-01-05 12:30:21,952 (volttron-log-statistics-0.1.0 6584) log_statistics.agent(126) DEBUG: publishing message {'timestamp': '2024-01-05T20:30:21.952503Z', 'log_size_delta': 338} on topic platform/log_statistics ``` -# Disclaimer Notice +## Disclaimer Notice This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor the United From a5bc120ade524ab86fdb2a19b5355be125a41e9a Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 19 Jan 2024 11:18:51 -0800 Subject: [PATCH 23/50] updated python version --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 53e1e4b..f64073c 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ The Log Statistics agent periodically reads the "volttron.log" file based on the ## Prerequisites -* Python 3.8 +* Python 3.10 ## Python
- -To install Python 3.8, we recommend using pyenv. +To install Python 3.10, we recommend using pyenv. ```bash # install pyenv @@ -21,11 +20,11 @@ export PATH="${HOME}/.pyenv/bin:${PATH}" export PYENV_ROOT="${HOME}/.pyenv" eval "$(pyenv init -)" -# install Python 3.8 -pyenv install 3.8.10 +# install Python 3.10 +pyenv install 3.10 # make it available globally -pyenv global system 3.8.10 +pyenv global system 3.10 ```
From 632b085c8f2174e13f4dc557de12fde5c22e8cf3 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 19 Jan 2024 12:27:31 -0800 Subject: [PATCH 24/50] added isort --- .pre-commit-config.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b6ddac..c54107e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.5.0 hooks: - id: check-yaml - id: check-json @@ -12,3 +12,8 @@ repos: - id: check-merge-conflict - id: no-commit-to-branch # blocks main commits. To bypass do git commit --allow-empty - id: pretty-format-json +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) From 1603410f0d4666a5e798432e1418faa6c2a8a6f0 Mon Sep 17 00:00:00 2001 From: riley206 Date: Fri, 19 Jan 2024 12:30:18 -0800 Subject: [PATCH 25/50] ran with isort --- README.md | 6 +++--- log_stat_config.json | 6 +++--- src/log_statistics/agent.py | 6 +++--- tests/test_log_statistics.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f64073c..6b62e54 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,9 @@ The Log Statistics agent has 4 required configuration values: - `file_path`: This should be the path to the "volttron.log" file - `analysis_interval_secs`: The interval in seconds between publishing the size delta statistic to the message bus -- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the +- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") -- `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the +- `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the historian framework ("datalogger", "record", "analysis", "devices") ```json @@ -108,4 +108,4 @@ trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or any agency thereof, or Battelle Memorial Institute. The views and opinions of authors expressed herein do not necessarily state or -reflect those of the United States Government or any agency thereof. \ No newline at end of file +reflect those of the United States Government or any agency thereof. diff --git a/log_stat_config.json b/log_stat_config.json index 2e39e4d..176d45e 100644 --- a/log_stat_config.json +++ b/log_stat_config.json @@ -1,6 +1,6 @@ { - "file_path" : "/home/riley/env/volttron_home/volttron.log", + "file_path" : "~/volttron/volttron.log", "analysis_interval_sec" : 60, "publish_topic" : "platform/log_statistics", - "historian_topic" : "record/log_statistics" -} \ No newline at end of file + "historian_topic" : "analysis/log_statistics" +} diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index e4a583c..e7189fa 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -25,11 +25,11 @@ import datetime import logging import os -import sys import statistics +import sys -from volttron.client.vip.agent import Agent, RPC, Core from volttron import utils +from volttron.client.vip.agent import RPC, Agent, Core from volttron.utils.time import get_aware_utc_now utils.setup_logging() @@ -151,4 +151,4 @@ def main(argv=sys.argv): try: sys.exit(main()) except KeyboardInterrupt: - pass \ No newline at end of file + pass diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py index 1d5546d..0eff6d0 100644 --- a/tests/test_log_statistics.py +++ b/tests/test_log_statistics.py @@ -22,11 +22,11 @@ # ===----------------------------------------------------------------------=== # }}} -import os import json +import os + import pytest from mock import MagicMock - from volttron.client.messaging.health import STATUS_GOOD from volttron.client.vip.agent import Agent from volttrontesting.fixtures.volttron_platform_fixtures import volttron_instance From 5f20c868fec60d68efbe520b970a37cd45bcf7de Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 5 Feb 2024 12:45:11 -0800 Subject: [PATCH 26/50] updatd pre-commit --- .pre-commit-config.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c54107e..46473cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,8 +12,13 @@ repos: - id: check-merge-conflict - id: no-commit-to-branch # blocks main commits. To bypass do git commit --allow-empty - id: pretty-format-json -- repo: https://github.com/pycqa/isort - rev: 5.13.2 + +- repo: https://github.com/google/yapf + rev: v0.40.2 # Use the sha / tag you want to point at hooks: - - id: isort - name: isort (python) + - id: yapf + name: yapf + description: "A formatter for Python files." + entry: yapf + language: python + types: [ python ] From 0af3112775f4e6c74a4442ce03541d56ec3fe8ab Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 5 Feb 2024 12:48:43 -0800 Subject: [PATCH 27/50] updated column limit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c0b9be..dfb3e41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ sphinx-rtd-theme = "^1.0.0" [tool.yapf] based_on_style = "pep8" spaces_before_comment = 4 -column_limit = 99 +column_limit = 120 split_before_logical_operator = true [tool.yapfignore] From 0ead1d3d7ba194221833d9b927cf0d3c755a3cae Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 5 Feb 2024 12:49:44 -0800 Subject: [PATCH 28/50] simplified readme --- README.md | 70 +++++++++++++++---------------------------------------- 1 file changed, 19 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 6b62e54..01f7779 100644 --- a/README.md +++ b/README.md @@ -2,61 +2,27 @@ The Log Statistics agent periodically reads the "volttron.log" file based on the configured interval, computes the size delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. -## Prerequisites +## Requires -* Python 3.10 - -## Python - -
-To install Python 3.10, we recommend using pyenv. - -```bash -# install pyenv -git clone https://github.com/pyenv/pyenv ~/.pyenv - -# setup pyenv (you should also put these three lines in .bashrc or similar) -export PATH="${HOME}/.pyenv/bin:${PATH}" -export PYENV_ROOT="${HOME}/.pyenv" -eval "$(pyenv init -)" - -# install Python 3.10 -pyenv install 3.10 - -# make it available globally -pyenv global system 3.10 -``` - -
+* python >= 3.10 +* volttron >= 10.0 ## Installation -1. Create and activate a virtual environment. - -```shell -python -m venv env -source env/bin/activate -``` +Before installing, VOLTTRON should be installed and running. Its virtual environment should be active. +Information on how to install of the VOLTTRON platform can be found +[here](https://github.com/eclipse-volttron/volttron-core). -2. Install volttron and start the platform. - -```shell -pip install volttron - -# Start platform with output going to volttron.log -volttron -vv -l volttron.log & -``` - -3. Create a config directory and navigate to it: +Create a directory called `config` and use the change directory command to enter it. ```shell mkdir config cd config ``` -### Configuration +After entering the config directory, create a file called `log_stat_config.json`. Use the below configuration section to populate your new file. -4. Navigate to the config directory and create a file called `log_stat_config.json` and adjust the 4 required configuration values to fit your needs. An example is shown below. +### Configuration The Log Statistics agent has 4 required configuration values: @@ -76,21 +42,23 @@ The Log Statistics agent has 4 required configuration values: } ``` -5. Install and start the log statistics agent +Install and start the log statistics agent ```bash -vctl install volttron-log-statistics --agent-config log_stat_config.json --json --vip-identity platform.log_statistics --start --force +vctl install volttron-log-statistics --agent-config log_stat_config.json --vip-identity platform.log_statistics --start --force ``` -### Periodic Publish +View the status of the installed agent. -The Log Statistics agent will run statistics publishes automatically based on the configured intervals. +```shell +vctl status +``` -The following is an example of a periodic size delta publish: +## Development -```log -2024-01-05 12:30:21,952 (volttron-log-statistics-0.1.0 6584) log_statistics.agent(126) DEBUG: publishing message {'timestamp': '2024-01-05T20:30:21.952503Z', 'log_size_delta': 338} on topic platform/log_statistics -``` +Please see the following for contributing guidelines [contributing](https://github.com/eclipse-volttron/volttron-core/blob/develop/CONTRIBUTING.md). + +Please see the following helpful guide about [developing modular VOLTTRON agents](https://github.com/eclipse-volttron/volttron-core/blob/develop/DEVELOPING_ON_MODULAR.md) ## Disclaimer Notice From cf42b52ac86466c1f5c944c219377347e19613f9 Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 5 Feb 2024 12:51:05 -0800 Subject: [PATCH 29/50] updated 2024 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 66b409e..b7346b6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2023 Battelle Memorial Institute +Copyright 2024 Battelle Memorial Institute Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From c177386f01168680223bc75172edf8bb7795966b Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 5 Feb 2024 12:53:14 -0800 Subject: [PATCH 30/50] updated to 2024 --- src/log_statistics/agent.py | 2 +- tests/test_log_statistics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index e7189fa..80c4485 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -5,7 +5,7 @@ # # ===----------------------------------------------------------------------=== # -# Copyright 2022 Battelle Memorial Institute +# Copyright 2024 Battelle Memorial Institute # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py index 0eff6d0..b3a0e42 100644 --- a/tests/test_log_statistics.py +++ b/tests/test_log_statistics.py @@ -5,7 +5,7 @@ # # ===----------------------------------------------------------------------=== # -# Copyright 2022 Battelle Memorial Institute +# Copyright 2024 Battelle Memorial Institute # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy From e52e2b23b9b253b06c61bad417d157c04704bb5f Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 11 Mar 2024 09:51:51 -0700 Subject: [PATCH 31/50] updated readme --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 01f7779..f4bb6d0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # volttron-log-statistics +![Passing?](https://github.com/eclipse-volttron/volttron-log-statistics/actions/workflows/run-tests.yml/badge.svg) +[![pypi version](https://img.shields.io/pypi/v/volttron-log-statistics.svg)](https://pypi.org/project/volttron-log-statistics/) + The Log Statistics agent periodically reads the "volttron.log" file based on the configured interval, computes the size delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. ## Requires @@ -24,10 +27,10 @@ After entering the config directory, create a file called `log_stat_config.json` ### Configuration -The Log Statistics agent has 4 required configuration values: +The Log Statistics agent has 4 configuration parameters, all of which are required: - `file_path`: This should be the path to the "volttron.log" file -- `analysis_interval_secs`: The interval in seconds between publishing the size delta statistic to the message bus +- `analysis_interval_secs`: The interval in seconds between publishes of the size delta statistic to the message bus - `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") - `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the @@ -45,7 +48,7 @@ The Log Statistics agent has 4 required configuration values: Install and start the log statistics agent ```bash -vctl install volttron-log-statistics --agent-config log_stat_config.json --vip-identity platform.log_statistics --start --force +vctl install volttron-log-statistics --agent-config log_stat_config.json --vip-identity platform.log_statistics --start ``` View the status of the installed agent. From 5cb5353b88d448ddf2f9b15c8f2c76371d7a6999 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 12 Mar 2024 15:18:57 -0700 Subject: [PATCH 32/50] updated config parameter descriptions --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f4bb6d0..e5e3b71 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # volttron-log-statistics -![Passing?](https://github.com/eclipse-volttron/volttron-log-statistics/actions/workflows/run-tests.yml/badge.svg) -[![pypi version](https://img.shields.io/pypi/v/volttron-log-statistics.svg)](https://pypi.org/project/volttron-log-statistics/) +![Passing?](https://github.com/eclipse-volttron/volttron-log-statistics/actions/workflows/run-tests.yml/badge.svg) +[![pypi version](https://img.shields.io/pypi/v/volttron-log-statistics.svg)](https://pypi.org/project/volttron-log-statistics/) -The Log Statistics agent periodically reads the "volttron.log" file based on the configured interval, computes the size delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. +The Log Statistics agent periodically reads ".log" files based on the configured interval, computes the size delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. ## Requires @@ -23,22 +23,22 @@ mkdir config cd config ``` -After entering the config directory, create a file called `log_stat_config.json`. Use the below configuration section to populate your new file. +After entering the config directory, create a file called `log_stat_config.json`. Use the below configuration section to populate your new file. ### Configuration The Log Statistics agent has 4 configuration parameters, all of which are required: -- `file_path`: This should be the path to the "volttron.log" file -- `analysis_interval_secs`: The interval in seconds between publishes of the size delta statistic to the message bus -- `publish_topic`: Can be used to specify a topic to publish log statistics to which does not get captured by the - historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices") +- `file_path`: The file path to the log file. If left as `null`, defaults to `'volttron.log'` located within your VOLTTRON_HOME environment variable. +- `analysis_interval_secs`: The interval in seconds between publishes of the size delta statistic to the message bus. If left as `null`, defaults to 60 seconds. +- `publish_topic`: Used to specify a topic to publish log statistics to which does not get captured by the + historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices"). If left as `null`, defaults to `"platform/log_statistics"`. - `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the - historian framework ("datalogger", "record", "analysis", "devices") + historian framework ("datalogger", "record", "analysis", "devices"). If left as `null`, defaults to `record/log_statistics`. ```json { - "file_path" : "~/volttron/volttron.log", + "file_path" : null, "analysis_interval_sec" : 60, "publish_topic" : "platform/log_statistics", "historian_topic" : "record/log_statistics" From cebae03b58a839c7995cd2ded3ef201087652ac3 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 12 Mar 2024 15:27:22 -0700 Subject: [PATCH 33/50] uses default config if not present --- log_stat_config.json | 8 ++++---- src/log_statistics/agent.py | 39 ++++++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/log_stat_config.json b/log_stat_config.json index 176d45e..5a365cf 100644 --- a/log_stat_config.json +++ b/log_stat_config.json @@ -1,6 +1,6 @@ { - "file_path" : "~/volttron/volttron.log", - "analysis_interval_sec" : 60, - "publish_topic" : "platform/log_statistics", - "historian_topic" : "analysis/log_statistics" + "analysis_interval_sec": 60, + "file_path": null, + "historian_topic": "analysis/log_statistics", + "publish_topic": "platform/log_statistics" } diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 80c4485..7019ddf 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -68,6 +68,23 @@ class LogStatisticsAgent(Agent): def __init__(self, config: dict, **kwargs): super(LogStatisticsAgent, self).__init__(**kwargs) + + # Getting volttron home env variable or using user/name as backup + volttron_home = os.getenv('VOLTTRON_HOME', os.path.expanduser("~") + '/volttron_home/') + + self.default_config = { + "file_path": f"{volttron_home}/volttron.log", + "analysis_interval_sec": 60, + "publish_topic": "platform/log_statistics", + "historian_topic": "analysis/log_statistics" + } + + # Update config with defaults for any keys not present in config + for key, value in self.default_config.items(): + if config.get(key) is None: + _log.info(f"Using default value for {key}: {value}") + config[key] = value + self.analysis_interval_sec = config["analysis_interval_sec"] self.file_path = config["file_path"] self.publish_topic = config["publish_topic"] @@ -105,10 +122,16 @@ def publish_analysis(self): headers = {'Date': datetime.datetime.utcnow().isoformat() + 'Z'} - publish_message = {'timestamp': datetime.datetime.utcnow().isoformat() + 'Z', - 'log_size_delta': size_delta} - historian_message = [{"log_size_delta ": size_delta}, - {"log_size_delta ": {'units': 'bytes', 'tz': 'UTC', 'type': 'float'}}] + publish_message = {'timestamp': datetime.datetime.utcnow().isoformat() + 'Z', 'log_size_delta': size_delta} + historian_message = [{ + "log_size_delta ": size_delta + }, { + "log_size_delta ": { + 'units': 'bytes', + 'tz': 'UTC', + 'type': 'float' + } + }] if len(self.size_delta_list) == 24: standard_deviation = statistics.stdev(self.size_delta_list) @@ -116,9 +139,11 @@ def publish_analysis(self): historian_message[0]['log_std_dev'] = standard_deviation historian_message[1]['log_std_dev'] = {'units': 'bytes', 'tz': 'UTC', 'type': 'float'} - _log.debug('publishing message {} with header {} on historian topic {}' - .format(historian_message, headers, self.historian_topic)) - self.vip.pubsub.publish(peer="pubsub", topic=self.historian_topic, headers=headers, + _log.debug('publishing message {} with header {} on historian topic {}'.format( + historian_message, headers, self.historian_topic)) + self.vip.pubsub.publish(peer="pubsub", + topic=self.historian_topic, + headers=headers, message=historian_message) self.size_delta_list = [] From 223b21ef7460b4cc6aac385eac3213037a8a4047 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 12 Mar 2024 15:32:36 -0700 Subject: [PATCH 34/50] fixed json --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e5e3b71..42cc44d 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,10 @@ The Log Statistics agent has 4 configuration parameters, all of which are requir ```json { - "file_path" : null, - "analysis_interval_sec" : 60, - "publish_topic" : "platform/log_statistics", - "historian_topic" : "record/log_statistics" + "analysis_interval_sec": 60, + "file_path": null, + "historian_topic": "analysis/log_statistics", + "publish_topic": "platform/log_statistics" } ``` From 6332a192636ca3f0e4e667522263d6a32a24f2ec Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 12 Mar 2024 15:52:35 -0700 Subject: [PATCH 35/50] standard deviation based off of time --- src/log_statistics/agent.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 7019ddf..cf75c50 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -69,6 +69,8 @@ class LogStatisticsAgent(Agent): def __init__(self, config: dict, **kwargs): super(LogStatisticsAgent, self).__init__(**kwargs) + self.last_std_dev_time = get_aware_utc_now() + # Getting volttron home env variable or using user/name as backup volttron_home = os.getenv('VOLTTRON_HOME', os.path.expanduser("~") + '/volttron_home/') @@ -133,7 +135,9 @@ def publish_analysis(self): } }] - if len(self.size_delta_list) == 24: + now = get_aware_utc_now() + hours_since_last_std_dev = (now - self.last_std_dev_time).total_seconds() / 3600 + if hours_since_last_std_dev >= 24: standard_deviation = statistics.stdev(self.size_delta_list) publish_message['log_std_dev'] = standard_deviation historian_message[0]['log_std_dev'] = standard_deviation @@ -145,6 +149,8 @@ def publish_analysis(self): topic=self.historian_topic, headers=headers, message=historian_message) + # Reset time + self.last_std_dev_time = now self.size_delta_list = [] From 394c042d7a8694068bfadd0f5ebcb37dfe32d290 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 12 Mar 2024 17:04:15 -0700 Subject: [PATCH 36/50] calculate mean and better logging --- src/log_statistics/agent.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index cf75c50..e8b3f9a 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -137,11 +137,33 @@ def publish_analysis(self): now = get_aware_utc_now() hours_since_last_std_dev = (now - self.last_std_dev_time).total_seconds() / 3600 + if hours_since_last_std_dev >= 24: - standard_deviation = statistics.stdev(self.size_delta_list) - publish_message['log_std_dev'] = standard_deviation - historian_message[0]['log_std_dev'] = standard_deviation - historian_message[1]['log_std_dev'] = {'units': 'bytes', 'tz': 'UTC', 'type': 'float'} + if self.size_delta_list: # make sure it has something in it + if len(self.size_delta_list) >= 2: # make sure it has more than two items + mean = statistics.mean(self.size_delta_list) + standard_deviation = statistics.stdev(self.size_delta_list) + + publish_message['log_mean'] = mean + print(f"Calculated mean: {mean}") + publish_message['log_std_dev'] = standard_deviation + + historian_message[0]['log_mean'] = mean + historian_message[0]['log_std_dev'] = standard_deviation + + historian_message[1]['log_mean'] = {'units': 'bytes', 'tz': 'UTC', 'type': 'float'} + historian_message[1]['log_std_dev'] = {'units': 'bytes', 'tz': 'UTC', 'type': 'float'} + + else: + _log.info("Not enough data points to calculate standard deviation") + + else: + _log.info("Not enough data points to calculate mean and standard deviation") + + # Reset time + self.last_std_dev_time = now + + self.size_delta_list = [] _log.debug('publishing message {} with header {} on historian topic {}'.format( historian_message, headers, self.historian_topic)) @@ -149,10 +171,6 @@ def publish_analysis(self): topic=self.historian_topic, headers=headers, message=historian_message) - # Reset time - self.last_std_dev_time = now - - self.size_delta_list = [] _log.debug('publishing message {} on topic {}'.format(publish_message, self.publish_topic)) self.vip.pubsub.publish(peer="pubsub", topic=self.publish_topic, message=publish_message) From bb9376476e474242dabb08da9dc5aa6b55398336 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 13 Mar 2024 09:19:26 -0700 Subject: [PATCH 37/50] better topic explanation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42cc44d..7257dc0 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![Passing?](https://github.com/eclipse-volttron/volttron-log-statistics/actions/workflows/run-tests.yml/badge.svg) [![pypi version](https://img.shields.io/pypi/v/volttron-log-statistics.svg)](https://pypi.org/project/volttron-log-statistics/) -The Log Statistics agent periodically reads ".log" files based on the configured interval, computes the size delta from the previous hour and publishes the difference in bytes with a timestamp. It also publishes standard deviation of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. +The Log Statistics agent periodically reads ".log" files based on the configured interval, computes the size delta from the previous interval and publishes the difference in bytes with a timestamp. It also publishes the standard deviation and mean of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. ## Requires From 76e0cd546b77fec2a333afc504d76118cb9b6c21 Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 18 Mar 2024 13:26:24 -0700 Subject: [PATCH 38/50] getting volttron home uasing clientContext --- src/log_statistics/agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index e8b3f9a..cdf6f2a 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -30,6 +30,7 @@ from volttron import utils from volttron.client.vip.agent import RPC, Agent, Core +from volttron.utils import ClientContext as cc from volttron.utils.time import get_aware_utc_now utils.setup_logging() @@ -71,8 +72,7 @@ def __init__(self, config: dict, **kwargs): self.last_std_dev_time = get_aware_utc_now() - # Getting volttron home env variable or using user/name as backup - volttron_home = os.getenv('VOLTTRON_HOME', os.path.expanduser("~") + '/volttron_home/') + volttron_home = cc.get_volttron_home() self.default_config = { "file_path": f"{volttron_home}/volttron.log", From f814e53715d3bf77bbcc0d278d32390aa6f51825 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 26 Mar 2024 12:58:59 -0700 Subject: [PATCH 39/50] using config store --- README.md | 17 +++++++++++------ log_stat_config.json | 2 +- src/log_statistics/agent.py | 23 ++++++++++++++++------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7257dc0..087581b 100644 --- a/README.md +++ b/README.md @@ -29,26 +29,31 @@ After entering the config directory, create a file called `log_stat_config.json` The Log Statistics agent has 4 configuration parameters, all of which are required: -- `file_path`: The file path to the log file. If left as `null`, defaults to `'volttron.log'` located within your VOLTTRON_HOME environment variable. -- `analysis_interval_secs`: The interval in seconds between publishes of the size delta statistic to the message bus. If left as `null`, defaults to 60 seconds. +- `file_path`: The file path to the log file. If no config provided, defaults to `'volttron.log'` located within your VOLTTRON_HOME environment variable. +- `analysis_interval_secs`: The interval in seconds between publishes of the size delta statistic to the message bus. If no config provided, defaults to 60 seconds. - `publish_topic`: Used to specify a topic to publish log statistics to which does not get captured by the - historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices"). If left as `null`, defaults to `"platform/log_statistics"`. + historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices"). If no config provided, defaults to `"platform/log_statistics"`. - `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the - historian framework ("datalogger", "record", "analysis", "devices"). If left as `null`, defaults to `record/log_statistics`. + historian framework ("datalogger", "record", "analysis", "devices"). If no config provided, defaults to `record/log_statistics`. +Here is an example configuration file named `log_stat_config.json`. ```json { "analysis_interval_sec": 60, - "file_path": null, + "file_path": "path/to/.log/", "historian_topic": "analysis/log_statistics", "publish_topic": "platform/log_statistics" } ``` +Store the configuration in the config store +```bash +vctl config store platform.log_statistics config /path/to/config +``` Install and start the log statistics agent ```bash -vctl install volttron-log-statistics --agent-config log_stat_config.json --vip-identity platform.log_statistics --start +vctl install volttron-log-statistics --vip-identity platform.log_statistics --start ``` View the status of the installed agent. diff --git a/log_stat_config.json b/log_stat_config.json index 5a365cf..108e296 100644 --- a/log_stat_config.json +++ b/log_stat_config.json @@ -1,6 +1,6 @@ { "analysis_interval_sec": 60, - "file_path": null, + "file_path": "/path/to/volttron.log", "historian_topic": "analysis/log_statistics", "publish_topic": "platform/log_statistics" } diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index cdf6f2a..8b05e46 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -29,7 +29,7 @@ import sys from volttron import utils -from volttron.client.vip.agent import RPC, Agent, Core +from volttron.client.vip.agent import Agent, Core from volttron.utils import ClientContext as cc from volttron.utils.time import get_aware_utc_now @@ -67,7 +67,7 @@ class LogStatisticsAgent(Agent): } """ - def __init__(self, config: dict, **kwargs): + def __init__(self, config_path=None, **kwargs): super(LogStatisticsAgent, self).__init__(**kwargs) self.last_std_dev_time = get_aware_utc_now() @@ -80,13 +80,16 @@ def __init__(self, config: dict, **kwargs): "publish_topic": "platform/log_statistics", "historian_topic": "analysis/log_statistics" } + self.vip.config.set_default("config", self.default_config) + self.vip.config.subscribe(self.configure_main, actions=["NEW", "UPDATE"], pattern="config") - # Update config with defaults for any keys not present in config - for key, value in self.default_config.items(): - if config.get(key) is None: - _log.info(f"Using default value for {key}: {value}") - config[key] = value + def configure_main(self, config_name, action, contents): + config = self.default_config.copy() + config.update(contents) + if action == "NEW" or "UPDATE": + self.reset_parameters(config) + def reset_parameters(self, config=None): self.analysis_interval_sec = config["analysis_interval_sec"] self.file_path = config["file_path"] self.publish_topic = config["publish_topic"] @@ -96,6 +99,8 @@ def __init__(self, config: dict, **kwargs): self.prev_file_size = None self._scheduled_event = None + self.publish_analysis() + @Core.receiver('onstart') def starting(self, sender, **kwargs): _log.info("Starting " + self.__class__.__name__ + " agent") @@ -106,6 +111,10 @@ def publish_analysis(self): Publishes file's size increment in previous time interval (60 minutes) with timestamp. Also publishes standard deviation of file's hourly size differences every 24 hour. """ + if not hasattr(self, '_scheduled_event'): + # The settings haven't been initialized, so skip the rest of the method + return + if self._scheduled_event is not None: self._scheduled_event.cancel() From ce647dc9b39c548b95c7682a578cbd05a58d50af Mon Sep 17 00:00:00 2001 From: riley206 Date: Thu, 28 Mar 2024 10:49:06 -0700 Subject: [PATCH 40/50] added config store support --- src/log_statistics/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 8b05e46..724c918 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -35,7 +35,7 @@ utils.setup_logging() _log = logging.getLogger(__name__) -__version__ = '1.0' +__version__ = '1.1' def log_statistics(config_path: str, **kwargs): From 9724ab1de1c1416cdfbe44578262815e57555839 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 2 Apr 2024 10:15:28 -0700 Subject: [PATCH 41/50] can use --agent-config --- src/log_statistics/agent.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 724c918..265e410 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -38,19 +38,6 @@ __version__ = '1.1' -def log_statistics(config_path: str, **kwargs): - """ - Load the LogStatisticsAgent agent configuration and returns and instance - of the agent created using that configuration. - :param config_path: Path to a configuration file. - :type config_path: str - :returns: LogStatisticsAgent agent instance - :rtype: LogStatisticsAgent agent - """ - config = utils.load_config(config_path) - return LogStatisticsAgent(config, **kwargs) - - class LogStatisticsAgent(Agent): """ LogStatisticsAgent reads volttron.log file size every hour, compute the size delta from previous hour and publish @@ -80,6 +67,9 @@ def __init__(self, config_path=None, **kwargs): "publish_topic": "platform/log_statistics", "historian_topic": "analysis/log_statistics" } + + if config_path: + self.default_config.update(utils.load_config(config_path)) self.vip.config.set_default("config", self.default_config) self.vip.config.subscribe(self.configure_main, actions=["NEW", "UPDATE"], pattern="config") @@ -201,7 +191,7 @@ def main(argv=sys.argv): """ Main method called by the platform. """ - utils.vip_main(log_statistics, identity='platform.log_statistics') + utils.vip_main(LogStatisticsAgent, identity='platform.log_statistics') if __name__ == '__main__': From a5921ff4b8d6ff0f7ed894eb16f0a60095e55c73 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 1 May 2024 15:21:05 -0700 Subject: [PATCH 42/50] Ability to change size units. B, KB, MB, GB --- log_stat_config.json | 7 ++++--- src/log_statistics/agent.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/log_stat_config.json b/log_stat_config.json index 108e296..5035a58 100644 --- a/log_stat_config.json +++ b/log_stat_config.json @@ -1,6 +1,7 @@ { - "analysis_interval_sec": 60, - "file_path": "/path/to/volttron.log", + "analysis_interval_sec": 20, + "file_path": "PORTS/volttron-log-statistics/volttron_home_new/volttron.log", "historian_topic": "analysis/log_statistics", - "publish_topic": "platform/log_statistics" + "publish_topic": "platform/log_statistics", + "unit": "mb" } diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 265e410..7ad9055 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -65,7 +65,8 @@ def __init__(self, config_path=None, **kwargs): "file_path": f"{volttron_home}/volttron.log", "analysis_interval_sec": 60, "publish_topic": "platform/log_statistics", - "historian_topic": "analysis/log_statistics" + "historian_topic": "analysis/log_statistics", + "unit": "bytes" } if config_path: @@ -84,6 +85,7 @@ def reset_parameters(self, config=None): self.file_path = config["file_path"] self.publish_topic = config["publish_topic"] self.historian_topic = config["historian_topic"] + self.unit = config["unit"] self.size_delta_list = [] self.file_start_size = None self.prev_file_size = None @@ -110,13 +112,15 @@ def publish_analysis(self): if self.prev_file_size is None: self.prev_file_size = self.get_file_size() - _log.debug("init_file_size = {}".format(self.prev_file_size)) + _log.debug(f"init_file_size = {self.convert_bytes(self.prev_file_size, self.unit)} {self.unit}") else: # read file size curr_file_size = self.get_file_size() # calculate size delta size_delta = curr_file_size - self.prev_file_size + size_delta = self.convert_bytes(size_delta, self.unit) + self.prev_file_size = curr_file_size self.size_delta_list.append(size_delta) @@ -128,7 +132,7 @@ def publish_analysis(self): "log_size_delta ": size_delta }, { "log_size_delta ": { - 'units': 'bytes', + 'units': f'{self.unit}', 'tz': 'UTC', 'type': 'float' } @@ -150,8 +154,8 @@ def publish_analysis(self): historian_message[0]['log_mean'] = mean historian_message[0]['log_std_dev'] = standard_deviation - historian_message[1]['log_mean'] = {'units': 'bytes', 'tz': 'UTC', 'type': 'float'} - historian_message[1]['log_std_dev'] = {'units': 'bytes', 'tz': 'UTC', 'type': 'float'} + historian_message[1]['log_mean'] = {'units': f'{self.unit}', 'tz': 'UTC', 'type': 'float'} + historian_message[1]['log_std_dev'] = {'units': f'{self.unit}', 'tz': 'UTC', 'type': 'float'} else: _log.info("Not enough data points to calculate standard deviation") @@ -164,14 +168,15 @@ def publish_analysis(self): self.size_delta_list = [] - _log.debug('publishing message {} with header {} on historian topic {}'.format( - historian_message, headers, self.historian_topic)) + _log.debug(f'publishing message {historian_message}' + f' with header {headers}' + f' on historian topic {self.historian_topic}') self.vip.pubsub.publish(peer="pubsub", topic=self.historian_topic, headers=headers, message=historian_message) - _log.debug('publishing message {} on topic {}'.format(publish_message, self.publish_topic)) + _log.debug(f'publishing message {publish_message} {self.unit} on topic {self.publish_topic}') self.vip.pubsub.publish(peer="pubsub", topic=self.publish_topic, message=publish_message) _log.debug('Scheduling next periodic call') @@ -185,6 +190,18 @@ def get_file_size(self): return os.path.getsize(self.file_path) except OSError as e: _log.error(e) + def convert_bytes(self, size, unit): + """ + Converts size from bytes to the specified unit + """ + unit = unit.lower() + if unit == 'kb': + return size / 1024 + elif unit == 'mb': + return size / 1024**2 + elif unit == 'gb': + return size / 1024**3 + return size def main(argv=sys.argv): From 9cc092b0d7495b89a9924cff7664b190a4ab6cb9 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 1 May 2024 15:24:36 -0700 Subject: [PATCH 43/50] docs reflect ability to change size units. B, KB, MB, GB --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 087581b..a5acbcc 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,11 @@ The Log Statistics agent has 4 configuration parameters, all of which are requir historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices"). If no config provided, defaults to `"platform/log_statistics"`. - `historian_topic`: Can be used to specify a topic to publish log statistics to which gets captured by the historian framework ("datalogger", "record", "analysis", "devices"). If no config provided, defaults to `record/log_statistics`. +- `unit`: Can be used to specify units. Defaults to `bytes`. + - "bytes" + - "kb" + - "mb" + - "gb" Here is an example configuration file named `log_stat_config.json`. ```json @@ -42,7 +47,8 @@ Here is an example configuration file named `log_stat_config.json`. "analysis_interval_sec": 60, "file_path": "path/to/.log/", "historian_topic": "analysis/log_statistics", - "publish_topic": "platform/log_statistics" + "publish_topic": "platform/log_statistics", + "unit": "bytes" } ``` Store the configuration in the config store From 7d160f1470fcf2c0f6d1cb9da022138246a540ad Mon Sep 17 00:00:00 2001 From: riley206 Date: Mon, 20 May 2024 10:40:48 -0700 Subject: [PATCH 44/50] removed onstart --- src/log_statistics/agent.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 7ad9055..24db9b5 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -79,6 +79,7 @@ def configure_main(self, config_name, action, contents): config.update(contents) if action == "NEW" or "UPDATE": self.reset_parameters(config) + _log.info("Starting " + self.__class__.__name__ + " agent") def reset_parameters(self, config=None): self.analysis_interval_sec = config["analysis_interval_sec"] @@ -93,11 +94,6 @@ def reset_parameters(self, config=None): self.publish_analysis() - @Core.receiver('onstart') - def starting(self, sender, **kwargs): - _log.info("Starting " + self.__class__.__name__ + " agent") - self.publish_analysis() - def publish_analysis(self): """ Publishes file's size increment in previous time interval (60 minutes) with timestamp. From 169baca109e30d126e49623ce051061e63f71705 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 22 May 2024 15:16:39 -0700 Subject: [PATCH 45/50] Fixed file paths --- README.md | 2 +- log_stat_config.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a5acbcc..150c974 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Here is an example configuration file named `log_stat_config.json`. ```json { "analysis_interval_sec": 60, - "file_path": "path/to/.log/", + "file_path": "path/to/.log", "historian_topic": "analysis/log_statistics", "publish_topic": "platform/log_statistics", "unit": "bytes" diff --git a/log_stat_config.json b/log_stat_config.json index 5035a58..aaefa2a 100644 --- a/log_stat_config.json +++ b/log_stat_config.json @@ -1,7 +1,7 @@ { - "analysis_interval_sec": 20, - "file_path": "PORTS/volttron-log-statistics/volttron_home_new/volttron.log", + "analysis_interval_sec": 60, + "file_path": "path/to/.log", "historian_topic": "analysis/log_statistics", "publish_topic": "platform/log_statistics", - "unit": "mb" -} + "unit": "bytes" +} \ No newline at end of file From 26d39e6f5315371f17c314ada0abab270e4c7760 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 22 May 2024 16:29:28 -0700 Subject: [PATCH 46/50] using volttrons get utc now --- src/log_statistics/agent.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 24db9b5..5d09ee7 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -30,8 +30,7 @@ from volttron import utils from volttron.client.vip.agent import Agent, Core -from volttron.utils import ClientContext as cc -from volttron.utils.time import get_aware_utc_now +from volttron.utils import ClientContext as cc, get_aware_utc_now utils.setup_logging() _log = logging.getLogger(__name__) From 7c1a9377483b5c0400d29a247809acded784203d Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 22 May 2024 16:30:24 -0700 Subject: [PATCH 47/50] using volttrons get utc now --- src/log_statistics/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log_statistics/agent.py b/src/log_statistics/agent.py index 5d09ee7..4b76cfb 100644 --- a/src/log_statistics/agent.py +++ b/src/log_statistics/agent.py @@ -187,7 +187,7 @@ def get_file_size(self): _log.error(e) def convert_bytes(self, size, unit): """ - Converts size from bytes to the specified unit + Converts size from bytes to the specified unit. """ unit = unit.lower() if unit == 'kb': From 1316f0acc55baebc7d5825f14199f01375674632 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 31 Jul 2024 09:53:35 -0700 Subject: [PATCH 48/50] fixed file path --- tests/test_log_statistics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_log_statistics.py b/tests/test_log_statistics.py index b3a0e42..1df551e 100644 --- a/tests/test_log_statistics.py +++ b/tests/test_log_statistics.py @@ -24,6 +24,7 @@ import json import os +from pathlib import Path import pytest from mock import MagicMock @@ -67,8 +68,9 @@ def test_log_stats(volttron_instance, publish_agent): test_config["file_path"] = volttron_instance.log_path print(f"File path: {test_config['file_path']}") + agent_path = Path(__file__).parents[1] stats_uuid = volttron_instance.install_agent( - agent_dir="volttron-log-statistics", + agent_dir=agent_path, config_file=test_config, start=True, vip_identity="health_test") From 042015339aef3fe2c0d06adb6f096c3122d978d7 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 31 Jul 2024 09:58:28 -0700 Subject: [PATCH 49/50] added docs --- docs/source/conf.py | 59 ++++++++++++++++++++++++++++++ docs/source/index.rst | 84 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..bcae01e --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Installable Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2022 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +# Configuration file for the Sphinx documentation builder. + +# -- Project information + +project = 'VOLTTRON Log Statistics Agent' +copyright = '2024, Pacific Northwest National Lab' +author = 'Pacific Northwest National Lab' + +release = '0.1' +version = '0.1.0' + +# -- General configuration + +extensions = [ + 'sphinx.ext.duration', + 'sphinx.ext.doctest', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', +] + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), +} +intersphinx_disabled_domains = ['std'] + +templates_path = ['_templates'] + +# -- Options for HTML output + +html_theme = 'sphinx_rtd_theme' + +# -- Options for EPUB output +# epub_show_urls = 'footnote' diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..e3d3dd3 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,84 @@ +volttron-log-statistics +======================= + +.. image:: https://github.com/eclipse-volttron/volttron-log-statistics/actions/workflows/run-tests.yml/badge.svg + :alt: Passing? +.. image:: https://img.shields.io/pypi/v/volttron-log-statistics.svg + :target: https://pypi.org/project/volttron-log-statistics/ + +The Log Statistics agent periodically reads ".log" files based on the configured interval, computes the size delta from the previous interval and publishes the difference in bytes with a timestamp. It also publishes the standard deviation and mean of the size delta every 24 hours. This agent can be useful for detecting unexpected changes to the system which may be an indication of some sort of failure or breach. + +Requires +-------- + +* python >= 3.10 +* volttron >= 10.0 + +Installation +------------ + +Before installing, VOLTTRON should be installed and running. Its virtual environment should be active. Information on how to install the VOLTTRON platform can be found `here `_. + +Create a directory called ``config`` and use the change directory command to enter it. + +.. code-block:: shell + + mkdir config + cd config + +After entering the config directory, create a file called ``log_stat_config.json``. Use the below configuration section to populate your new file. + +Configuration +~~~~~~~~~~~~~ + +The Log Statistics agent has 4 configuration parameters, all of which are required: + +- ``file_path``: The file path to the log file. If no config provided, defaults to ``'volttron.log'`` located within your VOLTTRON_HOME environment variable. +- ``analysis_interval_secs``: The interval in seconds between publishes of the size delta statistic to the message bus. If no config provided, defaults to 60 seconds. +- ``publish_topic``: Used to specify a topic to publish log statistics to which does not get captured by the historian framework (topics not prefixed by any of: "datalogger", "record", "analysis", "devices"). If no config provided, defaults to ``"platform/log_statistics"``. +- ``historian_topic``: Can be used to specify a topic to publish log statistics to which gets captured by the historian framework ("datalogger", "record", "analysis", "devices"). If no config provided, defaults to ``record/log_statistics``. +- ``unit``: Can be used to specify units. Defaults to ``bytes``. Possible values are "bytes", "kb", "mb", "gb". + +Here is an example configuration file named ``log_stat_config.json``. + +.. code-block:: json + + { + "analysis_interval_sec": 60, + "file_path": "path/to/.log", + "historian_topic": "analysis/log_statistics", + "publish_topic": "platform/log_statistics", + "unit": "bytes" + } + +Store the configuration in the config store: + +.. code-block:: bash + + vctl config store platform.log_statistics config /path/to/config + +Install and start the log statistics agent: + +.. code-block:: bash + + vctl install volttron-log-statistics --vip-identity platform.log_statistics --start + +View the status of the installed agent: + +.. code-block:: shell + + vctl status + +Development +----------- + +Please see the following for contributing guidelines `contributing `_. + +Please see the following helpful guide about `developing modular VOLTTRON agents `_. + +Disclaimer Notice +----------------- + +This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor the United States Department of Energy, nor Battelle, nor any of their employees, nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe privately owned rights. + +Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or any agency thereof, or Battelle Memorial Institute. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. From 5b676d34408d2c04837ab96db004909eef6f0a98 Mon Sep 17 00:00:00 2001 From: riley206 Date: Wed, 31 Jul 2024 12:00:33 -0700 Subject: [PATCH 50/50] updated docs --- docs/Makefile | 20 ++++++++++++++++++++ pyproject.toml | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 docs/Makefile diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/pyproject.toml b/pyproject.toml index dfb3e41..98aa38a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ coverage = "^6.3.2" isort = "^5.10.1" [tool.poetry.dev-dependencies] -Sphinx = "^4.5.0" -sphinx-rtd-theme = "^1.0.0" +Sphinx = "^6.0.0" +sphinx-rtd-theme = "^1.2.0" [tool.yapf] based_on_style = "pep8"