diff --git a/COPYRIGHT b/COPYRIGHT index 7b6f6aa..c49d6f6 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -3,12 +3,16 @@ Copyright 2023, Battelle Memorial Institute. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE diff --git a/README.md b/README.md index f1d6571..79c2341 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,173 @@ -# GridAPPS-D Toolbox Topology Processor +# GridAPPS-D Topology Processor -The Topology Processor is a lightweight service based on the LinkNet(TM) open-source data structure for mapping CIM ConnectivityNodes and Terminals developed by IncSys Corp. LinkNet(TM) is a trademark of Incremental Systems Corporation and is used with permission. +![GitHub Tag](https://img.shields.io/github/v/tag/GRIDAPPSD/topology-processor) +![GitHub Release Date](https://img.shields.io/github/release-date-pre/GRIDAPPSD/topology-processor) +![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/GRIDAPPSD/topology-processor/deploy-dev-release.yml) +![Libraries.io dependency status for GitHub repo](https://img.shields.io/librariesio/github/GRIDAPPSD/topology-processor) + + + +![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/GRIDAPPSD/topology-processor) +![GitHub Issues or Pull Requests](https://img.shields.io/github/issues-pr/GRIDAPPSD/topology-processor) +![GitHub commit activity](https://img.shields.io/github/commit-activity/t/GRIDAPPSD/topology-processor) + +![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/GRIDAPPSD/topology-processor/total?label=git%20downloads) +![GitHub License](https://img.shields.io/github/license/GRIDAPPSD/topology-processor) +![https://doi.org/10.1109/access.2022.3221132](https://img.shields.io/badge/doi-10.1109/access.2022.3221132-blue) + +This repo contains the GridAPPS-D services for transmission and distribution topology. The core algorithms are currently being migrated to https://github.com/PNNL-CIM-Tools/CIM-Graph-Topology-Processor and rebuilt using CIMantic Graphs labeled property graphs instead of the linked list data structures used in this repo. + +The original topology processor services have been moved into the `archive` directory and based on the LinkNet(TM) open-source data structure for mapping CIM ConnectivityNodes and Terminals developed by IncSys Corp. LinkNet(TM) is a trademark of Incremental Systems Corporation and is used with permission. + +## Switch-Delimited Topology Areas for Distributed Apps and Context Manager + +### Service Call + +The topology service uses a new topic and keyword. `mRID` can be that of a `cim:Feeder`, `cim:FeederArea`, or `cim:DistributionArea`. + +```python +topic = "goss.gridappsd.request.data.cimtopology" + +message = { + "requestType": "GET_DISTRIBUTED_AREAS", + "mRID": "FEEDER-1234-ABCD-MRID", + "resultFormat": "JSON" +} + +message = gapps.get_response(topic, message, timeout=30) +``` + +### Service Response + +The new topology processor response will be formatted as JSON-LD, with `mRID` replaced with `@id` and `@type`: + +```json +{ + "DistributionArea": { + "@id": "uuid-string", + "@type": "DistributionArea", + "Substations": [ + { + "@id": "uuid-string", + "@type": "Substation", + "NormalEnergizedFeeder": [ + { + "@id": "uuid-string", + "@type": "Feeder", + "FeederArea": { + "@id": "uuid-string", + "@type": "FeederArea", + "BoundaryTerminals": [ + { + "@id": "uuid-string", + "@type": "Terminal" + } + ], + "AddressableEquipment": [ + { + "@id": "uuid-string", + "@type": "Breaker" + }, + ], + "UnaddressableEquipment": [ + { + "@id": "uuid-string", + "@type": "PowerTransformer" + }, + { + "@id": "uuid-string", + "@type": "EnergySource" + } + ], + "Measurements": [ + { + "@id": "uuid-string", + "@type": "Analog" + }, + { + "@id": "uuid-string", + "@type": "Discrete" + } + ], + "SwitchAreas": [ + { + "@id": "uuid-string", + "@type": "SwitchArea", + "FeederArea": { + "@id": "uuid-string", + "@type": "FeederArea" + }, + "BoundaryTerminals": [ + { + "@id": "uuid-string", + "@type": "Terminal" + } + ], + "AddressableEquipment": [ + { + "@id": "uuid-string", + "@type": "LinearShuntCompensator" + } + ], + "UnaddressableEquipment": [ + { + "@id": "uuid-string", + "@type": "ACLineSegment" + } + "Measurements": [ + { + "@id": "uuid-string", + "@type": "Analog" + }, + { + "@id": "uuid-string", + "@type": "Discrete" + } + ], + "SecondaryAreas": [ + { + "@id": "uuid-string", + "@type": "SecondaryArea", + "SwitchArea": { + "@id": "uuid-string", + "@type": "SwitchArea" + }, + "BoundaryTerminals": [ + { + "@id": "9d06670e-f8ad-46a1-9854-bba7adaf1cf0", + "@type": "Terminal" + } + ], + "AddressableEquipment": [ + { + "@id": "uuid-string", + "@type": "PowerElectronicsConnection" + } + ], + "UnaddressableEquipment": [ + { + "@id": "uuid-string", + "@type": "EnergyConsumer" + } + ], + "Measurements": [ + { + "@id": "uuid-string", + "@type": "Analog" + } + ] + } + ] + } + ] + } + } + ] + } + ] + } +} +``` ## Real-time Topology Processor Service diff --git a/topo_service_tester.py b/archive/topo_service_tester.py similarity index 100% rename from topo_service_tester.py rename to archive/topo_service_tester.py diff --git a/topologyprocessor.py b/archive/topologyprocessor.py similarity index 100% rename from topologyprocessor.py rename to archive/topologyprocessor.py diff --git a/topologyservice.py b/archive/topologyservice.py similarity index 100% rename from topologyservice.py rename to archive/topologyservice.py diff --git a/toposervicedemo.ipynb b/archive/toposervicedemo.ipynb similarity index 100% rename from toposervicedemo.ipynb rename to archive/toposervicedemo.ipynb diff --git a/pyproject.toml b/pyproject.toml index 94b8d1e..685ffec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,13 +22,15 @@ repository = "https://github.com/GRIDAPPSD/topology-processor" documentation = "https://github.com/GRIDAPPSD/topology-processor" [tool.poetry.dependencies] -python = "^3.8" -gridappsd-python = {version = "^2023.5.1a9", allow-prereleases = true} +python = "^3.10" +gridappsd-python = {version = "^2024.6.0", allow-prereleases = true} +#cim-topology = { git = "https://github.com/PNNL-CIM-Tools/CIM-Graph-Topology-Processor", branch = "cimgraph-refactor"} +cim-graph = "^0.1.6a0" + [tool.poetry.group.dev.dependencies] mock = "^5.0.2" pytest = "^7.3.1" -yapf = "^0.33.0" pre-commit = "^3.3.2" [build-system] diff --git a/topo_background_config.py b/topo_background_config.py new file mode 100644 index 0000000..844e7d8 --- /dev/null +++ b/topo_background_config.py @@ -0,0 +1,14 @@ +{ + "id":"gridappsd-topology-background-service", + "description": "Topology Processor Background Service", + "creator":"PNNL", + "inputs":[], + "outputs":[], + "static_args":[], + "execution_path":"/gridappsd/services/gridappsd-topology-processor/topo_background_service.py", + "type":"PYTHON", + "launch_on_startup": true, + "prereqs":[], + "multiple_instances":false, + "environmentVariables":[] +} \ No newline at end of file diff --git a/topo_background_service.py b/topo_background_service.py new file mode 100644 index 0000000..85e841e --- /dev/null +++ b/topo_background_service.py @@ -0,0 +1,105 @@ +import os +import json +import time +import logging +from gridappsd import GridAPPSD +from gridappsd.topics import service_input_topic, service_output_topic +from cimgraph.databases import ConnectionParameters, BlazegraphConnection + +from topology_processor.utils import DistributedTopologyMessage +import cimgraph.data_profile.cimhub_2023 as cim + +class TopologyProcessor(GridAPPSD): + + def __init__(self): + os.environ['GRIDAPPSD_APPLICATION_ID'] = 'topology-background-service' + os.environ['GRIDAPPSD_APPLICATION_STATUS'] = 'STARTED' + os.environ['GRIDAPPSD_USER'] = 'app_user' + os.environ['GRIDAPPSD_PASSWORD'] = '1234App' + gapps = GridAPPSD() + assert gapps.connected + self.gapps = gapps + self.log = self.gapps.get_logger() + params = ConnectionParameters(url = "http://localhost:8889/bigdata/namespace/kb/sparql", cim_profile='cimhub_2023', iec61970_301=8) + self.blazegraph = BlazegraphConnection(params) + + + self.log.info('Topology Background Service Started') + + + # GridAPPS-D service + def on_message(self, headers, message): + model_mrid = message['mRID'] + reply_to = headers['reply-to'] + + + if message['requestType'] == 'GET_DISTRIBUTED_AREAS': + + self.log.info(f'Building Distributed Areas for {model_mrid}') + + topo_message = DistributedTopologyMessage() + container = self.blazegraph.get_object(mrid=model_mrid) + + if isinstance(container, cim.Feeder): + topo_message.get_context_from_feeder(container, self.blazegraph) + + elif isinstance(container, cim.FeederArea): + topo_message.get_context_from_feeder_area(container, self.blazegraph) + + elif isinstance(container, cim.DistributionArea): + topo_message.get_context_from_distribution_area(container, self.blazegraph) + + return_message = json.dumps(topo_message.message, indent=4) + del topo_message + self.gapps.send(reply_to, return_message) + + elif message['requestType'] == 'GET_BASE_TOPOLOGY': + Topology = self.get_base_topology(model_mrid) + return_message = { + "response": "not yet supported" + # 'modelID': model_mrid, + # 'feeders': Topology.Feeders, + # 'islands': Topology.Islands, + # 'connectivity': Topology.ConnNodeDict, + # 'equipment': Topology.EquipDict + } + self.gapps.send(reply_to, message) + + elif message['requestType'] == 'GET_SNAPSHOT_TOPOLOGY': + # [Topology, timestamp] = self.get_snapshot_topology(model_mrid, message['simulationID'], message['timestamp']) + message = { + "response": "not yet supported" + # 'modelID': model_mrid, + # 'feeders': Topology.Feeders, + # 'islands': Topology.Islands, + # 'timestamp': timestamp + } + self.gapps.send(reply_to, message) + else: + message = "No valid requestType specified" + self.gapps.send(reply_to, message) + + def get_switch_areas(self, model_mrid): + self.log.info('Building switch areas for ' + str(model_mrid)) + # DistTopo = DistributedTopology(self.gapps, model_mrid) + # message = DistTopo.create_switch_areas(model_mrid) + # return message + + + + + + +def _main(): + topic = "goss.gridappsd.request.data.cimtopology" + os.environ['GRIDAPPSD_USER'] = 'app_user' + os.environ['GRIDAPPSD_PASSWORD'] = '1234App' + gapps = GridAPPSD() + assert gapps.connected + topology = TopologyProcessor() + gapps.subscribe(topic, topology) + while True: + time.sleep(0.1) + +if __name__ == "__main__": + _main() \ No newline at end of file diff --git a/topology_processor/utils/__init__.py b/topology_processor/utils/__init__.py new file mode 100644 index 0000000..e374ef2 --- /dev/null +++ b/topology_processor/utils/__init__.py @@ -0,0 +1,2 @@ +from topology_processor.utils.addressable import * +from topology_processor.utils.distributed_topo_message import DistributedTopologyMessage diff --git a/topology_processor/utils/addressable.py b/topology_processor/utils/addressable.py new file mode 100644 index 0000000..9f79502 --- /dev/null +++ b/topology_processor/utils/addressable.py @@ -0,0 +1,67 @@ +import json +from cimgraph.models import DistributedArea +from uuid import UUID +# import cimgraph.data_profile.cimhub_2023 as cim + +jsonld = dict["@id":str(UUID),"@type":str(type)] + +def identify_addressable(network: DistributedArea, + equipment_list: list[jsonld] = None) -> list[jsonld]: + + if equipment_list is None: + equipment_list = [] + + cim = network.cim + addressable_classes = [cim.Switch, cim.Breaker, cim.Disconnector, + cim.Recloser, cim.LoadBreakSwitch, cim.Sectionaliser, + cim.PowerElectronicsConnection, cim.PowerElectronicsUnit, + cim.PhotovoltaicUnit, cim.BatteryUnit, cim.PowerElectronicsWindUnit, + cim.RatioTapChanger, # Not technically equipment, but included for GridAPPSD + cim.ShuntCompensator, cim.LinearShuntCompensator, + cim.AsynchronousMachine, cim.SynchronousMachine] + + for class_type in addressable_classes: + for equipment in network.graph.get(class_type, {}).values(): + # obj_json = {"@id":equipment.uri(), "@type":equipment.__class__.__name__} + equipment_list.append(json.loads(equipment.__repr__())) + return equipment_list + +def identify_unaddressable(network: DistributedArea, + equipment_list: list[jsonld] = None) -> list[jsonld]: + + if equipment_list is None: + equipment_list = [] + + cim = network.cim + unaddressable_classes = [cim.ACLineSegment, cim.Fuse, + cim.PowerTransformer, cim.TransformerTank, + cim.EnergySource, cim.EnergyConsumer] + + for class_type in unaddressable_classes: + for equipment in network.graph.get(class_type, {}).values(): + # obj_json = {"@id":equipment.uri(), "@type":equipment.__class__.__name__} + equipment_list.append(json.loads(equipment.__repr__())) + return equipment_list + +def identify_measurements(network: DistributedArea, + meas_list: list[jsonld] = None) -> list[jsonld]: + + if meas_list is None: + meas_list = [] + + cim = network.cim + meas_classes = [cim.Measurement, cim.Analog, cim.Discrete] + + for class_type in meas_classes: + for measurement in network.graph.get(class_type, {}).values(): + meas_list.append(json.loads(measurement.__repr__())) + return meas_list + +def identify_boundaries(network: DistributedArea) -> list[jsonld]: + + terminals = [] + for terminal in network.container.BoundaryTerminals: + terminals.append(json.loads(terminal.__repr__())) + + return terminals + diff --git a/topology_processor/utils/distributed_topo_message.py b/topology_processor/utils/distributed_topo_message.py new file mode 100644 index 0000000..e8b54f7 --- /dev/null +++ b/topology_processor/utils/distributed_topo_message.py @@ -0,0 +1,170 @@ +import json +import logging + +from cimgraph.databases import ConnectionInterface +from cimgraph.models import FeederModel, DistributedArea +import cimgraph.data_profile.cimhub_2023 as cim +import topology_processor.utils as utils + +_log = logging.getLogger(__name__) + +class DistributedTopologyMessage(): + + def __init__(self) -> None: + self.message = {} + self.message['DistributionArea'] = {} + + def get_context_from_feeder(self, feeder: cim.Feeder, connection: ConnectionInterface) -> None: + + cim = connection.cim + if not isinstance(feeder, cim.Feeder): + raise TypeError('feeder argument should be a cim.Feeder object') + + feeder_model = FeederModel(container=feeder, connection=connection, distributed=True) + feeder_area_model = feeder_model.distributed_areas[0] + + try: + distribution_area = feeder_model.get_from_triple(feeder, 'Feeder.DistributionArea')[0] + self.message['DistributionArea'] = json.loads(distribution_area.__repr__()) + except: + _log.warning(f'Feeder does not have an associated DistributionArea') + + try: + substation = feeder_model.get_from_triple(feeder, 'Feeder.NormalEnergizingSubstation')[0] + sub_msg = json.loads(substation.__repr__()) + sub_msg['NormalEnergizedFeeder'] = [] + + + fdr_msg = json.loads(feeder.__repr__()) + fdr_msg['FeederArea'] = self.add_feeder_area(feeder_area_model) + + sub_msg['NormalEnergizedFeeder'].append(fdr_msg) + self.message['DistributionArea']['Substations'] = [] + self.message['DistributionArea']['Substations'].append(sub_msg) + + except: + _log.warning(f'Feeder does not have an associated Normal Energizing Substation') + + def get_context_from_feeder_area(self, feeder_area: cim.FeederArea, + connection: ConnectionInterface) -> None: + + cim = connection.cim + if not isinstance(feeder_area, cim.FeederArea): + raise TypeError('feeder argument should be a cim.FeederArea object') + + feeder_area_model = DistributedArea(container=feeder_area, + connection=connection, distributed=True) + substation = feeder_area_model.get_from_triple(feeder_area, 'FeederArea.Substation')[0] + feeders = feeder_area_model.get_from_triple(feeder_area ) + + #TODO: Finish this method + + + def get_context_from_distribution_area(self, distribution_area: cim.DistributionArea, + connection:ConnectionInterface) -> None: + + cim = connection.cim + if not isinstance(distribution_area, cim.DistributionArea): + raise TypeError('feeder argument should be a cim.DistributionArea object') + + distribution_area_model = DistributedArea(container=distribution_area, + connection=connection, distributed=True) + distribution_area_model.get_all_edges(cim.DistributionArea) + distribution_area_model.get_all_edges(cim.Substation) + + self.message['DistributionArea'] = json.loads(distribution_area.__repr__()) + self.message['DistributionArea']['Substations'] = [] + + + for substation in distribution_area.Substations: + sub_msg = json.loads(substation.__repr__()) + sub_msg['NormalEnergizedFeeder'] = [] + + for feeder in substation.NormalEnergizedFeeder: + feeder_model = FeederModel(container=feeder, connection=connection, distributed=True) + + fdr_msg = json.loads(feeder.__repr__()) + # try: + feeder_area_model = feeder_model.distributed_areas[0] + feeder_area_model.get_all_edges(cim.FeederArea) + fdr_msg['FeederArea'] = self.add_feeder_area(feeder_area_model) + # except: + # _log.warning(f'Feeder {feeder.uri()} does not have distributed areas') + + sub_msg['NormalEnergizedFeeder'].append(fdr_msg) + del feeder_model + + new_sub = cim.Substation() + new_sub.uuid(uri = substation.uri()) + sub_model = DistributedArea(container=new_sub, + connection=connection, distributed=False) + sub_model.get_all_edges(cim.Substation) + sub_model.get_all_edges(cim.ConnectivityNode) + sub_model.get_all_edges(cim.Terminal) + sub_msg['AddressableEquipment'] = utils.identify_addressable(sub_model) + sub_msg['UnaddressableEquipment'] = utils.identify_unaddressable(sub_model) + sub_msg['Measurements'] = utils.identify_measurements(sub_model) + self.message['DistributionArea']['Substations'].append(sub_msg) + del sub_msg + del sub_model + + + + + + + + + def add_feeder_area(self, feeder_area_model: DistributedArea) -> dict: + feeder_area_model.get_all_edges(cim.FeederArea) + feeder_area = feeder_area_model.container + + msg = json.loads(feeder_area.__repr__()) + msg['BoundaryTerminals'] = utils.identify_boundaries(feeder_area_model) + msg['AddressableEquipment'] = utils.identify_addressable(feeder_area_model) + msg['UnaddressableEquipment'] = utils.identify_unaddressable(feeder_area_model) + msg['Measurements'] = utils.identify_measurements(feeder_area_model) + msg['SwitchAreas'] = [] + + for switch_area_model in feeder_area_model.distributed_areas: + sw_msg = self.add_switch_area(switch_area_model) + msg['SwitchAreas'].append(sw_msg) + + return msg + + + + def add_switch_area(self, switch_area_model:DistributedArea) -> dict: + + switch_area_model.get_all_edges(cim.SwitchArea) + switch_area = switch_area_model.container + feeder_area = switch_area_model.get_from_triple(switch_area, 'SwitchArea.FeederArea')[0] + + sw_msg = json.loads(switch_area.__repr__()) + sw_msg['FeederArea'] = json.loads(feeder_area.__repr__()) + sw_msg['BoundaryTerminals'] = utils.identify_boundaries(switch_area_model) + sw_msg['AddressableEquipment'] = utils.identify_addressable(switch_area_model) + sw_msg['UnaddressableEquipment'] = utils.identify_unaddressable(switch_area_model) + sw_msg['Measurements'] = utils.identify_measurements(switch_area_model) + sw_msg['SecondaryAreas'] = [] + + for secondary_area_model in switch_area_model.distributed_areas: + sec_msg = self.add_secondary_area(secondary_area_model) + sw_msg['SecondaryAreas'].append(sec_msg) + + return sw_msg + + def add_secondary_area(self, secondary_area_model: DistributedArea) -> dict: + + secondary_area_model.get_all_edges(cim.SecondaryArea) + secondary_area = secondary_area_model.container + switch_area = secondary_area_model.get_from_triple(secondary_area, 'SecondaryArea.SwitchArea')[0] + + sa_msg = json.loads(secondary_area.__repr__()) + sa_msg['SwitchArea'] = json.loads(switch_area.__repr__()) + sa_msg['BoundaryTerminals'] = utils.identify_boundaries(secondary_area_model) + sa_msg['AddressableEquipment'] = utils.identify_addressable(secondary_area_model) + sa_msg['UnaddressableEquipment'] = utils.identify_unaddressable(secondary_area_model) + sa_msg['Measurements'] = utils.identify_measurements(secondary_area_model) + + return sa_msg