Merge pull request #572 from BDonnot/bd_dev

Bd dev

CHANGELOG.rst

@@ -34,11 +34,19 @@ Change Log
 
 [1.9.8] - 20xx-yy-zz
 ----------------------
+- [FIXED] the `backend.check_kirchoff` function was not correct when some elements were disconnected 
+  (the wrong columns of the p_bus and q_bus was set in case of disconnected elements)
+- [FIXED] `PandapowerBackend`, when no slack was present
+- [FIXED] the "BaseBackendTest" class did not correctly detect divergence in most cases (which lead 
+  to weird bugs in failing tests)
+- [ADDED] A type of environment that does not perform the "emulation of the protections"
+  for some part of the grid (`MaskedEnvironment`) see https://github.com/rte-france/Grid2Op/issues/571
 - [IMPROVED] the CI speed: by not testing every possible numpy version but only most ancient and most recent
 - [IMPROVED] Runner now test grid2op version 1.9.6 and 1.9.7
 - [IMPROVED] refacto `gridobj_cls._clear_class_attribute` and `gridobj_cls._clear_grid_dependant_class_attributes`
 - [IMPROVED] the bahviour of the generic class `MakeBackend` used for the test suite.
 - [IMPROVED] re introducing python 12 testing
+- [IMPROVED] error messages in the automatic test suite (`AAATestBackendAPI`)
 
 [1.9.7] - 2023-12-01
 ----------------------

grid2op/Backend/backend.py

@@ -1023,10 +1023,12 @@ def next_grid_state(self,
             ] = True
 
             # disconnect the current power lines
-            if to_disc[lines_status].sum() == 0:
-                # no powerlines have been disconnected at this time step, i stop the computation there
+            if to_disc[lines_status].any() == 0:
+                # no powerlines have been disconnected at this time step, 
+                # i stop the computation there
                 break
             disconnected_during_cf[to_disc] = ts
+
             # perform the disconnection action
             for i, el in enumerate(to_disc):
                 if el:
@@ -1124,30 +1126,35 @@ def check_kirchoff(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray
         p_ex, q_ex, v_ex, *_ = self.lines_ex_info()
         p_gen, q_gen, v_gen = self.generators_info()
         p_load, q_load, v_load = self.loads_info()
-        if self.n_storage > 0:
+        cls = type(self)
+        if cls.n_storage > 0:
             p_storage, q_storage, v_storage = self.storages_info()
 
         # fist check the "substation law" : nothing is created at any substation
-        p_subs = np.zeros(self.n_sub, dtype=dt_float)
-        q_subs = np.zeros(self.n_sub, dtype=dt_float)
+        p_subs = np.zeros(cls.n_sub, dtype=dt_float)
+        q_subs = np.zeros(cls.n_sub, dtype=dt_float)
 
         # check for each bus
-        p_bus = np.zeros((self.n_sub, 2), dtype=dt_float)
-        q_bus = np.zeros((self.n_sub, 2), dtype=dt_float)
+        p_bus = np.zeros((cls.n_sub, 2), dtype=dt_float)
+        q_bus = np.zeros((cls.n_sub, 2), dtype=dt_float)
         v_bus = (
-            np.zeros((self.n_sub, 2, 2), dtype=dt_float) - 1.0
+            np.zeros((cls.n_sub, 2, 2), dtype=dt_float) - 1.0
         )  # sub, busbar, [min,max]
         topo_vect = self.get_topo_vect()
 
         # bellow i'm "forced" to do a loop otherwise, numpy do not compute the "+=" the way I want it to.
         # for example, if two powerlines are such that line_or_to_subid is equal (eg both connected to substation 0)
         # then numpy do not guarantee that `p_subs[self.line_or_to_subid] += p_or` will add the two "corresponding p_or"
         # TODO this can be vectorized with matrix product, see example in obs.flow_bus_matrix (BaseObervation.py)
-        for i in range(self.n_line):
-            sub_or_id = self.line_or_to_subid[i]
-            sub_ex_id = self.line_ex_to_subid[i]
-            loc_bus_or = topo_vect[self.line_or_pos_topo_vect[i]] - 1
-            loc_bus_ex = topo_vect[self.line_ex_pos_topo_vect[i]] - 1
+        for i in range(cls.n_line):
+            sub_or_id = cls.line_or_to_subid[i]
+            sub_ex_id = cls.line_ex_to_subid[i]
+            if (topo_vect[cls.line_or_pos_topo_vect[i]] == -1 or
+                topo_vect[cls.line_ex_pos_topo_vect[i]] == -1):
+                # line is disconnected
+                continue
+            loc_bus_or = topo_vect[cls.line_or_pos_topo_vect[i]] - 1
+            loc_bus_ex = topo_vect[cls.line_ex_pos_topo_vect[i]] - 1
 
             # for substations
             p_subs[sub_or_id] += p_or[i]
@@ -1184,92 +1191,104 @@ def check_kirchoff(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray
                 v_bus[sub_ex_id,loc_bus_ex,][0] = min(v_bus[sub_ex_id,loc_bus_ex,][0],v_ex[i],)
                 v_bus[sub_ex_id,loc_bus_ex,][1] = max(v_bus[sub_ex_id,loc_bus_ex,][1],v_ex[i],)
 
-        for i in range(self.n_gen):
+        for i in range(cls.n_gen):
+            if topo_vect[cls.gen_pos_topo_vect[i]] == -1:
+                # gen is disconnected
+                continue
+
             # for substations
-            p_subs[self.gen_to_subid[i]] -= p_gen[i]
-            q_subs[self.gen_to_subid[i]] -= q_gen[i]
+            p_subs[cls.gen_to_subid[i]] -= p_gen[i]
+            q_subs[cls.gen_to_subid[i]] -= q_gen[i]
 
             # for bus
             p_bus[
-                self.gen_to_subid[i], topo_vect[self.gen_pos_topo_vect[i]] - 1
+                cls.gen_to_subid[i], topo_vect[cls.gen_pos_topo_vect[i]] - 1
             ] -= p_gen[i]
             q_bus[
-                self.gen_to_subid[i], topo_vect[self.gen_pos_topo_vect[i]] - 1
+                cls.gen_to_subid[i], topo_vect[cls.gen_pos_topo_vect[i]] - 1
             ] -= q_gen[i]
 
             # compute max and min values
             if v_gen[i]:
                 # but only if gen is connected
-                v_bus[self.gen_to_subid[i], topo_vect[self.gen_pos_topo_vect[i]] - 1][
+                v_bus[cls.gen_to_subid[i], topo_vect[cls.gen_pos_topo_vect[i]] - 1][
                     0
                 ] = min(
                     v_bus[
-                        self.gen_to_subid[i], topo_vect[self.gen_pos_topo_vect[i]] - 1
+                        cls.gen_to_subid[i], topo_vect[cls.gen_pos_topo_vect[i]] - 1
                     ][0],
                     v_gen[i],
                 )
-                v_bus[self.gen_to_subid[i], topo_vect[self.gen_pos_topo_vect[i]] - 1][
+                v_bus[cls.gen_to_subid[i], topo_vect[cls.gen_pos_topo_vect[i]] - 1][
                     1
                 ] = max(
                     v_bus[
-                        self.gen_to_subid[i], topo_vect[self.gen_pos_topo_vect[i]] - 1
+                        cls.gen_to_subid[i], topo_vect[cls.gen_pos_topo_vect[i]] - 1
                     ][1],
                     v_gen[i],
                 )
 
-        for i in range(self.n_load):
+        for i in range(cls.n_load):
+            if topo_vect[cls.load_pos_topo_vect[i]] == -1:
+                # load is disconnected
+                continue
+
             # for substations
-            p_subs[self.load_to_subid[i]] += p_load[i]
-            q_subs[self.load_to_subid[i]] += q_load[i]
+            p_subs[cls.load_to_subid[i]] += p_load[i]
+            q_subs[cls.load_to_subid[i]] += q_load[i]
 
             # for buses
             p_bus[
-                self.load_to_subid[i], topo_vect[self.load_pos_topo_vect[i]] - 1
+                cls.load_to_subid[i], topo_vect[cls.load_pos_topo_vect[i]] - 1
             ] += p_load[i]
             q_bus[
-                self.load_to_subid[i], topo_vect[self.load_pos_topo_vect[i]] - 1
+                cls.load_to_subid[i], topo_vect[cls.load_pos_topo_vect[i]] - 1
             ] += q_load[i]
 
             # compute max and min values
             if v_load[i]:
                 # but only if load is connected
-                v_bus[self.load_to_subid[i], topo_vect[self.load_pos_topo_vect[i]] - 1][
+                v_bus[cls.load_to_subid[i], topo_vect[cls.load_pos_topo_vect[i]] - 1][
                     0
                 ] = min(
                     v_bus[
-                        self.load_to_subid[i], topo_vect[self.load_pos_topo_vect[i]] - 1
+                        cls.load_to_subid[i], topo_vect[cls.load_pos_topo_vect[i]] - 1
                     ][0],
                     v_load[i],
                 )
-                v_bus[self.load_to_subid[i], topo_vect[self.load_pos_topo_vect[i]] - 1][
+                v_bus[cls.load_to_subid[i], topo_vect[cls.load_pos_topo_vect[i]] - 1][
                     1
                 ] = max(
                     v_bus[
-                        self.load_to_subid[i], topo_vect[self.load_pos_topo_vect[i]] - 1
+                        cls.load_to_subid[i], topo_vect[cls.load_pos_topo_vect[i]] - 1
                     ][1],
                     v_load[i],
                 )
 
-        for i in range(self.n_storage):
-            p_subs[self.storage_to_subid[i]] += p_storage[i]
-            q_subs[self.storage_to_subid[i]] += q_storage[i]
+        for i in range(cls.n_storage):
+            if topo_vect[cls.storage_pos_topo_vect[i]] == -1:
+                # storage is disconnected
+                continue
+
+            p_subs[cls.storage_to_subid[i]] += p_storage[i]
+            q_subs[cls.storage_to_subid[i]] += q_storage[i]
             p_bus[
-                self.storage_to_subid[i], topo_vect[self.storage_pos_topo_vect[i]] - 1
+                cls.storage_to_subid[i], topo_vect[cls.storage_pos_topo_vect[i]] - 1
             ] += p_storage[i]
             q_bus[
-                self.storage_to_subid[i], topo_vect[self.storage_pos_topo_vect[i]] - 1
+                cls.storage_to_subid[i], topo_vect[cls.storage_pos_topo_vect[i]] - 1
             ] += q_storage[i]
 
             # compute max and min values
             if v_storage[i] > 0:
                 # the storage unit is connected
                 v_bus[
-                    self.storage_to_subid[i],
-                    topo_vect[self.storage_pos_topo_vect[i]] - 1,
+                    cls.storage_to_subid[i],
+                    topo_vect[cls.storage_pos_topo_vect[i]] - 1,
                 ][0] = min(
                     v_bus[
-                        self.storage_to_subid[i],
-                        topo_vect[self.storage_pos_topo_vect[i]] - 1,
+                        cls.storage_to_subid[i],
+                        topo_vect[cls.storage_pos_topo_vect[i]] - 1,
                     ][0],
                     v_storage[i],
                 )
@@ -1278,29 +1297,33 @@ def check_kirchoff(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray
                     topo_vect[self.storage_pos_topo_vect[i]] - 1,
                 ][1] = max(
                     v_bus[
-                        self.storage_to_subid[i],
-                        topo_vect[self.storage_pos_topo_vect[i]] - 1,
+                        cls.storage_to_subid[i],
+                        topo_vect[cls.storage_pos_topo_vect[i]] - 1,
                     ][1],
                     v_storage[i],
                 )
 
-        if type(self).shunts_data_available:
+        if cls.shunts_data_available:
             p_s, q_s, v_s, bus_s = self.shunt_info()
-            for i in range(self.n_shunt):
+            for i in range(cls.n_shunt):
+                if bus_s[i] == -1:
+                    # shunt is disconnected
+                    continue
+
                 # for substations
-                p_subs[self.shunt_to_subid[i]] += p_s[i]
-                q_subs[self.shunt_to_subid[i]] += q_s[i]
+                p_subs[cls.shunt_to_subid[i]] += p_s[i]
+                q_subs[cls.shunt_to_subid[i]] += q_s[i]
 
                 # for buses
-                p_bus[self.shunt_to_subid[i], bus_s[i] - 1] += p_s[i]
-                q_bus[self.shunt_to_subid[i], bus_s[i] - 1] += q_s[i]
+                p_bus[cls.shunt_to_subid[i], bus_s[i] - 1] += p_s[i]
+                q_bus[cls.shunt_to_subid[i], bus_s[i] - 1] += q_s[i]
 
                 # compute max and min values
-                v_bus[self.shunt_to_subid[i], bus_s[i] - 1][0] = min(
-                    v_bus[self.shunt_to_subid[i], bus_s[i] - 1][0], v_s[i]
+                v_bus[cls.shunt_to_subid[i], bus_s[i] - 1][0] = min(
+                    v_bus[cls.shunt_to_subid[i], bus_s[i] - 1][0], v_s[i]
                 )
-                v_bus[self.shunt_to_subid[i], bus_s[i] - 1][1] = max(
-                    v_bus[self.shunt_to_subid[i], bus_s[i] - 1][1], v_s[i]
+                v_bus[cls.shunt_to_subid[i], bus_s[i] - 1][1] = max(
+                    v_bus[cls.shunt_to_subid[i], bus_s[i] - 1][1], v_s[i]
                 )
         else:
             warnings.warn(

grid2op/Environment/__init__.py

@@ -5,7 +5,8 @@
     "SingleEnvMultiProcess",
     "MultiEnvMultiProcess",
     "MultiMixEnvironment",
-    "TimedOutEnvironment"
+    "TimedOutEnvironment",
+    "MaskedEnvironment"
 ]
 
 from grid2op.Environment.baseEnv import BaseEnv
@@ -15,3 +16,4 @@
 from grid2op.Environment.multiEnvMultiProcess import MultiEnvMultiProcess
 from grid2op.Environment.multiMixEnv import MultiMixEnvironment
 from grid2op.Environment.timedOutEnv import TimedOutEnvironment
+from grid2op.Environment.maskedEnvironment import MaskedEnvironment

grid2op/Environment/baseEnv.py

@@ -342,7 +342,7 @@ def __init__(
         )
         self._timestep_overflow: np.ndarray = None
         self._nb_timestep_overflow_allowed: np.ndarray = None
-        self._hard_overflow_threshold: float = self._parameters.HARD_OVERFLOW_THRESHOLD
+        self._hard_overflow_threshold: np.ndarray  = None
 
         # store actions "cooldown"
         self._times_before_line_status_actionable: np.ndarray = None
@@ -626,7 +626,7 @@ def _custom_deepcopy_for_copy(self, new_obj, dict_=None):
         new_obj._nb_timestep_overflow_allowed = copy.deepcopy(
             self._nb_timestep_overflow_allowed
         )
-        new_obj._hard_overflow_threshold = self._hard_overflow_threshold
+        new_obj._hard_overflow_threshold = copy.deepcopy(self._hard_overflow_threshold)
 
         # store actions "cooldown"
         new_obj._times_before_line_status_actionable = copy.deepcopy(
@@ -1204,7 +1204,6 @@ def _has_been_initialized(self):
         self._gen_downtime = np.zeros(self.n_gen, dtype=dt_int)
         self._gen_activeprod_t = np.zeros(self.n_gen, dtype=dt_float)
         self._gen_activeprod_t_redisp = np.zeros(self.n_gen, dtype=dt_float)
-        self._nb_timestep_overflow_allowed = np.ones(shape=self.n_line, dtype=dt_int)
         self._max_timestep_line_status_deactivated = (
             self._parameters.NB_TIMESTEP_COOLDOWN_LINE
         )
@@ -1220,6 +1219,11 @@ def _has_been_initialized(self):
             fill_value=self._parameters.NB_TIMESTEP_OVERFLOW_ALLOWED,
             dtype=dt_int,
         )
+        self._hard_overflow_threshold = np.full(
+            shape=(self.n_line,),
+            fill_value=self._parameters.HARD_OVERFLOW_THRESHOLD,
+            dtype=dt_float,
+        )
         self._timestep_overflow = np.zeros(shape=(self.n_line,), dtype=dt_int)
 
         # update the parameters
@@ -1261,7 +1265,6 @@ def _update_parameters(self):
         # type of power flow to play
         # if True, then it will not disconnect lines above their thermal limits
         self._no_overflow_disconnection = self._parameters.NO_OVERFLOW_DISCONNECTION
-        self._hard_overflow_threshold = self._parameters.HARD_OVERFLOW_THRESHOLD
 
         # store actions "cooldown"
         self._max_timestep_line_status_deactivated = (
@@ -1275,7 +1278,7 @@ def _update_parameters(self):
         self._nb_timestep_overflow_allowed[
             :
         ] = self._parameters.NB_TIMESTEP_OVERFLOW_ALLOWED
-
+        self._hard_overflow_threshold[:] = self._parameters.HARD_OVERFLOW_THRESHOLD
         # hard overflow part
         self._env_dc = self._parameters.ENV_DC
 
@@ -2957,16 +2960,19 @@ def _aux_register_env_converged(self, disc_lines, action, init_line_status, new_
         # TODO is non zero and disconnected, this should be ok.
         self._time_extract_obs += time.perf_counter() - beg_res
 
+    def _backend_next_grid_state(self):
+        """overlaoded in MaskedEnv"""
+        return self.backend.next_grid_state(env=self, is_dc=self._env_dc)
+
     def _aux_run_pf_after_state_properly_set(
         self, action, init_line_status, new_p, except_
     ):
         has_error = True
+        detailed_info = None
         try:
             # compute the next _grid state
             beg_pf = time.perf_counter()
-            disc_lines, detailed_info, conv_ = self.backend.next_grid_state(
-                env=self, is_dc=self._env_dc
-            )
+            disc_lines, detailed_info, conv_ = self._backend_next_grid_state()
             self._disc_lines[:] = disc_lines
             self._time_powerflow += time.perf_counter() - beg_pf
             if conv_ is None:
@@ -3327,7 +3333,7 @@ def _reset_vectors_and_timings(self):
         ] = self._parameters.NB_TIMESTEP_OVERFLOW_ALLOWED
 
         self.nb_time_step = 0  # to have the first step at 0
-        self._hard_overflow_threshold = self._parameters.HARD_OVERFLOW_THRESHOLD
+        self._hard_overflow_threshold[:] = self._parameters.HARD_OVERFLOW_THRESHOLD
         self._env_dc = self._parameters.ENV_DC
 
         self._times_before_line_status_actionable[:] = 0

grid2op/Environment/environment.py

@@ -418,7 +418,7 @@ def _init_backend(
             raise Grid2OpException(
                 "Impossible to initialize the powergrid, the powerflow diverge at iteration 0. "
                 "Available information are: {}".format(info)
-            )
+            ) from info["exception"][0]
 
         # test the backend returns object of the proper size
         if need_process_backend: