Move implem specific rao parameters into extensions

Signed-off-by: Pauline Jean-Marie <[email protected]>

python-util/rao_parameter_modification.py

@@ -6,105 +6,160 @@
 
 import os
 import json
+import re
 from json import JSONDecodeError
-import yaml
 
 current_directory = os.getcwd()
 
+relevant_rao_param_names = ["objective-function", "range-actions-optimization", "topological-actions-optimization",
+                            "second-preventive-rao", "load-flow-and-sensitivity-computation", "multi-threading"]
 
-tag_by_file_type = {"json": "objective-function", "yaml": "rao-objective-function"}
 
 def rao_parameters_file(file_path):
     # do not work with yaml yet
     correct_version = False
-    score = 0
-    if "target" not in file_path and (file_path.endswith(".json") or file_path.endswith(".yml")):
-        ftype = "json" if file_path.endswith(".json") else "yaml"
+    has_rao_param_name = False
+    if "target" not in file_path and (file_path.endswith(".json")):
         with open(os.path.join(dirpath, filename), 'r') as file:
             for line in file:
                 if '"version" : "2.4"' in line or '"version" : "2.5"' in line:
                     correct_version = True
-                if tag_by_file_type[ftype] in line:
-                    score += 1
-                if "MAX_MIN_MARGIN" in line or "MAX_MIN_RELATIVE_MARGIN" in line:
-                    score += 1
-                if correct_version and score >= 2:
+                if any(name in line for name in relevant_rao_param_names):
+                    has_rao_param_name = True
+                if correct_version and has_rao_param_name:
                     return True
     return False
 
-def read_data(file_path) -> tuple[dict, str]:
-    if file_path.endswith(".json"):
-        with open(file_path, 'r') as file:
-            try:
-                return json.load(file), "json"
-            except JSONDecodeError as je:
-                print("in file " + file_path)
-                raise je
-    if file_path.endswith(".yml"):
-        with open(file_path, 'r') as file:
-            return yaml.safe_load(file), "yaml"  # ["rao-parameters"]
-
-def extract_leading_whitespace(line):
-    leading_whitespace = ""
-    for char in line:
-        if char.isspace():
-            leading_whitespace += char
-        else:
-            break
-    return leading_whitespace
 
-def write_data(new_data, file_path, file_type):
+# do not work with yaml yet
+def read_data(file_path) -> dict:
     with open(file_path, 'r') as file:
-        lines = file.readlines()
-        lines_to_write = []
-        inside_obj_fun_to_replace = False
-        for line in lines:
-            if tag_by_file_type[file_type] in line and tag_by_file_type[file_type] in new_data:
-                leading_whitespace = extract_leading_whitespace(line)
-                inside_obj_fun_to_replace = True
-            if inside_obj_fun_to_replace and ((file_type == "json" and "}" in line) or (file_type == "yaml" and line == "\n")):
-                obj_fun_str = f'"{tag_by_file_type[file_type]}" : ' + obj_function_as_str_lines(new_data, file_type)
-                for new_line in obj_fun_str.splitlines(True):
-                    lines_to_write.append(leading_whitespace + new_line)
-                inside_obj_fun_to_replace = False
-            elif not inside_obj_fun_to_replace:
-                lines_to_write.append(line)
-    with open(file_path, 'w') as file:
-        file.writelines(lines_to_write)
-
-
-def obj_function_as_str_lines(new_data, file_type):
-    if file_type == "json":
-        return json.dumps(new_data[tag_by_file_type[file_type]], indent=2, separators=(',', ' : ')) + ',\n'
-    else:
-        return yaml.dump(new_data[tag_by_file_type[file_type]], default_flow_style=False) + '\n'
-
-
-
-def new_rao_param(data: dict, file_path: str, file_type: str) -> dict:
+        try:
+            return json.load(file)
+        except JSONDecodeError as je:
+            print("in file " + file_path)
+            raise je
+
+
+class SpecialJSONEncoder(json.JSONEncoder):
+    """A JSON Encoder closer to actual rao parameter json format"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.indentation_level = 0
+
+    def encode(self, o):
+        """Encode JSON object *o* with respect to single line lists."""
+
+        if isinstance(o, (list, tuple)):
+            if len(o) == 0:
+                return "[ ]"
+            else:
+                return "[ " + ", ".join(self.encode(el) for el in o) + " ]"
+
+        elif isinstance(o, dict):
+            self.indentation_level += 1
+            output = [self.indent_str + f"{json.dumps(k)} : {self.encode(v)}" for k, v in o.items()]
+            self.indentation_level -= 1
+            return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
+
+        elif isinstance(o, float):
+            pattern = r'(\d+).?(\d*)e-0(\d+)'
+
+            def python_to_java_sci(match):
+                return f"{match.group(1)}{'.0' if not any(match.group(2)) else ('.' + match.group(2))}E-{match.group(3)}"
+
+            float_as_str = re.sub(pattern, python_to_java_sci, str(o))
+            if float_as_str == "0.0001":
+                return "1.0E-4"
+            return float_as_str
+
+        else:
+            return json.dumps(o)
+
+    @property
+    def indent_str(self) -> str:
+        return " " * self.indentation_level * self.indent
+
+    def iterencode(self, o, **kwargs):
+        """Required to also work with `json.dump`."""
+        return self.encode(o)
+
+
+def new_rao_param(data: dict, file_path: str) -> dict:
     try:
-        obj_fun = data[tag_by_file_type[file_type]]
-        prev_secure = "preventive-stop-criterion" not in obj_fun or obj_fun["preventive-stop-criterion"] == "SECURE"
-        if prev_secure and ("type" in obj_fun or "preventive-stop-criterion" in obj_fun):
-            obj_fun["type"] = "SECURE_FLOW"
-        elif not prev_secure and "type" not in obj_fun:
-            obj_fun["type"] = "MAX_MIN_MARGIN"
-        if "preventive-stop-criterion" in obj_fun:
-            del obj_fun["preventive-stop-criterion"]
+        move_to_extension(data, "objective-function", ["curative-min-obj-improvement"])
+        move_to_extension(data, "range-actions-optimization",
+                          ["max-mip-iterations", "pst-sensitivity-threshold", "pst-model",
+                           "hvdc-sensitivity-threshold", "injection-ra-sensitivity-threshold",
+                           "ra-range-shrinking", "linear-optimization-solver"])
+        move_to_extension(data, "topological-actions-optimization",
+                          ["max-preventive-search-tree-depth", "max-auto-search-tree-depth",
+                           "max-curative-search-tree-depth", "predefined-combinations",
+                           "skip-actions-far-from-most-limiting-element",
+                           "max-number-of-boundaries-for-skipping-actions"])
+        move_to_extension(data, "second-preventive-rao")
+        move_to_extension(data, "load-flow-and-sensitivity-computation")
+        if "range-actions-optimization" in data:
+            new_names = {"pst-penalty-cost": "pst-ra-min-impact-threshold",
+                         "hvdc-penalty-cost": "hvdc-ra-min-impact-threshold",
+                         "injection-ra-penalty-cost": "injection-ra-min-impact-threshold"}
+            data["range-actions-optimization"] = {new_names[k] if k in new_names else k: v for k, v in
+                                                  data["range-actions-optimization"].items()}
+        if "multi-threading" in data and any(data["multi-threading"]):
+            data["multi-threading"] = {"available-cpus": max(v for k, v in data["multi-threading"].items() if k in ("contingency-scenarios-in-parallel", "preventive-leaves-in-parallel"))}
+        move_to_extension(data, "multi-threading")
+        if "extensions" in data:
+            extensions = data["extensions"]
+            # put extensions at the end:
+            del data["extensions"]
+            data["extensions"] = extensions
     except KeyError as ke:
         raise KeyError("in file " + file_path) from ke
-    data[tag_by_file_type[file_type]] = obj_fun
     return data
 
 
+def move_to_extension(data: dict, name_level1: str, names_level2: list | None = None):
+    if name_level1 in data:
+        param_level_1: dict = data[name_level1]
+        if names_level2 is None:
+            st_params = get_or_create_st_params(data)
+            st_params[name_level1] = param_level_1
+            del data[name_level1]
+        else:
+            if any(set(names_level2).intersection(param_level_1.keys())):
+                st_params = get_or_create_st_params(data)
+                if name_level1 not in st_params:
+                    st_params[name_level1] = {}
+                for name_level_2 in names_level2:
+                    if name_level_2 in param_level_1:
+                        st_params[name_level1][name_level_2] = param_level_1[name_level_2]
+                        del param_level_1[name_level_2]
+                if not any(param_level_1):
+                    del data[name_level1]
+
+
+def get_or_create_st_params(data: dict) -> dict:
+    if "extensions" not in data:
+        data["extensions"] = {}
+    if "open-rao-search-tree-parameters" not in data["extensions"]:
+        data["extensions"]["open-rao-search-tree-parameters"] = {}
+    return data["extensions"]["open-rao-search-tree-parameters"]
+
+
+def write_data():
+    with open(file_path, 'w') as f:
+        json.dump(new_rao_params, f, indent=2, separators=(',', ' : '), cls=SpecialJSONEncoder)
+
 
+# do not work with yaml yet
 if __name__ == "__main__":
     base_dir = os.path.join(current_directory, "..")
     print(base_dir)
     for dirpath, dirnames, filenames in os.walk(base_dir):
         for filename in filenames:
             file_path = os.path.join(dirpath, filename)
             if rao_parameters_file(file_path):
-                data, file_type = read_data(file_path)
-                new_rao_params = new_rao_param(data, file_path, file_type)
-                write_data(new_rao_params, file_path, file_type)
+                old_rao_param = read_data(file_path)
+                new_rao_params = new_rao_param(old_rao_param, file_path)
+                write_data()

ra-optimisation/rao-api/src/main/java/com/powsybl/openrao/raoapi/RaoParametersCommons.java

@@ -24,6 +24,8 @@ private RaoParametersCommons() {
     // objective function parameters
     public static final String OBJECTIVE_FUNCTION = "objective-function";
     public static final String OBJECTIVE_FUNCTION_SECTION = "rao-objective-function";
+    public static final String ST_OBJECTIVE_FUNCTION_SECTION = "search-tree-objective-function";
+
     public static final String TYPE = "type";
     public static final String UNIT = "unit";
     public static final String CURATIVE_MIN_OBJ_IMPROVEMENT = "curative-min-obj-improvement";
@@ -32,16 +34,18 @@ private RaoParametersCommons() {
     // range actions optimization parameters
     public static final String RANGE_ACTIONS_OPTIMIZATION = "range-actions-optimization";
     public static final String RANGE_ACTIONS_OPTIMIZATION_SECTION = "rao-range-actions-optimization";
+    public static final String ST_RANGE_ACTIONS_OPTIMIZATION_SECTION = "search-tree-range-actions-optimization";
+
     public static final String MAX_MIP_ITERATIONS = "max-mip-iterations";
-    public static final String PST_PENALTY_COST = "pst-penalty-cost";
+    public static final String PST_RA_MIN_IMPACT_THRESHOLD = "pst-ra-min-impact-threshold";
     public static final String PST_SENSITIVITY_THRESHOLD = "pst-sensitivity-threshold";
     public static final String PST_MODEL = "pst-model";
-    public static final String HVDC_PENALTY_COST = "hvdc-penalty-cost";
+    public static final String HVDC_RA_MIN_IMPACT_THRESHOLD = "hvdc-ra-min-impact-threshold";
     public static final String HVDC_SENSITIVITY_THRESHOLD = "hvdc-sensitivity-threshold";
-    public static final String INJECTION_RA_PENALTY_COST = "injection-ra-penalty-cost";
+    public static final String INJECTION_RA_MIN_IMPACT_THRESHOLD = "injection-ra-min-impact-threshold";
     public static final String INJECTION_RA_SENSITIVITY_THRESHOLD = "injection-ra-sensitivity-threshold";
     public static final String LINEAR_OPTIMIZATION_SOLVER = "linear-optimization-solver";
-    public static final String LINEAR_OPTIMIZATION_SOLVER_SECTION = "rao-linear-optimization-solver";
+    public static final String LINEAR_OPTIMIZATION_SOLVER_SECTION = "search-tree-linear-optimization-solver";
     public static final String SOLVER = "solver";
     public static final String RELATIVE_MIP_GAP = "relative-mip-gap";
     public static final String SOLVER_SPECIFIC_PARAMETERS = "solver-specific-parameters";
@@ -50,6 +54,8 @@ private RaoParametersCommons() {
     // topological actions optimization parameters
     public static final String TOPOLOGICAL_ACTIONS_OPTIMIZATION = "topological-actions-optimization";
     public static final String TOPOLOGICAL_ACTIONS_OPTIMIZATION_SECTION = "rao-topological-actions-optimization";
+    public static final String ST_TOPOLOGICAL_ACTIONS_OPTIMIZATION_SECTION = "search-tree-topological-actions-optimization";
+
     public static final String MAX_PREVENTIVE_SEARCH_TREE_DEPTH = "max-preventive-search-tree-depth";
     public static final String MAX_AUTO_SEARCH_TREE_DEPTH = "max-auto-search-tree-depth";
     public static final String MAX_CURATIVE_SEARCH_TREE_DEPTH = "max-curative-search-tree-depth";
@@ -61,38 +67,24 @@ private RaoParametersCommons() {
 
     // Multi-threading parameters
     public static final String MULTI_THREADING = "multi-threading";
-    public static final String MULTI_THREADING_SECTION = "rao-multi-threading";
-    public static final String CONTINGENCY_SCENARIOS_IN_PARALLEL = "contingency-scenarios-in-parallel";
-    public static final String PREVENTIVE_LEAVES_IN_PARALLEL = "preventive-leaves-in-parallel";
-    public static final String AUTO_LEAVES_IN_PARALLEL = "auto-leaves-in-parallel";
-    public static final String CURATIVE_LEAVES_IN_PARALLEL = "curative-leaves-in-parallel";
+    public static final String MULTI_THREADING_SECTION = "search-tree-multi-threading";
+    public static final String AVAILABLE_CPUS = "available-cpus";
 
     // Second Preventive RAO parameters
     public static final String SECOND_PREVENTIVE_RAO = "second-preventive-rao";
-    public static final String SECOND_PREVENTIVE_RAO_SECTION = "rao-second-preventive-rao";
+    public static final String SECOND_PREVENTIVE_RAO_SECTION = "search-tree-second-preventive-rao";
     public static final String EXECUTION_CONDITION = "execution-condition";
     public static final String RE_OPTIMIZE_CURATIVE_RANGE_ACTIONS = "re-optimize-curative-range-actions";
     public static final String HINT_FROM_FIRST_PREVENTIVE_RAO = "hint-from-first-preventive-rao";
 
-    // RA usage limits per contingency parameters
-    public static final String RA_USAGE_LIMITS_PER_CONTINGENCY = "ra-usage-limits-per-contingency";
-    public static final String RA_USAGE_LIMITS_PER_CONTINGENCY_SECTION = "rao-ra-usage-limits-per-contingency";
-
-    public static final String MAX_CURATIVE_RA = "max-curative-ra";
-    public static final String MAX_CURATIVE_TSO = "max-curative-tso";
-    public static final String MAX_CURATIVE_TOPO_PER_TSO = "max-curative-topo-per-tso";
-    public static final String MAX_CURATIVE_PST_PER_TSO = "max-curative-pst-per-tso";
-    public static final String MAX_CURATIVE_RA_PER_TSO = "max-curative-ra-per-tso";
-
     // Not optimized cnecs parameters
     public static final String NOT_OPTIMIZED_CNECS = "not-optimized-cnecs";
     public static final String NOT_OPTIMIZED_CNECS_SECTION = "rao-not-optimized-cnecs";
     public static final String DO_NOT_OPTIMIZE_CURATIVE_CNECS = "do-not-optimize-curative-cnecs-for-tsos-without-cras";
 
-    // Not optimized cnecs parameters
+    // Load flow and sensitivity parameters
     public static final String LOAD_FLOW_AND_SENSITIVITY_COMPUTATION = "load-flow-and-sensitivity-computation";
-    public static final String LOAD_FLOW_AND_SENSITIVITY_COMPUTATION_SECTION = "rao-load-flow-and-sensitivity-computation";
-
+    public static final String LOAD_FLOW_AND_SENSITIVITY_COMPUTATION_SECTION = "search-tree-load-flow-and-sensitivity-computation";
     public static final String LOAD_FLOW_PROVIDER = "load-flow-provider";
     public static final String SENSITIVITY_PROVIDER = "sensitivity-provider";
     public static final String SENSITIVITY_FAILURE_OVERCOST = "sensitivity-failure-overcost";
@@ -120,6 +112,7 @@ private RaoParametersCommons() {
     public static final String RELATIVE_MARGINS_SECTION = "rao-relative-margins-parameters";
     public static final String PTDF_BOUNDARIES = "ptdf-boundaries";
     public static final String PTDF_SUM_LOWER_BOUND = "ptdf-sum-lower-bound";
+    public static final String SEARCH_TREE_PARAMETERS = "open-rao-search-tree-parameters";
 
     public static PtdfApproximation stringToPtdfApproximation(String string) {
         try {