From ba113d0142bad77d458c092fff9cf9fdf93fb682 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 10:45:04 -0600 Subject: [PATCH 1/9] add keywords to the settings --- src/core/settings.jl | 4 ++++ src/operation/decision_model.jl | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/settings.jl b/src/core/settings.jl index d9dd095cbc..063ad1cce0 100644 --- a/src/core/settings.jl +++ b/src/core/settings.jl @@ -16,6 +16,7 @@ struct Settings export_pwl_vars::Bool allow_fails::Bool rebuild_model::Bool + export_optimization_model::Bool store_variable_names::Bool check_numerical_bounds::Bool ext::Dict{String, Any} @@ -41,6 +42,7 @@ function Settings( allow_fails::Bool = false, check_numerical_bounds = true, rebuild_model = false, + export_optimization_model = false, store_variable_names = false, ext = Dict{String, Any}(), ) @@ -77,6 +79,7 @@ function Settings( export_pwl_vars, allow_fails, rebuild_model, + export_optimization_model, store_variable_names, check_numerical_bounds, ext, @@ -151,6 +154,7 @@ get_detailed_optimizer_stats(settings::Settings) = settings.detailed_optimizer_s get_direct_mode_optimizer(settings::Settings) = settings.direct_mode_optimizer get_store_variable_names(settings::Settings) = settings.store_variable_names get_rebuild_model(settings::Settings) = settings.rebuild_model +get_export_optimization_model(settings::Settings) = settings.export_optimization_model use_time_series_cache(settings::Settings) = settings.time_series_cache_size > 0 function set_horizon!(settings::Settings, horizon::Dates.TimePeriod) diff --git a/src/operation/decision_model.jl b/src/operation/decision_model.jl index b0718de645..83dfe253d3 100644 --- a/src/operation/decision_model.jl +++ b/src/operation/decision_model.jl @@ -114,6 +114,7 @@ function DecisionModel{M}( direct_mode_optimizer = false, store_variable_names = false, rebuild_model = false, + export_optimization_model = false, check_numerical_bounds = true, initial_time = UNSET_INI_TIME, time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES, @@ -139,6 +140,7 @@ function DecisionModel{M}( check_numerical_bounds = check_numerical_bounds, store_variable_names = store_variable_names, rebuild_model = rebuild_model, + export_optimization_model = export_optimization_model, ) return DecisionModel{M}(template, sys, settings, jump_model; name = name) end @@ -444,7 +446,7 @@ keyword arguments to that function. - `console_level = Logging.Error`: - `file_level = Logging.Info`: - `disable_timer_outputs = false` : Enable/Disable timing outputs - - `serialize::Bool = true`: If true, serialize the model to a file to allow re-execution later. + - `export_optimization_problem::Bool = true`: If true, serialize the model to a file to allow re-execution later. # Examples @@ -459,7 +461,7 @@ function solve!( console_level = Logging.Error, file_level = Logging.Info, disable_timer_outputs = false, - serialize = true, + export_optimization_problem = true, kwargs..., ) build_if_not_already_built!( @@ -500,7 +502,7 @@ function solve!( current_time, ) end - if serialize + if export_optimization_problem TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Serialize" begin serialize_problem(model; optimizer = optimizer) serialize_optimization_model(model) From 02c57b87c8a822ae7f652b3145cd321f5b8158f2 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 10:45:17 -0600 Subject: [PATCH 2/9] match interface for em --- src/operation/emulation_model.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/operation/emulation_model.jl b/src/operation/emulation_model.jl index c635cf35b6..079ea31573 100644 --- a/src/operation/emulation_model.jl +++ b/src/operation/emulation_model.jl @@ -507,7 +507,7 @@ keyword arguments to that function. - `export_problem_results::Bool`: If true, export OptimizationProblemResults DataFrames to CSV files. - `output_dir::String`: Required if the model is not already built, otherwise ignored - `enable_progress_bar::Bool`: Enables/Disable progress bar printing - - `serialize::Bool`: If true, serialize the model to a file to allow re-execution later. + - `export_optimization_model::Bool`: If true, serialize the model to a file to allow re-execution later. # Examples @@ -522,7 +522,7 @@ function run!( console_level = Logging.Error, file_level = Logging.Info, disable_timer_outputs = false, - serialize = true, + export_optimization_model = true, kwargs..., ) build_if_not_already_built!( @@ -555,7 +555,7 @@ function run!( run_impl!(model; kwargs...) set_run_status!(model, RunStatus.SUCCESSFULLY_FINALIZED) end - if serialize + if export_optimization_model TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Serialize" begin optimizer = get(kwargs, :optimizer, nothing) serialize_problem(model; optimizer = optimizer) From c675fd5e06dbe302081a462fbb1f6ef223e10376 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 10:45:25 -0600 Subject: [PATCH 3/9] add export at each run --- src/operation/operation_model_interface.jl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index 1eb0e8ecc3..e833b5e522 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -104,13 +104,23 @@ end function solve_impl!(model::OperationModel) container = get_optimization_container(model) + model_name = get_name(model) + ts = get_current_timestamp(model) + output_dir = get_output_dir(model) + + if get_export_optimization_model(get_settings(model)) + model_output_dir = joinpath(output_dir, "optimization_model_exports") + if !ispath(model_output_dir) + mkdir(model_output_dir) + end + model_export_path = joinpath(model_output_dir, "exported_$(model_name)_$(ts).json") + serialize_optimization_model(container, model_export_path) + end + status = solve_impl!(container, get_system(model)) set_run_status!(model, status) if status != RunStatus.SUCCESSFULLY_FINALIZED settings = get_settings(model) - model_name = get_name(model) - ts = get_current_timestamp(model) - output_dir = get_output_dir(model) infeasible_opt_path = joinpath(output_dir, "infeasible_$(model_name).json") @error("Serializing Infeasible Problem at $(infeasible_opt_path)") serialize_optimization_model(container, infeasible_opt_path) From a194f20f073c577a1260717a72a6baff49bcb7f1 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 10:45:31 -0600 Subject: [PATCH 4/9] add new tests --- test/test_simulation_execute.jl | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/test_simulation_execute.jl b/test/test_simulation_execute.jl index 6bc4581769..22b8cbb251 100644 --- a/test/test_simulation_execute.jl +++ b/test/test_simulation_execute.jl @@ -1,4 +1,5 @@ -function test_single_stage_sequential(in_memory, rebuild) +function test_single_stage_sequential(in_memory, rebuild, export_model) + tmp_dir = mktempdir(; cleanup = true) template_ed = get_template_nomin_ed_simulation() c_sys = PSB.build_system(PSITestSystems, "c_sys5_uc") models = SimulationModels([ @@ -8,6 +9,7 @@ function test_single_stage_sequential(in_memory, rebuild) name = "ED", optimizer = ipopt_optimizer, rebuild_model = rebuild, + export_optimization_model = export_model, ), ]) test_sequence = @@ -20,20 +22,29 @@ function test_single_stage_sequential(in_memory, rebuild) steps = 2, models = models, sequence = test_sequence, - simulation_folder = mktempdir(; cleanup = true), + simulation_folder = tmp_dir, ) build_out = build!(sim_single) @test build_out == PSI.SimulationBuildStatus.BUILT execute_out = execute!(sim_single; in_memory = in_memory) @test execute_out == PSI.RunStatus.SUCCESSFULLY_FINALIZED + return tmp_dir end @testset "Single stage sequential tests" begin for in_memory in (true, false), rebuild in (true, false) - test_single_stage_sequential(in_memory, rebuild) + test_single_stage_sequential(in_memory, rebuild, false) end end +@testset "Test model export at each solve" begin + folder = test_single_stage_sequential(true, false, true) + test_path = + joinpath(folder, "consecutive", "problems", "ED", "optimization_model_exports") + @test ispath(test_path) + @test length(readdir(test_path)) == 2 +end + function test_2_stage_decision_models_with_feedforwards(in_memory) template_uc = get_template_basic_uc_simulation() template_ed = get_template_nomin_ed_simulation() From ed9ea74bd0d787c99f8f30b3f8578f5e369f6bd6 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 11:04:29 -0600 Subject: [PATCH 5/9] fix em model --- test/test_model_emulation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_model_emulation.jl b/test/test_model_emulation.jl index f824c7ec4b..9688073310 100644 --- a/test/test_model_emulation.jl +++ b/test/test_model_emulation.jl @@ -223,7 +223,7 @@ end model; executions = 10, output_dir = mktempdir(; cleanup = true), - serialize = serialize, + export_optimization_model = serialize, ) == PSI.RunStatus.SUCCESSFULLY_FINALIZED end end From 5855e2b72ff10f750f164af6e4340bc87ed1141f Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 12:08:32 -0600 Subject: [PATCH 6/9] replace colon for windows compat --- src/operation/operation_model_interface.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index e833b5e522..1250bcc1ab 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -113,7 +113,8 @@ function solve_impl!(model::OperationModel) if !ispath(model_output_dir) mkdir(model_output_dir) end - model_export_path = joinpath(model_output_dir, "exported_$(model_name)_$(ts).json") + tss = replace!("$(ts)", ":" => "_") + model_export_path = joinpath(model_output_dir, "exported_$(model_name)_$(tss).json") serialize_optimization_model(container, model_export_path) end From 9b1727b23eb0b190cd1a113d0d1c72bcc8707eba Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 13:37:35 -0600 Subject: [PATCH 7/9] remove bang --- src/operation/operation_model_interface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index 1250bcc1ab..0ae14105e0 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -113,7 +113,7 @@ function solve_impl!(model::OperationModel) if !ispath(model_output_dir) mkdir(model_output_dir) end - tss = replace!("$(ts)", ":" => "_") + tss = replace("$(ts)", ":" => "_") model_export_path = joinpath(model_output_dir, "exported_$(model_name)_$(tss).json") serialize_optimization_model(container, model_export_path) end From 0ae7426e50186caee45a984406148e57591711fc Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 15:06:19 -0600 Subject: [PATCH 8/9] use mkpath --- src/operation/operation_model_interface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index 0ae14105e0..739648573b 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -111,7 +111,7 @@ function solve_impl!(model::OperationModel) if get_export_optimization_model(get_settings(model)) model_output_dir = joinpath(output_dir, "optimization_model_exports") if !ispath(model_output_dir) - mkdir(model_output_dir) + mkpath(model_output_dir) end tss = replace("$(ts)", ":" => "_") model_export_path = joinpath(model_output_dir, "exported_$(model_name)_$(tss).json") From 2b45701aab6f050b96b62c4aa995b718b25e264a Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 11 Sep 2024 15:41:48 -0600 Subject: [PATCH 9/9] remove if condition --- src/operation/operation_model_interface.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index 739648573b..10efe61f36 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -110,9 +110,7 @@ function solve_impl!(model::OperationModel) if get_export_optimization_model(get_settings(model)) model_output_dir = joinpath(output_dir, "optimization_model_exports") - if !ispath(model_output_dir) - mkpath(model_output_dir) - end + mkpath(model_output_dir) tss = replace("$(ts)", ":" => "_") model_export_path = joinpath(model_output_dir, "exported_$(model_name)_$(tss).json") serialize_optimization_model(container, model_export_path)