Fix tolerance calculation in tests

lbonaldo · Nov 1, 2023 · bf40ed4 · bf40ed4
1 parent 4e3ef50
commit bf40ed4
Show file tree

Hide file tree

Showing 15 changed files with 77 additions and 147 deletions.
diff --git a/test/Electrolyzer/highs_settings.yml b/test/Electrolyzer/highs_settings.yml
@@ -1,13 +1,7 @@
 # HiGHS Solver Parameters
 # Common solver settings
-Feasib_Tol: 1.0e-04        # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-Optimal_Tol: 1.0e-04       # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-TimeLimit: 1.0e23             # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
-Pre_Solve: on          # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
-Method: ipm             #HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
-
-#highs-specific solver settings
-
-# run the crossover routine for ipx
-# [type: string, advanced: "on", range: {"off", "on"}, default: "off"]
-run_crossover: "on"
+Feasib_Tol: 1.0e-04       # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+TimeLimit: 1.0e23         # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
+Pre_Solve: on             # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
+Method: ipm               # HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
+ipm_optimality_tolerance: 1e-04
diff --git a/test/MethodofMorris/highs_settings.yml b/test/MethodofMorris/highs_settings.yml
@@ -1,11 +1,8 @@
 # HiGHS Solver Parameters
 # Common solver settings
-Feasib_Tol: 1.0e-05        # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-Optimal_Tol: 1.0e-05       # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-TimeLimit: 1.0e23              # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
-Pre_Solve: choose          # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
-Method: ipm            #HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
-
-# run the crossover routine for ipx
-# [type: string, advanced: "on", range: {"off", "on"}, default: "off"]
-run_crossover: "off"
+Feasib_Tol: 1.0e-05     # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+Optimal_Tol: 1.0e-05    # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+TimeLimit: 1.0e23       # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
+Pre_Solve: on           # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
+Method: ipm             # HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
+ipm_optimality_tolerance: 1e-04
diff --git a/test/MultiStage/highs_settings.yml b/test/MultiStage/highs_settings.yml
@@ -1,10 +1,11 @@
 # HiGHS Solver Parameters
 # Common solver settings
-Feasib_Tol: 1.0e-05        # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-Optimal_Tol: 1.0e-05       # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-TimeLimit: 1.0e23              # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
-Pre_Solve: choose          # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
-Method: ipm             #HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
+Feasib_Tol: 1.0e-05     # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+Optimal_Tol: 1.0e-05    # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+TimeLimit: 1.0e23       # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
+Pre_Solve: on           # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
+Method: ipm             # HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
+ipm_optimality_tolerance: 1e-04
 
 # run the crossover routine for ipx
 # [type: string, advanced: "on", range: {"off", "on"}, default: "off"]

diff --git a/test/PiecewiseFuel_CO2/highs_settings.yml b/test/PiecewiseFuel_CO2/highs_settings.yml
@@ -1,13 +1,7 @@
 # HiGHS Solver Parameters
 # Common solver settings
-Feasib_Tol: 1.0e-05        # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-Optimal_Tol: 1.0e-05       # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-TimeLimit: 1.0e23             # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
-Pre_Solve: choose          # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
-Method: choose             #HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
-
-#highs-specific solver settings
-
-# run the crossover routine for ipx
-# [type: string, advanced: "on", range: {"off", "on"}, default: "off"]
-run_crossover: "off"
+Feasib_Tol: 1.0e-05     # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+Optimal_Tol: 1.0e-05    # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+TimeLimit: 1.0e23       # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
+Pre_Solve: on           # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
+Method: simplex         # HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
diff --git a/test/ThreeZones/highs_settings.yml b/test/ThreeZones/highs_settings.yml
@@ -1,10 +1,10 @@
 # HiGHS Solver Parameters
 # Common solver settings
-Feasib_Tol: 1.0e-05        # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-Optimal_Tol: 1.0e-05       # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-TimeLimit: 1.0e23             # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
-Pre_Solve: choose          # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
-Method: ipm           #HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
+Feasib_Tol: 1.0e-05    # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+TimeLimit: 1.0e23      # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
+Pre_Solve: on          # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
+Method: ipm            # HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
+ipm_optimality_tolerance: 1e-04
 
 # run the crossover routine for ipx
 # [type: string, advanced: "on", range: {"off", "on"}, default: "off"]

diff --git a/test/VREStor/highs_settings.yml b/test/VREStor/highs_settings.yml
@@ -1,13 +1,8 @@
 # HiGHS Solver Parameters
 # Common solver settings
-Feasib_Tol: 1.0e-05        # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-Optimal_Tol: 1.0e-05       # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
-TimeLimit: 1.0e23             # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
-Pre_Solve: choose          # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
-Method: choose             #HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
+Feasib_Tol: 1.0e-05   # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+Optimal_Tol: 1.0e-05  # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07]
+TimeLimit: 1.0e23     # Time limit # [type: double, advanced: false, range: [0, inf], default: inf]
+Pre_Solve: on         # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"]
+Method: simplex       # HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"]
 
-#highs-specific solver settings
-
-# run the crossover routine for ipx
-# [type: string, advanced: "on", range: {"off", "on"}, default: "off"]
-run_crossover: "on"
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -17,12 +17,8 @@ end
     include("expression_manipulation_test.jl")
 end
 
-@testset "Utilities" begin
-    include("test_utilities.jl")
-end
-
 # Test GenX modules
-@testset verbose=true "GenX modules" begin
+@testset verbose = true "GenX modules" begin
     @testset "Three zones" begin
         include("test_threezones.jl")
     end
@@ -33,7 +29,7 @@ end
 
     @testset "PiecewiseFuel CO2" begin
         include("test_piecewisefuel_CO2.jl")
-    end 
+    end
 
     @testset "VRE and storage" begin
         include("test_VREStor.jl")

diff --git a/test/test_VREStor.jl b/test/test_VREStor.jl
@@ -25,7 +25,7 @@ genx_setup = Dict(
     "ParameterScale" => 1,
     "Trans_Loss_Segments" => 1,
     "CapacityReserveMargin" => 1,
-    "ModelingtoGenerateAlternativeSlack: 0" =>1,
+    "ModelingtoGenerateAlternativeSlack: 0" => 1,
     "Solver" => "HiGHS",
     "ModelingToGenerateAlternatives" => 0,
     "WriteShadowPrices" => 1,
@@ -37,17 +37,16 @@ genx_setup = Dict(
 EP, _, _ = redirect_stdout(devnull) do
     run_genx_case_testing(test_path, genx_setup)
 end
-
 obj_test = objective_value(EP)
-optimal_tol = get_attribute(EP, "dual_feasibility_tolerance")
-
-# Round the objective value to the same number of digits as the tolerance
-obj_test = round_objfromtol!(obj_test, optimal_tol)
+optimal_tol_rel = get_attribute(EP, "dual_feasibility_tolerance")
+optimal_tol = optimal_tol_rel * obj_test  # Convert to absolute tolerance
 
 # Test the objective value
-test_result = @test obj_test ≈ obj_true atol=optimal_tol
+test_result = @test obj_test ≈ obj_true atol = optimal_tol
 
-# Add the results to the test log
+# Round objective value and tolerance. Write to test log.
+obj_test = round_from_tol!(obj_test, optimal_tol)
+optimal_tol = round_from_tol!(optimal_tol, optimal_tol)
 write_testlog(test_path, obj_test, optimal_tol, test_result)
 
 end # module TestVREStor

diff --git a/test/test_electrolyzer.jl b/test/test_electrolyzer.jl
@@ -4,7 +4,7 @@ using Test
 
 include(joinpath(@__DIR__, "utilities.jl"))
 
-obj_true = 6946.9819
+obj_true = 6946.9618
 test_path = "Electrolyzer"
 
 # Define test inputs
@@ -34,17 +34,16 @@ genx_setup = Dict(
 EP, _, _ = redirect_stdout(devnull) do
     run_genx_case_testing(test_path, genx_setup)
 end
-
 obj_test = objective_value(EP)
-optimal_tol = get_attribute(EP, "dual_feasibility_tolerance")
-
-# Round the objective value to the same number of digits as the tolerance
-obj_test = round_objfromtol!(obj_test, optimal_tol)
+optimal_tol_rel = get_attribute(EP, "ipm_optimality_tolerance")
+optimal_tol = optimal_tol_rel * obj_test  # Convert to absolute tolerance
 
 # Test the objective value
-test_result = @test obj_test ≈ obj_true atol=optimal_tol
+test_result = @test obj_test ≈ obj_true atol = optimal_tol
 
-# Add the results to the test log
+# Round objective value and tolerance. Write to test log.
+obj_test = round_from_tol!(obj_test, optimal_tol)
+optimal_tol = round_from_tol!(optimal_tol, optimal_tol)
 write_testlog(test_path, obj_test, optimal_tol, test_result)
 
 end # module TestElectrolyzer
diff --git a/test/test_methodofmorris.jl b/test/test_methodofmorris.jl
@@ -34,24 +34,21 @@ genx_setup = Dict(
 )
 
 # Run the case and check if the model was built
-built = false
+built_and_run = false
 try
     Morris_range = redirect_stdout(devnull) do
         EP, inputs, OPTIMIZER = run_genx_case_testing(test_path, genx_setup)
         morris(EP, test_path, genx_setup, inputs, test_path, OPTIMIZER)
         rm(joinpath(@__DIR__, test_path, "morris.csv"))
     end
     #TODO: test Morris range 
-    built = true
+    global built_and_run = true
 
 catch BoundsError
 end
 
-@static if VERSION ≥ VersionNumber(1, 7)
-    test_result = Test.@test built broken = true
-else
-    test_result = built ? "Test Passed" : "Test Failed"
-end
+# Test if the 
+test_result = Test.@test built_and_run
 
 # Add the results to the test log
 write_testlog(test_path, "Build and Run", test_result)

diff --git a/test/test_multistage.jl b/test/test_multistage.jl
@@ -4,13 +4,13 @@ using Test
 
 include(joinpath(@__DIR__, "utilities.jl"))
 
-obj_true = [79734.80032, 41630.03494, 27855.20632]
+obj_true = [79734.80032, 41630.03494, 27855.20631]
 test_path = "MultiStage"
 
 # Define test inputs
 multistage_setup = Dict(
-    "NumStages" => 3 ,
-    "StageLengths" => [10,10,10],
+    "NumStages" => 3,
+    "StageLengths" => [10, 10, 10],
     "WACC" => 0.045,
     "ConvergenceTolerance" => 0.01,
     "Myopic" => 0,
@@ -47,17 +47,16 @@ genx_setup = Dict(
 EP, _, _ = redirect_stdout(devnull) do
     run_genx_case_testing(test_path, genx_setup)
 end
-
-obj_test = objective_value.([EP[i] for i in 1:3])
-optimal_tol = get_attribute.([EP[i] for i in 1:3], "dual_feasibility_tolerance")
-
-# Round the objective value to the same number of digits as the tolerance
-obj_test = round_objfromtol!.(obj_test, optimal_tol)
+obj_test = objective_value.(EP[i] for i in 1:multistage_setup["NumStages"])
+optimal_tol_rel = get_attribute.((EP[i] for i in 1:multistage_setup["NumStages"]), "ipm_optimality_tolerance")
+optimal_tol = optimal_tol_rel .* obj_test  # Convert to absolute tolerance
 
 # Test the objective value
 test_result = @test all(obj_true .- optimal_tol .<= obj_test .<= obj_true .+ optimal_tol)
 
-# Add the results to the test log
+# Round objective value and tolerance. Write to test log.
+obj_test = round_from_tol!.(obj_test, optimal_tol)
+optimal_tol = round_from_tol!.(optimal_tol, optimal_tol)
 write_testlog(test_path, obj_test, optimal_tol, test_result)
 
 end # module TestMultiStage
diff --git a/test/test_piecewisefuel_CO2.jl b/test/test_piecewisefuel_CO2.jl
@@ -30,17 +30,16 @@ genx_setup = Dict(
 EP, _, _ = redirect_stdout(devnull) do
     run_genx_case_testing(test_path, genx_setup)
 end
-
 obj_test = objective_value(EP)
-optimal_tol = get_attribute(EP, "dual_feasibility_tolerance")
-
-# Round the objective value to the same number of digits as the tolerance
-obj_test = round_objfromtol!(obj_test, optimal_tol)
+optimal_tol_rel = get_attribute(EP, "dual_feasibility_tolerance")
+optimal_tol = optimal_tol_rel * obj_test  # Convert to absolute tolerance
 
 # Test the objective value
-test_result = @test obj_test ≈ obj_true atol=optimal_tol
+test_result = @test obj_test ≈ obj_true atol = optimal_tol
 
-# Add the results to the test log
+# Round objective value and tolerance. Write to test log.
+obj_test = round_from_tol!(obj_test, optimal_tol)
+optimal_tol = round_from_tol!(optimal_tol, optimal_tol)
 write_testlog(test_path, obj_test, optimal_tol, test_result)
 
 end # module TestPiecewiseFuelCO2

diff --git a/test/test_threezones.jl b/test/test_threezones.jl
@@ -18,8 +18,8 @@ genx_setup = Dict(
     "CapacityReserveMargin" => 0,
     "CO2Cap" => 2,
     "StorageLosses" => 1,
-    "MinCapReq" => 1 ,
-    "MaxCapReq" => 0 ,
+    "MinCapReq" => 1,
+    "MaxCapReq" => 0,
     "Solver" => "HiGHS",
     "ParameterScale" => 1,
     "WriteShadowPrices" => 1,
@@ -39,17 +39,16 @@ genx_setup = Dict(
 EP, _, _ = redirect_stdout(devnull) do
     run_genx_case_testing(test_path, genx_setup)
 end
-
 obj_test = objective_value(EP)
-optimal_tol = get_attribute(EP, "dual_feasibility_tolerance")
-
-# Round the objective value to the same number of digits as the tolerance
-obj_test = round_objfromtol!(obj_test, optimal_tol)
+optimal_tol_rel = get_attribute(EP, "ipm_optimality_tolerance")
+optimal_tol = optimal_tol_rel * obj_test  # Convert to absolute tolerance
 
 # Test the objective value
-test_result = @test obj_test ≈ obj_true atol=optimal_tol
+test_result = @test obj_test ≈ obj_true atol = optimal_tol
 
-# Add the results to the test log
+# Round objective value and tolerance. Write to test log.
+obj_test = round_from_tol!(obj_test, optimal_tol)
+optimal_tol = round_from_tol!(optimal_tol, optimal_tol)
 write_testlog(test_path, obj_test, optimal_tol, test_result)
 
 end # module TestThreeZones
diff --git a/test/test_utilities.jl b/test/test_utilities.jl
diff --git a/test/utilities.jl b/test/utilities.jl
@@ -95,7 +95,7 @@ function get_exponent_sciform(number::Real)
     return number == 0.0 ? 0 : Int(floor(log10(abs(number))))
 end
 
-function round_objfromtol!(obj::Real, tol::Real)
+function round_from_tol!(obj::Real, tol::Real)
     # Round the objective value to the same number of digits as the tolerance
-    return round(obj, digits=(-1)*get_exponent_sciform(tol))
+    return round(obj, digits=(-1) * get_exponent_sciform(tol))
 end