Skip to content

Commit 98608b8

Browse files
odowodow
authored
Add the skeleton of the new MOI.Test submodule (#1408)
1 parent f304eb3 commit 98608b8

File tree

7 files changed

+523
-176
lines changed

7 files changed

+523
-176
lines changed

‎docs/make.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ makedocs(
6363
],
6464
"Test" => [
6565
"Overview" => "submodules/Test/overview.md",
66+
"API Reference" => "submodules/Test/reference.md",
6667
],
6768
],
6869
],

‎docs/src/submodules/Test/overview.md‎

Lines changed: 161 additions & 174 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ so that all solvers can benefit.
2020
## How to test a solver
2121

2222
The skeleton below can be used for the wrapper test file of a solver named
23-
`FooBar`. Remove unnecessary tests as appropriate, for example tests for
24-
features that the solver does not support (tests are not skipped depending
25-
on the value of `supports`).
23+
`FooBar`.
2624

2725
```julia
2826
# ============================ /test/MOI_wrapper.jl ============================
@@ -34,219 +32,208 @@ using Test
3432

3533
const MOI = MathOptInterface
3634

37-
const OPTIMIZER_CONSTRUCTOR = MOI.OptimizerWithAttributes(
38-
FooBar.Optimizer,
39-
MOI.Silent() => true
35+
const OPTIMIZER = MOI.instantiate(
36+
MOI.OptimizerWithAttributes(FooBar.Optimizer, MOI.Silent() => true),
4037
)
41-
const OPTIMIZER = MOI.instantiate(OPTIMIZER_CONSTRUCTOR)
4238

4339
const BRIDGED = MOI.instantiate(
44-
OPTIMIZER_CONSTRUCTOR, with_bridge_type = Float64
40+
MOI.OptimizerWithAttributes(FooBar.Optimizer, MOI.Silent() => true),
41+
with_bridge_type = Float64,
4542
)
46-
const CONFIG = MOI.DeprecatedTest.Config(
43+
44+
# See the docstring of MOI.Test.Config for other arguments.
45+
const CONFIG = MOI.Test.Config(
4746
# Modify tolerances as necessary.
4847
atol = 1e-6,
4948
rtol = 1e-6,
50-
# Set false if dual solutions are not generated
51-
duals = true,
52-
# Set false if infeasibility certificates are not generated
53-
infeas_certificates = true,
5449
# Use MOI.LOCALLY_SOLVED for local solvers.
5550
optimal_status = MOI.OPTIMAL,
56-
# Set true if basis information is available
57-
basis = false,
5851
)
5952

60-
function test_SolverName()
61-
@test MOI.get(OPTIMIZER, MOI.SolverName()) == "FooBar"
62-
end
53+
"""
54+
runtests()
6355
64-
function test_supports_incremental_interface()
65-
@test MOI.supports_incremental_interface(OPTIMIZER, false)
66-
# Use `@test !...` if names are not supported
67-
@test MOI.supports_incremental_interface(OPTIMIZER, true)
56+
This function runs all functions in the this Module starting with `test_`.
57+
"""
58+
function runtests()
59+
for name in names(@__MODULE__; all = true)
60+
if startswith("$(name)", "test_")
61+
@testset "$(name)" begin
62+
getfield(@__MODULE__, name)()
63+
end
64+
end
65+
end
6866
end
6967

70-
function test_unittest()
71-
# Test all the functions included in dictionary `MOI.DeprecatedTest.unittests`,
72-
# except functions "number_threads" and "solve_qcp_edge_cases."
73-
MOI.DeprecatedTest.unittest(
68+
"""
69+
test_runtests()
70+
71+
This function runs all the tests in MathOptInterface.Test.
72+
73+
Pass arguments to `exclude` to skip tests for functionality that is not
74+
implemented or that your solver doesn't support.
75+
"""
76+
function test_runtests()
77+
MOI.Test.runtests(
7478
BRIDGED,
7579
CONFIG,
76-
["number_threads", "solve_qcp_edge_cases"]
80+
exclude = [
81+
"test_attribute_NumberOfThreads",
82+
"test_quadratic_",
83+
]
7784
)
85+
return
7886
end
7987

80-
function test_modification()
81-
MOI.DeprecatedTest.modificationtest(BRIDGED, CONFIG)
82-
end
83-
84-
function test_contlinear()
85-
MOI.DeprecatedTest.contlineartest(BRIDGED, CONFIG)
86-
end
88+
"""
89+
test_SolverName()
8790
88-
function test_contquadratictest()
89-
MOI.DeprecatedTest.contquadratictest(OPTIMIZER, CONFIG)
91+
You can also write new tests for solver-specific functionality. Write each new
92+
test as a function with a name beginning with `test_`.
93+
"""
94+
function test_SolverName()
95+
@test MOI.get(FooBar.Optimizer(), MOI.SolverName()) == "FooBar"
96+
return
9097
end
9198

92-
function test_contconic()
93-
MOI.DeprecatedTest.contlineartest(BRIDGED, CONFIG)
94-
end
99+
end # module TestFooBar
95100

96-
function test_intconic()
97-
MOI.DeprecatedTest.intconictest(BRIDGED, CONFIG)
98-
end
101+
# This line at tne end of the file runs all the tests!
102+
TestFooBar.runtests()
103+
```
99104

100-
function test_default_objective_test()
101-
MOI.DeprecatedTest.default_objective_test(OPTIMIZER)
102-
end
105+
Then modify your `runtests.jl` file to include the `MOI_wrapper.jl` file:
106+
```julia
107+
# ============================ /test/runtests.jl ============================
103108

104-
function test_default_status_test()
105-
MOI.DeprecatedTest.default_status_test(OPTIMIZER)
106-
end
109+
using Test
107110

108-
function test_nametest()
109-
MOI.DeprecatedTest.nametest(OPTIMIZER)
111+
@testset "MOI" begin
112+
include("test/MOI_wrapper.jl")
110113
end
114+
```
111115

112-
function test_validtest()
113-
MOI.DeprecatedTest.validtest(OPTIMIZER)
114-
end
116+
!!! info
117+
The optimizer `BRIDGED` constructed with [`instantiate`](@ref)
118+
automatically bridges constraints that are not supported by `OPTIMIZER`
119+
using the bridges listed in [Bridges](@ref). It is recommended for an
120+
implementation of MOI to only support constraints that are natively
121+
supported by the solver and let bridges transform the constraint to the
122+
appropriate form. For this reason it is expected that tests may not pass if
123+
`OPTIMIZER` is used instead of `BRIDGED`.
115124

116-
function test_emptytest()
117-
MOI.DeprecatedTest.emptytest(OPTIMIZER)
118-
end
125+
## How to add a test
119126

120-
function test_orderedindicestest()
121-
MOI.DeprecatedTest.orderedindicestest(OPTIMIZER)
122-
end
127+
To detect bugs in solvers, we add new tests to `MOI.Test`.
123128

124-
function test_scalar_function_constant_not_zero()
125-
MOI.DeprecatedTest.scalar_function_constant_not_zero(OPTIMIZER)
126-
end
129+
As an example, ECOS errored calling [`optimize!`](@ref) twice in a row. (See
130+
[ECOS.jl PR #72](https://github.com/jump-dev/ECOS.jl/pull/72).) We could add a
131+
test to ECOS.jl, but that would only stop us from re-introducing the bug to
132+
ECOS.jl in the future, but it would not catch other solvers in the ecosystem
133+
with the same bug! Instead, if we add a test to `MOI.Test`, then all solvers
134+
will also check that they handle a double optimize call!
127135

128-
# This function runs all functions in this module starting with `test_`.
129-
function runtests()
130-
for name in names(@__MODULE__; all = true)
131-
if startswith("$(name)", "test_")
132-
@testset "$(name)" begin
133-
getfield(@__MODULE__, name)()
134-
end
135-
end
136-
end
137-
end
136+
For this test, we care about correctness, rather than performance. therefore, we
137+
don't expect solvers to efficiently decide that they have already solved the
138+
problem, only that calling [`optimize!`](@ref) twice doesn't throw an error or
139+
give the wrong answer.
138140

139-
end # module TestFooBar
141+
**Step 1**
140142

141-
TestFooBar.runtests()
143+
Install the `MathOptInterface` julia package in `dev` mode
144+
([ref](https://julialang.github.io/Pkg.jl/v1/managing-packages/#developing-1)):
145+
```julia
146+
julia> ]
147+
(@v1.6) pkg> dev MathOptInterface
142148
```
143149

144-
Test functions like `MOI.DeprecatedTest.unittest` and `MOI.DeprecatedTest.modificationtest` are
145-
wrappers around corresponding dictionaries `MOI.DeprecatedTest.unittests` and
146-
`MOI.DeprecatedTest.modificationtests`. Exclude tests by passing a vector of strings
147-
corresponding to the test keys you want to exclude as the third positional
148-
argument to the test function.
150+
**Step 2**
149151

150-
!!! tip
151-
Print a list of all keys using `println.(keys(MOI.DeprecatedTest.unittests))`
152-
153-
The optimizer `BRIDGED` constructed with [`instantiate`](@ref)
154-
automatically bridges constraints that are not supported by `OPTIMIZER`
155-
using the bridges listed in [Bridges](@ref). It is recommended for an
156-
implementation of MOI to only support constraints that are natively supported
157-
by the solver and let bridges transform the constraint to the appropriate form.
158-
For this reason it is expected that tests may not pass if `OPTIMIZER` is used
159-
instead of `BRIDGED`.
160-
161-
To test that a specific problem can be solved without bridges, a specific test
162-
can be added with `OPTIMIZER` instead of `BRIDGED`. For example:
152+
From here on, proceed with making the following changes in the
153+
`~/.julia/dev/MathOptInterface` folder (or equivalent `dev` path on your
154+
machine).
155+
156+
**Step 3**
157+
158+
Since the double-optimize error involves solving an optimization problem,
159+
add a new test to [src/Test/UnitTests/solve.jl](https://github.com/jump-dev/MathOptInterface.jl/blob/master/src/Test/UnitTests/solve.jl).
160+
161+
The test should be something like
163162
```julia
164-
function test_interval_constraints()
165-
MOI.DeprecatedTest.linear10test(OPTIMIZER, CONFIG)
163+
"""
164+
test_unit_optimize!_twice(model::MOI.ModelLike, config::Config)
165+
166+
Test that calling `MOI.optimize!` twice does not error.
167+
168+
This problem was first detected in ECOS.jl PR#72:
169+
https://github.com/jump-dev/ECOS.jl/pull/72
170+
"""
171+
function test_unit_optimize!_twice(
172+
model::MOI.ModelLike,
173+
config::Config{T},
174+
) where {T}
175+
if !config.supports_optimize
176+
# Use `config` to modify the behavior of the tests. Since this test is
177+
# concerned with `optimize!`, we should skip the test if
178+
# `config.solve == false`.
179+
return
180+
end
181+
# If needed, you can test that the model is empty at the start of the test.
182+
# You can assume that this will be the case for tests run via `runtests`.
183+
# User's calling tests individually need to call `MOI.empty!` themselves.
184+
@test MOI.is_empty(model)
185+
# Create a simple model. Try to make this as simple as possible so that the
186+
# majority of solvers can run the test.
187+
x = MOI.add_variable(model)
188+
MOI.add_constraint(model, MOI.SingleVariable(x), MOI.GreaterThan(one(T)))
189+
MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE)
190+
MOI.set(
191+
model,
192+
MOI.ObjectiveFunction{MOI.SingleVariable}(),
193+
MOI.SingleVariable(x),
194+
)
195+
# The main component of the test: does calling `optimize!` twice error?
196+
MOI.optimize!(model)
197+
MOI.optimize!(model)
198+
# Check we have a solution.
199+
@test MOI.get(model, MOI.TerminationStatus()) == MOI.OPTIMAL
200+
# There is a three-argument version of `Base.isapprox` for checking
201+
# approximate equality based on the tolerances defined in `config`:
202+
@test isapprox(MOI.get(model, MOI.VariablePrimal(), x), one(T), config)
203+
# For code-style, these tests should always `return` `nothing`.
204+
return
166205
end
167206
```
168-
checks that `OPTIMIZER` implements support for
169-
[`ScalarAffineFunction`](@ref)-in-[`Interval`](@ref).
170-
171-
## How to add a test
172207

173-
To give an example, ECOS errored calling [`optimize!`](@ref) twice in a row.
174-
(See [ECOS.jl PR #72](https://github.com/jump-dev/ECOS.jl/pull/72).)
208+
!!! info
209+
Make sure the function is agnoistic to the number type `T`! Don't assume it
210+
is a `Float64` capable solver!
175211

176-
We could add a test to ECOS.jl, but that would only stop us from re-introducing
177-
the bug to ECOS.jl in the future.
212+
We also need to write a test for the test. Place this function immediately below
213+
the test you just wrote in the same file:
214+
```julia
215+
function setup_test(
216+
::typeof(test_unit_optimize!_twice),
217+
model::MOI.Utilities.MockOptimizer,
218+
::Config,
219+
)
220+
MOI.Utilities.set_mock_optimize!(
221+
model,
222+
(mock::MOI.Utilities.MockOptimizer) -> MOIU.mock_optimize!(
223+
mock,
224+
MOI.OPTIMAL,
225+
(MOI.FEASIBLE_POINT, [1.0]),
226+
),
227+
)
228+
return
229+
end
230+
```
178231

179-
Instead, if we add a test to `MOI.DeprecatedTest`, then all solvers will also check that
180-
they handle a double optimize call!
232+
**Step 6**
181233

182-
For this test, we care about correctness, rather than performance. therefore, we
183-
don't expect solvers to efficiently decide that they have already solved the
184-
problem, only that calling [`optimize!`](@ref) twice doesn't throw an error or
185-
give the wrong answer.
234+
Commit the changes to git from `~/.julia/dev/MathOptInterface` and
235+
submit the PR for review.
186236

187-
To resolve this issue, follow these steps (tested on Julia v1.5):
188-
189-
1. Install the `MathOptInterface` julia package in `dev` mode
190-
([ref](https://julialang.github.io/Pkg.jl/v1/managing-packages/#developing-1)):
191-
```julia
192-
julia> ]
193-
(@v1.5) pkg> dev ECOS
194-
(@v1.5) pkg> dev MathOptInterface
195-
```
196-
2. From here on, proceed with making the following changes in the
197-
`~/.julia/dev/MathOptInterface` folder (or equivalent `dev` path on your
198-
machine).
199-
3. Since the double-optimize error involves solving an optimization problem,
200-
add a new test to [src/Test/UnitTests/solve.jl](https://github.com/jump-dev/MathOptInterface.jl/blob/master/src/Test/UnitTests/solve.jl).
201-
The test should be something like
202-
```julia
203-
function solve_twice(model::MOI.ModelLike, config::Config)
204-
MOI.empty!(model)
205-
x = MOI.add_variable(model)
206-
c = MOI.add_constraint(model, MOI.SingleVariable(x), MOI.GreaterThan(1.0))
207-
MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE)
208-
MOI.set(model, MOI.ObjectiveFunction{MOI.SingleVariable}(), MOI.SingleVariable(x))
209-
if config.solve
210-
MOI.optimize!(model)
211-
MOI.optimize!(model)
212-
MOI.get(model, MOI.TerminationStatus()) == MOI.OPTIMAL
213-
MOI.get(model, MOI.VariablePrimal(), x) == 1.0
214-
end
215-
end
216-
unittests["solve_twice"] = solve_twice
217-
```
218-
2. Add a test for the test you just wrote. (We test the tests!)
219-
a. Add the name of the test (`"solve_twice"`) to the end of the array in
220-
`MOI.DeprecatedTest.unittest(...)` ([link](https://github.com/jump-dev/MathOptInterface.jl/blob/7543afe4b5151cf36bbd18181c1bb5c83266ae2f/test/Test/unit.jl#L51-L52)).
221-
b. Add a test for the test towards the end of the "Unit Tests" test set
222-
([link](https://github.com/jump-dev/MathOptInterface.jl/blob/7543afe4b5151cf36bbd18181c1bb5c83266ae2f/test/Test/unit.jl#L394)).
223-
The test should look something like
224-
```julia
225-
@testset "solve_twice" begin
226-
MOI.Utilities.set_mock_optimize!(mock,
227-
(mock::MOI.Utilities.MockOptimizer) -> MOI.Utilities.mock_optimize!(
228-
mock,
229-
MOI.OPTIMAL,
230-
(MOI.FEASIBLE_POINT, [1.0]),
231-
),
232-
(mock::MOI.Utilities.MockOptimizer) -> MOI.Utilities.mock_optimize!(
233-
mock,
234-
MOI.OPTIMAL,
235-
(MOI.FEASIBLE_POINT, [1.0]),
236-
)
237-
)
238-
MOI.DeprecatedTest.solve_twice(mock, config)
239-
end
240-
```
241-
In the above `mock` is a `MOI.Utilities.MockOptimizer` that is defined
242-
tesearlier in the file. In this test, `MOI.Utilities.set_mock_optimize!` loads
243-
`mock` with two results. Each says that the
244-
[`TerminationStatus`](@ref) is `MOI.OPTIMAL`, that the
245-
[`PrimalStatus`](@ref) is `MOI.FEASIBLE_POINT`, and that there is one
246-
variable with a `MOI.VariableValue` or `1.0`
247-
3. Run the tests:
248-
```julia
249-
(@v1.5) pkg> test ECOS
250-
```
251-
4. Finally, commit the changes to git from `~/.julia/dev/MathOptInterface` and
252-
submit the PR for review.
237+
!!! tip
238+
If you need help writing a test, [open an issue on GitHub](https://github.com/jump-dev/MathOptInterface.jl/issues/new),
239+
or ask the [Developer Chatroom](https://gitter.im/JuliaOpt/JuMP.jl)

0 commit comments

Comments
 (0)