From 136606b0e3d470b221885326af6e8e699662ee34 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Mon, 4 Mar 2024 19:03:44 -0500 Subject: [PATCH 01/11] Move code from existing PR --- lib/OptimizationManopt/Project.toml | 10 + .../src/OptimizationManopt.jl | 306 ++++++++++++++++++ lib/OptimizationManopt/test/runtests.jl | 101 ++++++ 3 files changed, 417 insertions(+) create mode 100644 lib/OptimizationManopt/Project.toml create mode 100644 lib/OptimizationManopt/src/OptimizationManopt.jl create mode 100644 lib/OptimizationManopt/test/runtests.jl diff --git a/lib/OptimizationManopt/Project.toml b/lib/OptimizationManopt/Project.toml new file mode 100644 index 000000000..821a4cfc1 --- /dev/null +++ b/lib/OptimizationManopt/Project.toml @@ -0,0 +1,10 @@ +name = "OptimizationManopt" +uuid = "e57b7fff-7ee7-4550-b4f0-90e9476e9fb6" +authors = ["Mateusz Baran "] +version = "0.1.0" + +[deps] +Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e" +ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb" +Manopt = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5" +Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" \ No newline at end of file diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl new file mode 100644 index 000000000..c53db17f1 --- /dev/null +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -0,0 +1,306 @@ +module OptimizationManopt + +using Optimization, Manopt, ManifoldsBase + +""" + abstract type AbstractManoptOptimizer end + +A Manopt solver without things specified by a call to `solve` (stopping criteria) and +internal state. +""" +abstract type AbstractManoptOptimizer end + +function stopping_criterion_to_kwarg(stopping_criterion::Nothing) + return NamedTuple() +end +function stopping_criterion_to_kwarg(stopping_criterion::StoppingCriterion) + return (; stopping_criterion = stopping_criterion) +end + +## gradient descent + +struct GradientDescentOptimizer{ + Teval <: AbstractEvaluationType, + TM <: AbstractManifold, + TLS <: Linesearch + } <: AbstractManoptOptimizer + M::TM + stepsize::TLS +end + +function GradientDescentOptimizer(M::AbstractManifold; + eval::AbstractEvaluationType = MutatingEvaluation(), + stepsize::Stepsize = ArmijoLinesearch(M)) + GradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, stepsize) +end + +function call_manopt_optimizer(opt::GradientDescentOptimizer{Teval}, + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType + } + sckwarg = stopping_criterion_to_kwarg(stopping_criterion) + opts = gradient_descent(opt.M, + loss, + gradF, + x0; + return_options = true, + evaluation = Teval(), + stepsize = opt.stepsize, + sckwarg...) + # we unwrap DebugOptions here + minimizer = Manopt.get_solver_result(opts) + return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), + :who_knows +end + +## Nelder-Mead + +struct NelderMeadOptimizer{ + TM <: AbstractManifold, + Tpop <: AbstractVector + } <: AbstractManoptOptimizer + M::TM + initial_population::Tpop +end + +function NelderMeadOptimizer(M::AbstractManifold) + initial_population = [rand(M) for _ in 1:(manifold_dimension(M) + 1)] + return NelderMeadOptimizer{typeof(M), typeof(initial_population)}(M, initial_population) +end + +function call_manopt_optimizer(opt::NelderMeadOptimizer, + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) + sckwarg = stopping_criterion_to_kwarg(stopping_criterion) + + opts = NelderMead(opt.M, + loss, + opt.initial_population; + return_options = true, + sckwarg...) + minimizer = Manopt.get_solver_result(opts) + return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), + :who_knows +end + +## conjugate gradient descent + +struct ConjugateGradientDescentOptimizer{Teval <: AbstractEvaluationType, + TM <: AbstractManifold, TLS <: Stepsize} <: + AbstractManoptOptimizer + M::TM + stepsize::TLS +end + +function ConjugateGradientDescentOptimizer(M::AbstractManifold; + eval::AbstractEvaluationType = MutatingEvaluation(), + stepsize::Stepsize = ArmijoLinesearch(M)) + ConjugateGradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, + stepsize) +end + +function call_manopt_optimizer(opt::ConjugateGradientDescentOptimizer{Teval}, + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType + } + sckwarg = stopping_criterion_to_kwarg(stopping_criterion) + opts = conjugate_gradient_descent(opt.M, + loss, + gradF, + x0; + return_options = true, + evaluation = Teval(), + stepsize = opt.stepsize, + sckwarg...) + # we unwrap DebugOptions here + minimizer = Manopt.get_solver_result(opts) + return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), + :who_knows +end + +## particle swarm + +struct ParticleSwarmOptimizer{Teval <: AbstractEvaluationType, + TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, + Tinvretr <: AbstractInverseRetractionMethod, + Tvt <: AbstractVectorTransportMethod} <: + AbstractManoptOptimizer + M::TM + retraction_method::Tretr + inverse_retraction_method::Tinvretr + vector_transport_method::Tvt + population_size::Int +end + +function ParticleSwarmOptimizer(M::AbstractManifold; + eval::AbstractEvaluationType = MutatingEvaluation(), + population_size::Int = 100, + retraction_method::AbstractRetractionMethod = default_retraction_method(M), + inverse_retraction_method::AbstractInverseRetractionMethod = default_inverse_retraction_method(M), + vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M)) + ParticleSwarmOptimizer{typeof(eval), typeof(M), typeof(retraction_method), + typeof(inverse_retraction_method), + typeof(vector_transport_method)}(M, + retraction_method, + inverse_retraction_method, + vector_transport_method, + population_size) +end + +function call_manopt_optimizer(opt::ParticleSwarmOptimizer{Teval}, + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType + } + sckwarg = stopping_criterion_to_kwarg(stopping_criterion) + initial_population = vcat([x0], [rand(opt.M) for _ in 1:(opt.population_size - 1)]) + opts = particle_swarm(opt.M, + loss; + x0 = initial_population, + n = opt.population_size, + return_options = true, + retraction_method = opt.retraction_method, + inverse_retraction_method = opt.inverse_retraction_method, + vector_transport_method = opt.vector_transport_method, + sckwarg...) + # we unwrap DebugOptions here + minimizer = Manopt.get_solver_result(opts) + return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), + :who_knows +end + +## quasi Newton + +struct QuasiNewtonOptimizer{Teval <: AbstractEvaluationType, + TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, + Tvt <: AbstractVectorTransportMethod, TLS <: Stepsize} <: + AbstractManoptOptimizer + M::TM + retraction_method::Tretr + vector_transport_method::Tvt + stepsize::TLS +end + +function QuasiNewtonOptimizer(M::AbstractManifold; + eval::AbstractEvaluationType = MutatingEvaluation(), + retraction_method::AbstractRetractionMethod = default_retraction_method(M), + vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M), + stepsize = WolfePowellLinesearch(M; + retraction_method = retraction_method, + vector_transport_method = vector_transport_method, + linesearch_stopsize = 1e-12)) + QuasiNewtonOptimizer{typeof(eval), typeof(M), typeof(retraction_method), + typeof(vector_transport_method), typeof(stepsize)}(M, + retraction_method, + vector_transport_method, + stepsize) +end + +function call_manopt_optimizer(opt::QuasiNewtonOptimizer{Teval}, + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType + } + sckwarg = stopping_criterion_to_kwarg(stopping_criterion) + opts = quasi_Newton(opt.M, + loss, + gradF, + x0; + return_options = true, + evaluation = Teval(), + retraction_method = opt.retraction_method, + vector_transport_method = opt.vector_transport_method, + stepsize = opt.stepsize, + sckwarg...) + # we unwrap DebugOptions here + minimizer = Manopt.get_solver_result(opts) + return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), + :who_knows +end + +## Optimization.jl stuff + +function build_loss(f::OptimizationFunction, prob, cur) + function (::AbstractManifold, θ) + x = f.f(θ, prob.p, cur...) + __x = first(x) + return prob.sense === Optimization.MaxSense ? -__x : __x + end +end + +function build_gradF(f::OptimizationFunction{true}, prob, cur) + function (::AbstractManifold, G, θ) + X = f.grad(G, θ, cur...) + if prob.sense === Optimization.MaxSense + return -X # TODO: check + else + return X + end + end +end + +# TODO: +# 1) convert tolerances and other stopping criteria +# 2) return convergence information +# 3) add callbacks to Manopt.jl + +function SciMLBase.__solve(prob::OptimizationProblem, + opt::AbstractManoptOptimizer, + data = Optimization.DEFAULT_DATA; + callback = (args...) -> (false), + maxiters::Union{Number, Nothing} = nothing, + maxtime::Union{Number, Nothing} = nothing, + abstol::Union{Number, Nothing} = nothing, + reltol::Union{Number, Nothing} = nothing, + progress = false, + kwargs...) + local x, cur, state + + if data !== Optimization.DEFAULT_DATA + maxiters = length(data) + end + + cur, state = iterate(data) + + stopping_criterion = nothing + if maxiters !== nothing + stopping_criterion = StopAfterIteration(maxiters) + end + + maxiters = Optimization._check_and_convert_maxiters(maxiters) + maxtime = Optimization._check_and_convert_maxtime(maxtime) + + f = Optimization.instantiate_function(prob.f, prob.u0, prob.f.adtype, prob.p) + + _loss = build_loss(f, prob, cur) + + gradF = build_gradF(f, prob, cur) + + opt_res, opt_ret = call_manopt_optimizer(opt, _loss, gradF, prob.u0, stopping_criterion) + + return SciMLBase.build_solution(SciMLBase.DefaultOptimizationCache(prob.f, prob.p), + opt, + opt_res.minimizer, + prob.sense === Optimization.MaxSense ? + -opt_res.minimum : opt_res.minimum; + original = opt_res.options, + retcode = opt_ret) +end + +end # module OptimizationManopt \ No newline at end of file diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl new file mode 100644 index 000000000..3233abae4 --- /dev/null +++ b/lib/OptimizationManopt/test/runtests.jl @@ -0,0 +1,101 @@ +using OptimizationManopt +using Optimization +using Manifolds +using ForwardDiff +using Manopt +using Test +using SciMLBase + +rosenbrock(x, p) = (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2 + +function rosenbrock_grad!(storage, x, p) + storage[1] = -2.0 * (p[1] - x[1]) - 4.0 * p[2] * (x[2] - x[1]^2) * x[1] + storage[2] = 2.0 * p[2] * (x[2] - x[1]^2) +end + +R2 = Euclidean(2) + +@testset "Gradient descent" begin + x0 = zeros(2) + p = [1.0, 100.0] + + stepsize = Manopt.ArmijoLinesearch(R2) + opt = OptimizationManopt.GradientDescentOptimizer(R2, + stepsize = stepsize) + + optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) + prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p) + sol = Optimization.solve(prob_forwarddiff, opt) + @test sol.minimum < 0.2 + + optprob_grad = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!) + prob_grad = OptimizationProblem(optprob_grad, x0, p) + sol = Optimization.solve(prob_grad, opt) + @test sol.minimum < 0.2 +end + +@testset "Nelder-Mead" begin + x0 = zeros(2) + p = [1.0, 100.0] + + opt = OptimizationManopt.NelderMeadOptimizer(R2, [[1.0, 0.0], [0.0, 1.0], [0.0, 0.0]]) + + optprob = OptimizationFunction(rosenbrock) + prob = OptimizationProblem(optprob, x0, p) + + sol = Optimization.solve(prob, opt) + @test sol.minimum < 0.7 +end + +@testset "Conjugate gradient descent" begin + x0 = zeros(2) + p = [1.0, 100.0] + + stepsize = Manopt.ArmijoLinesearch(R2) + opt = OptimizationManopt.ConjugateGradientDescentOptimizer(R2, + stepsize = stepsize) + + optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) + prob = OptimizationProblem(optprob, x0, p) + + sol = Optimization.solve(prob, opt) + @test sol.minimum < 0.2 +end + +@testset "Quasi Newton" begin + x0 = zeros(2) + p = [1.0, 100.0] + + opt = OptimizationManopt.QuasiNewtonOptimizer(R2) + + optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) + prob = OptimizationProblem(optprob, x0, p) + + sol = Optimization.solve(prob, opt) + @test sol.minimum < 1e-16 +end + +@testset "Particle swarm" begin + x0 = zeros(2) + p = [1.0, 100.0] + + opt = OptimizationManopt.ParticleSwarmOptimizer(R2) + + optprob = OptimizationFunction(rosenbrock) + prob = OptimizationProblem(optprob, x0, p) + + sol = Optimization.solve(prob, opt) + @test sol.minimum < 0.1 +end + +@testset "Custom constraints" begin + cons(res, x, p) = (res .= [x[1]^2 + x[2]^2, x[1] * x[2]]) + + x0 = zeros(2) + p = [1.0, 100.0] + opt = OptimizationManopt.GradientDescentOptimizer(R2) + + optprob_cons = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!, cons = cons) + prob_cons = OptimizationProblem(optprob_cons, x0, p) + @test_throws SciMLBase.IncompatibleOptimizerError Optimization.solve(prob_cons, opt) +end \ No newline at end of file From 7af2b81bff403e815cecf889a8ca25f69de8bba0 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Wed, 6 Mar 2024 17:43:21 -0500 Subject: [PATCH 02/11] wip --- .../src/OptimizationManopt.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index c53db17f1..37d439ed5 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -29,7 +29,7 @@ struct GradientDescentOptimizer{ end function GradientDescentOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = MutatingEvaluation(), + eval::AbstractEvaluationType = Manopt.AllocatingEvaluation(), stepsize::Stepsize = ArmijoLinesearch(M)) GradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, stepsize) end @@ -99,7 +99,7 @@ struct ConjugateGradientDescentOptimizer{Teval <: AbstractEvaluationType, end function ConjugateGradientDescentOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = MutatingEvaluation(), + eval::AbstractEvaluationType = InplaceEvaluation(), stepsize::Stepsize = ArmijoLinesearch(M)) ConjugateGradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, stepsize) @@ -143,7 +143,7 @@ struct ParticleSwarmOptimizer{Teval <: AbstractEvaluationType, end function ParticleSwarmOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = MutatingEvaluation(), + eval::AbstractEvaluationType = InplaceEvaluation(), population_size::Int = 100, retraction_method::AbstractRetractionMethod = default_retraction_method(M), inverse_retraction_method::AbstractInverseRetractionMethod = default_inverse_retraction_method(M), @@ -195,7 +195,7 @@ struct QuasiNewtonOptimizer{Teval <: AbstractEvaluationType, end function QuasiNewtonOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = MutatingEvaluation(), + eval::AbstractEvaluationType = InplaceEvaluation(), retraction_method::AbstractRetractionMethod = default_retraction_method(M), vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M), stepsize = WolfePowellLinesearch(M; @@ -245,12 +245,13 @@ function build_loss(f::OptimizationFunction, prob, cur) end function build_gradF(f::OptimizationFunction{true}, prob, cur) - function (::AbstractManifold, G, θ) - X = f.grad(G, θ, cur...) + function (M::AbstractManifold, G, θ) + f.grad(G, θ, cur...) + G .= riemannian_gradient(M, θ, G) if prob.sense === Optimization.MaxSense - return -X # TODO: check + return -G # TODO: check else - return X + return G end end end From 8d7fbb5f53bb6c25c6cb21754e9f2200286cd6ba Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Wed, 13 Mar 2024 01:26:21 -0400 Subject: [PATCH 03/11] Euclidean tests pass, use riemannian gradient wrapper --- lib/OptimizationManopt/Project.toml | 1 + .../src/OptimizationManopt.jl | 32 ++++++++----------- lib/OptimizationManopt/test/runtests.jl | 12 +++---- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/lib/OptimizationManopt/Project.toml b/lib/OptimizationManopt/Project.toml index 821a4cfc1..baa8591c9 100644 --- a/lib/OptimizationManopt/Project.toml +++ b/lib/OptimizationManopt/Project.toml @@ -4,6 +4,7 @@ authors = ["Mateusz Baran "] version = "0.1.0" [deps] +ManifoldDiff = "af67fdf4-a580-4b9f-bbec-742ef357defd" Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e" ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb" Manopt = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5" diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index 37d439ed5..5e0c51887 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -1,6 +1,6 @@ module OptimizationManopt -using Optimization, Manopt, ManifoldsBase +using Optimization, Manopt, ManifoldsBase, ManifoldDiff """ abstract type AbstractManoptOptimizer end @@ -52,7 +52,7 @@ function call_manopt_optimizer(opt::GradientDescentOptimizer{Teval}, stepsize = opt.stepsize, sckwarg...) # we unwrap DebugOptions here - minimizer = Manopt.get_solver_result(opts) + minimizer = opts return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -61,15 +61,12 @@ end struct NelderMeadOptimizer{ TM <: AbstractManifold, - Tpop <: AbstractVector } <: AbstractManoptOptimizer M::TM - initial_population::Tpop end function NelderMeadOptimizer(M::AbstractManifold) - initial_population = [rand(M) for _ in 1:(manifold_dimension(M) + 1)] - return NelderMeadOptimizer{typeof(M), typeof(initial_population)}(M, initial_population) + return NelderMeadOptimizer{typeof(M)}(M) end function call_manopt_optimizer(opt::NelderMeadOptimizer, @@ -80,11 +77,10 @@ function call_manopt_optimizer(opt::NelderMeadOptimizer, sckwarg = stopping_criterion_to_kwarg(stopping_criterion) opts = NelderMead(opt.M, - loss, - opt.initial_population; + loss; return_options = true, sckwarg...) - minimizer = Manopt.get_solver_result(opts) + minimizer = opts return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -123,7 +119,7 @@ function call_manopt_optimizer(opt::ConjugateGradientDescentOptimizer{Teval}, stepsize = opt.stepsize, sckwarg...) # we unwrap DebugOptions here - minimizer = Manopt.get_solver_result(opts) + minimizer = opts return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -177,7 +173,7 @@ function call_manopt_optimizer(opt::ParticleSwarmOptimizer{Teval}, vector_transport_method = opt.vector_transport_method, sckwarg...) # we unwrap DebugOptions here - minimizer = Manopt.get_solver_result(opts) + minimizer = opts return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -229,7 +225,7 @@ function call_manopt_optimizer(opt::QuasiNewtonOptimizer{Teval}, stepsize = opt.stepsize, sckwarg...) # we unwrap DebugOptions here - minimizer = Manopt.get_solver_result(opts) + minimizer = opts return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -245,14 +241,14 @@ function build_loss(f::OptimizationFunction, prob, cur) end function build_gradF(f::OptimizationFunction{true}, prob, cur) - function (M::AbstractManifold, G, θ) + function g(M::AbstractManifold, G, θ) f.grad(G, θ, cur...) G .= riemannian_gradient(M, θ, G) - if prob.sense === Optimization.MaxSense - return -G # TODO: check - else - return G - end + end + function g(M::AbstractManifold, θ) + G = zero(θ) + f.grad(G, θ, cur...) + return riemannian_gradient(M, θ, G) end end diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index 3233abae4..cced97463 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -4,7 +4,7 @@ using Manifolds using ForwardDiff using Manopt using Test -using SciMLBase +using Optimization.SciMLBase rosenbrock(x, p) = (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2 @@ -17,7 +17,7 @@ R2 = Euclidean(2) @testset "Gradient descent" begin x0 = zeros(2) - p = [1.0, 100.0] + p = [1.0, 100.0] stepsize = Manopt.ArmijoLinesearch(R2) opt = OptimizationManopt.GradientDescentOptimizer(R2, @@ -38,13 +38,13 @@ end x0 = zeros(2) p = [1.0, 100.0] - opt = OptimizationManopt.NelderMeadOptimizer(R2, [[1.0, 0.0], [0.0, 1.0], [0.0, 0.0]]) + opt = OptimizationManopt.NelderMeadOptimizer(R2) optprob = OptimizationFunction(rosenbrock) prob = OptimizationProblem(optprob, x0, p) sol = Optimization.solve(prob, opt) - @test sol.minimum < 0.7 + @test sol.minimum < 1e-6 end @testset "Conjugate gradient descent" begin @@ -59,7 +59,7 @@ end prob = OptimizationProblem(optprob, x0, p) sol = Optimization.solve(prob, opt) - @test sol.minimum < 0.2 + @test sol.minimum < 0.5 end @testset "Quasi Newton" begin @@ -72,7 +72,7 @@ end prob = OptimizationProblem(optprob, x0, p) sol = Optimization.solve(prob, opt) - @test sol.minimum < 1e-16 + @test sol.minimum < 1e-14 end @testset "Particle swarm" begin From 3bf832723da3b9c08add61c767176a54f3235e57 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Wed, 13 Mar 2024 10:51:43 -0400 Subject: [PATCH 04/11] Pass return_state and add spd test --- .../src/OptimizationManopt.jl | 20 ++++++++--------- lib/OptimizationManopt/test/runtests.jl | 22 ++++++++++++++++++- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index 5e0c51887..9d2ee5d9b 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -47,12 +47,12 @@ function call_manopt_optimizer(opt::GradientDescentOptimizer{Teval}, loss, gradF, x0; - return_options = true, + return_state = true, evaluation = Teval(), stepsize = opt.stepsize, sckwarg...) # we unwrap DebugOptions here - minimizer = opts + minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -78,9 +78,9 @@ function call_manopt_optimizer(opt::NelderMeadOptimizer, opts = NelderMead(opt.M, loss; - return_options = true, + return_state = true, sckwarg...) - minimizer = opts + minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -114,12 +114,12 @@ function call_manopt_optimizer(opt::ConjugateGradientDescentOptimizer{Teval}, loss, gradF, x0; - return_options = true, + return_state = true, evaluation = Teval(), stepsize = opt.stepsize, sckwarg...) # we unwrap DebugOptions here - minimizer = opts + minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -167,13 +167,13 @@ function call_manopt_optimizer(opt::ParticleSwarmOptimizer{Teval}, loss; x0 = initial_population, n = opt.population_size, - return_options = true, + return_state = true, retraction_method = opt.retraction_method, inverse_retraction_method = opt.inverse_retraction_method, vector_transport_method = opt.vector_transport_method, sckwarg...) # we unwrap DebugOptions here - minimizer = opts + minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end @@ -218,14 +218,14 @@ function call_manopt_optimizer(opt::QuasiNewtonOptimizer{Teval}, loss, gradF, x0; - return_options = true, + return_state = true, evaluation = Teval(), retraction_method = opt.retraction_method, vector_transport_method = opt.vector_transport_method, stepsize = opt.stepsize, sckwarg...) # we unwrap DebugOptions here - minimizer = opts + minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), :who_knows end diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index cced97463..82e3b10ad 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -1,10 +1,11 @@ using OptimizationManopt using Optimization using Manifolds -using ForwardDiff +using ForwardDiff, Zygote, Enzyme using Manopt using Test using Optimization.SciMLBase +using LinearAlgebra rosenbrock(x, p) = (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2 @@ -98,4 +99,23 @@ end optprob_cons = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!, cons = cons) prob_cons = OptimizationProblem(optprob_cons, x0, p) @test_throws SciMLBase.IncompatibleOptimizerError Optimization.solve(prob_cons, opt) +end + +@testset "SPD Manifold" begin + M = SymmetricPositiveDefinite(5) + m = 100 + σ = 0.005 + q = Matrix{Float64}(I, 5, 5) .+ 2.0 + data2 = [exp(M, q, σ * rand(M; vector_at=q)) for i in 1:m]; + + f(M, x, p = nothing) = sum(distance(M, x, data2[i])^2 for i in 1:m) + f(x, p = nothing) = sum(distance(M, x, data2[i])^2 for i in 1:m) + + optf = OptimizationFunction(f, Optimization.AutoForwardDiff()) + prob = OptimizationProblem(optf, data2[1]) + + opt = OptimizationManopt.GradientDescentOptimizer(M) + @time sol = Optimization.solve(prob, opt) + + @test sol.u ≈ q end \ No newline at end of file From 7d791712bc0e4cee68236c314be44b7b4bcb4b2b Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Sat, 23 Mar 2024 21:49:32 -0400 Subject: [PATCH 05/11] Pass in manifold as kwarg to OptimizationProblem and throw error if not passes or doesn't match solver --- .../src/OptimizationManopt.jl | 9 ++++-- lib/OptimizationManopt/test/runtests.jl | 31 ++++++++++++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index 9d2ee5d9b..f4cdecd5f 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -65,9 +65,6 @@ struct NelderMeadOptimizer{ M::TM end -function NelderMeadOptimizer(M::AbstractManifold) - return NelderMeadOptimizer{typeof(M)}(M) -end function call_manopt_optimizer(opt::NelderMeadOptimizer, loss, @@ -269,6 +266,12 @@ function SciMLBase.__solve(prob::OptimizationProblem, kwargs...) local x, cur, state + manifold = haskey(prob.kwargs, :manifold) ? prob.kwargs[:manifold] : nothing + + if manifold === nothing || manifold !== opt.M + throw(ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`")) + end + if data !== Optimization.DEFAULT_DATA maxiters = length(data) end diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index 82e3b10ad..515c3ebe9 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -16,7 +16,7 @@ end R2 = Euclidean(2) -@testset "Gradient descent" begin +@testset "Error on no or mismatching manifolds" begin x0 = zeros(2) p = [1.0, 100.0] @@ -26,11 +26,28 @@ R2 = Euclidean(2) optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p) + @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve(prob_forwarddiff, opt) + + optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) + prob = OptimizationProblem(optprob, x0, p; manifold = SymmetricPositiveDefinite(5)) + @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve(prob, opt) +end + +@testset "Gradient descent" begin + x0 = zeros(2) + p = [1.0, 100.0] + + stepsize = Manopt.ArmijoLinesearch(R2) + opt = OptimizationManopt.GradientDescentOptimizer(R2, + stepsize = stepsize) + + optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) + prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p; manifold = R2) sol = Optimization.solve(prob_forwarddiff, opt) @test sol.minimum < 0.2 optprob_grad = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!) - prob_grad = OptimizationProblem(optprob_grad, x0, p) + prob_grad = OptimizationProblem(optprob_grad, x0, p; manifold = R2) sol = Optimization.solve(prob_grad, opt) @test sol.minimum < 0.2 end @@ -42,7 +59,7 @@ end opt = OptimizationManopt.NelderMeadOptimizer(R2) optprob = OptimizationFunction(rosenbrock) - prob = OptimizationProblem(optprob, x0, p) + prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = Optimization.solve(prob, opt) @test sol.minimum < 1e-6 @@ -57,7 +74,7 @@ end stepsize = stepsize) optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) - prob = OptimizationProblem(optprob, x0, p) + prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = Optimization.solve(prob, opt) @test sol.minimum < 0.5 @@ -70,7 +87,7 @@ end opt = OptimizationManopt.QuasiNewtonOptimizer(R2) optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) - prob = OptimizationProblem(optprob, x0, p) + prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = Optimization.solve(prob, opt) @test sol.minimum < 1e-14 @@ -83,7 +100,7 @@ end opt = OptimizationManopt.ParticleSwarmOptimizer(R2) optprob = OptimizationFunction(rosenbrock) - prob = OptimizationProblem(optprob, x0, p) + prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = Optimization.solve(prob, opt) @test sol.minimum < 0.1 @@ -112,7 +129,7 @@ end f(x, p = nothing) = sum(distance(M, x, data2[i])^2 for i in 1:m) optf = OptimizationFunction(f, Optimization.AutoForwardDiff()) - prob = OptimizationProblem(optf, data2[1]) + prob = OptimizationProblem(optf, data2[1]; manifold = M) opt = OptimizationManopt.GradientDescentOptimizer(M) @time sol = Optimization.solve(prob, opt) From feb9d0c8b5a509f77f34d0b32cf27f7d2ae74c36 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Sat, 23 Mar 2024 21:56:27 -0400 Subject: [PATCH 06/11] a test with enzyme --- lib/OptimizationManopt/test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index 515c3ebe9..1b56ed570 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -41,7 +41,7 @@ end opt = OptimizationManopt.GradientDescentOptimizer(R2, stepsize = stepsize) - optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) + optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoEnzyme()) prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p; manifold = R2) sol = Optimization.solve(prob_forwarddiff, opt) @test sol.minimum < 0.2 From bcbc649499353ea0c45d3e37642b7b0367ef39c1 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Sat, 23 Mar 2024 21:59:45 -0400 Subject: [PATCH 07/11] format --- .../src/OptimizationManopt.jl | 253 +++++++++--------- lib/OptimizationManopt/test/runtests.jl | 20 +- 2 files changed, 137 insertions(+), 136 deletions(-) diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index f4cdecd5f..982618615 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -20,113 +20,112 @@ end ## gradient descent struct GradientDescentOptimizer{ - Teval <: AbstractEvaluationType, - TM <: AbstractManifold, - TLS <: Linesearch - } <: AbstractManoptOptimizer + Teval <: AbstractEvaluationType, + TM <: AbstractManifold, + TLS <: Linesearch +} <: AbstractManoptOptimizer M::TM stepsize::TLS end function GradientDescentOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = Manopt.AllocatingEvaluation(), - stepsize::Stepsize = ArmijoLinesearch(M)) + eval::AbstractEvaluationType = Manopt.AllocatingEvaluation(), + stepsize::Stepsize = ArmijoLinesearch(M)) GradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, stepsize) end function call_manopt_optimizer(opt::GradientDescentOptimizer{Teval}, - loss, - gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType - } + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType +} sckwarg = stopping_criterion_to_kwarg(stopping_criterion) opts = gradient_descent(opt.M, - loss, - gradF, - x0; - return_state = true, - evaluation = Teval(), - stepsize = opt.stepsize, - sckwarg...) + loss, + gradF, + x0; + return_state = true, + evaluation = Teval(), + stepsize = opt.stepsize, + sckwarg...) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + :who_knows end ## Nelder-Mead struct NelderMeadOptimizer{ - TM <: AbstractManifold, - } <: AbstractManoptOptimizer + TM <: AbstractManifold, +} <: AbstractManoptOptimizer M::TM end - function call_manopt_optimizer(opt::NelderMeadOptimizer, - loss, - gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) sckwarg = stopping_criterion_to_kwarg(stopping_criterion) opts = NelderMead(opt.M, - loss; - return_state = true, - sckwarg...) + loss; + return_state = true, + sckwarg...) minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + :who_knows end ## conjugate gradient descent struct ConjugateGradientDescentOptimizer{Teval <: AbstractEvaluationType, - TM <: AbstractManifold, TLS <: Stepsize} <: + TM <: AbstractManifold, TLS <: Stepsize} <: AbstractManoptOptimizer M::TM stepsize::TLS end function ConjugateGradientDescentOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = InplaceEvaluation(), - stepsize::Stepsize = ArmijoLinesearch(M)) + eval::AbstractEvaluationType = InplaceEvaluation(), + stepsize::Stepsize = ArmijoLinesearch(M)) ConjugateGradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, - stepsize) + stepsize) end function call_manopt_optimizer(opt::ConjugateGradientDescentOptimizer{Teval}, - loss, - gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType - } + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType +} sckwarg = stopping_criterion_to_kwarg(stopping_criterion) opts = conjugate_gradient_descent(opt.M, - loss, - gradF, - x0; - return_state = true, - evaluation = Teval(), - stepsize = opt.stepsize, - sckwarg...) + loss, + gradF, + x0; + return_state = true, + evaluation = Teval(), + stepsize = opt.stepsize, + sckwarg...) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + :who_knows end ## particle swarm struct ParticleSwarmOptimizer{Teval <: AbstractEvaluationType, - TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, - Tinvretr <: AbstractInverseRetractionMethod, - Tvt <: AbstractVectorTransportMethod} <: + TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, + Tinvretr <: AbstractInverseRetractionMethod, + Tvt <: AbstractVectorTransportMethod} <: AbstractManoptOptimizer M::TM retraction_method::Tretr @@ -136,50 +135,50 @@ struct ParticleSwarmOptimizer{Teval <: AbstractEvaluationType, end function ParticleSwarmOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = InplaceEvaluation(), - population_size::Int = 100, - retraction_method::AbstractRetractionMethod = default_retraction_method(M), - inverse_retraction_method::AbstractInverseRetractionMethod = default_inverse_retraction_method(M), - vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M)) + eval::AbstractEvaluationType = InplaceEvaluation(), + population_size::Int = 100, + retraction_method::AbstractRetractionMethod = default_retraction_method(M), + inverse_retraction_method::AbstractInverseRetractionMethod = default_inverse_retraction_method(M), + vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M)) ParticleSwarmOptimizer{typeof(eval), typeof(M), typeof(retraction_method), - typeof(inverse_retraction_method), - typeof(vector_transport_method)}(M, - retraction_method, - inverse_retraction_method, - vector_transport_method, - population_size) + typeof(inverse_retraction_method), + typeof(vector_transport_method)}(M, + retraction_method, + inverse_retraction_method, + vector_transport_method, + population_size) end function call_manopt_optimizer(opt::ParticleSwarmOptimizer{Teval}, - loss, - gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType - } + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType +} sckwarg = stopping_criterion_to_kwarg(stopping_criterion) initial_population = vcat([x0], [rand(opt.M) for _ in 1:(opt.population_size - 1)]) opts = particle_swarm(opt.M, - loss; - x0 = initial_population, - n = opt.population_size, - return_state = true, - retraction_method = opt.retraction_method, - inverse_retraction_method = opt.inverse_retraction_method, - vector_transport_method = opt.vector_transport_method, - sckwarg...) + loss; + x0 = initial_population, + n = opt.population_size, + return_state = true, + retraction_method = opt.retraction_method, + inverse_retraction_method = opt.inverse_retraction_method, + vector_transport_method = opt.vector_transport_method, + sckwarg...) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + :who_knows end ## quasi Newton struct QuasiNewtonOptimizer{Teval <: AbstractEvaluationType, - TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, - Tvt <: AbstractVectorTransportMethod, TLS <: Stepsize} <: + TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, + Tvt <: AbstractVectorTransportMethod, TLS <: Stepsize} <: AbstractManoptOptimizer M::TM retraction_method::Tretr @@ -188,43 +187,43 @@ struct QuasiNewtonOptimizer{Teval <: AbstractEvaluationType, end function QuasiNewtonOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = InplaceEvaluation(), - retraction_method::AbstractRetractionMethod = default_retraction_method(M), - vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M), - stepsize = WolfePowellLinesearch(M; - retraction_method = retraction_method, - vector_transport_method = vector_transport_method, - linesearch_stopsize = 1e-12)) + eval::AbstractEvaluationType = InplaceEvaluation(), + retraction_method::AbstractRetractionMethod = default_retraction_method(M), + vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M), + stepsize = WolfePowellLinesearch(M; + retraction_method = retraction_method, + vector_transport_method = vector_transport_method, + linesearch_stopsize = 1e-12)) QuasiNewtonOptimizer{typeof(eval), typeof(M), typeof(retraction_method), - typeof(vector_transport_method), typeof(stepsize)}(M, - retraction_method, - vector_transport_method, - stepsize) + typeof(vector_transport_method), typeof(stepsize)}(M, + retraction_method, + vector_transport_method, + stepsize) end function call_manopt_optimizer(opt::QuasiNewtonOptimizer{Teval}, - loss, - gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType - } + loss, + gradF, + x0, + stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { + Teval <: + AbstractEvaluationType +} sckwarg = stopping_criterion_to_kwarg(stopping_criterion) opts = quasi_Newton(opt.M, - loss, - gradF, - x0; - return_state = true, - evaluation = Teval(), - retraction_method = opt.retraction_method, - vector_transport_method = opt.vector_transport_method, - stepsize = opt.stepsize, - sckwarg...) + loss, + gradF, + x0; + return_state = true, + evaluation = Teval(), + retraction_method = opt.retraction_method, + vector_transport_method = opt.vector_transport_method, + stepsize = opt.stepsize, + sckwarg...) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + :who_knows end ## Optimization.jl stuff @@ -255,15 +254,15 @@ end # 3) add callbacks to Manopt.jl function SciMLBase.__solve(prob::OptimizationProblem, - opt::AbstractManoptOptimizer, - data = Optimization.DEFAULT_DATA; - callback = (args...) -> (false), - maxiters::Union{Number, Nothing} = nothing, - maxtime::Union{Number, Nothing} = nothing, - abstol::Union{Number, Nothing} = nothing, - reltol::Union{Number, Nothing} = nothing, - progress = false, - kwargs...) + opt::AbstractManoptOptimizer, + data = Optimization.DEFAULT_DATA; + callback = (args...) -> (false), + maxiters::Union{Number, Nothing} = nothing, + maxtime::Union{Number, Nothing} = nothing, + abstol::Union{Number, Nothing} = nothing, + reltol::Union{Number, Nothing} = nothing, + progress = false, + kwargs...) local x, cur, state manifold = haskey(prob.kwargs, :manifold) ? prob.kwargs[:manifold] : nothing @@ -295,12 +294,12 @@ function SciMLBase.__solve(prob::OptimizationProblem, opt_res, opt_ret = call_manopt_optimizer(opt, _loss, gradF, prob.u0, stopping_criterion) return SciMLBase.build_solution(SciMLBase.DefaultOptimizationCache(prob.f, prob.p), - opt, - opt_res.minimizer, - prob.sense === Optimization.MaxSense ? - -opt_res.minimum : opt_res.minimum; - original = opt_res.options, - retcode = opt_ret) + opt, + opt_res.minimizer, + prob.sense === Optimization.MaxSense ? + -opt_res.minimum : opt_res.minimum; + original = opt_res.options, + retcode = opt_ret) end -end # module OptimizationManopt \ No newline at end of file +end # module OptimizationManopt diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index 1b56ed570..4c5809cfb 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -18,28 +18,30 @@ R2 = Euclidean(2) @testset "Error on no or mismatching manifolds" begin x0 = zeros(2) - p = [1.0, 100.0] + p = [1.0, 100.0] stepsize = Manopt.ArmijoLinesearch(R2) opt = OptimizationManopt.GradientDescentOptimizer(R2, - stepsize = stepsize) + stepsize = stepsize) optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p) - @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve(prob_forwarddiff, opt) + @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve( + prob_forwarddiff, opt) optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) prob = OptimizationProblem(optprob, x0, p; manifold = SymmetricPositiveDefinite(5)) - @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve(prob, opt) + @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve( + prob, opt) end @testset "Gradient descent" begin x0 = zeros(2) - p = [1.0, 100.0] + p = [1.0, 100.0] stepsize = Manopt.ArmijoLinesearch(R2) opt = OptimizationManopt.GradientDescentOptimizer(R2, - stepsize = stepsize) + stepsize = stepsize) optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoEnzyme()) prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p; manifold = R2) @@ -71,7 +73,7 @@ end stepsize = Manopt.ArmijoLinesearch(R2) opt = OptimizationManopt.ConjugateGradientDescentOptimizer(R2, - stepsize = stepsize) + stepsize = stepsize) optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) prob = OptimizationProblem(optprob, x0, p; manifold = R2) @@ -123,7 +125,7 @@ end m = 100 σ = 0.005 q = Matrix{Float64}(I, 5, 5) .+ 2.0 - data2 = [exp(M, q, σ * rand(M; vector_at=q)) for i in 1:m]; + data2 = [exp(M, q, σ * rand(M; vector_at = q)) for i in 1:m] f(M, x, p = nothing) = sum(distance(M, x, data2[i])^2 for i in 1:m) f(x, p = nothing) = sum(distance(M, x, data2[i])^2 for i in 1:m) @@ -135,4 +137,4 @@ end @time sol = Optimization.solve(prob, opt) @test sol.u ≈ q -end \ No newline at end of file +end From 051ec6c8bedb6507bc27590ff0cf8aced4cb3acb Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Mon, 25 Mar 2024 10:56:41 -0400 Subject: [PATCH 08/11] Reexport --- lib/OptimizationManopt/Project.toml | 3 ++- lib/OptimizationManopt/src/OptimizationManopt.jl | 8 +++++--- lib/OptimizationManopt/test/runtests.jl | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/OptimizationManopt/Project.toml b/lib/OptimizationManopt/Project.toml index baa8591c9..a8c083470 100644 --- a/lib/OptimizationManopt/Project.toml +++ b/lib/OptimizationManopt/Project.toml @@ -8,4 +8,5 @@ ManifoldDiff = "af67fdf4-a580-4b9f-bbec-742ef357defd" Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e" ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb" Manopt = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5" -Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" \ No newline at end of file +Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" +Reexport = "189a3867-3050-52da-a836-e630ba90ab69" diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index 982618615..b2e14a2fe 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -1,5 +1,7 @@ module OptimizationManopt +using Reexport +@reexport using Manopt using Optimization, Manopt, ManifoldsBase, ManifoldDiff """ @@ -228,9 +230,9 @@ end ## Optimization.jl stuff -function build_loss(f::OptimizationFunction, prob, cur) +function build_loss(f::OptimizationFunction, prob) function (::AbstractManifold, θ) - x = f.f(θ, prob.p, cur...) + x = f.f(θ) __x = first(x) return prob.sense === Optimization.MaxSense ? -__x : __x end @@ -287,7 +289,7 @@ function SciMLBase.__solve(prob::OptimizationProblem, f = Optimization.instantiate_function(prob.f, prob.u0, prob.f.adtype, prob.p) - _loss = build_loss(f, prob, cur) + _loss = build_loss(f, prob) gradF = build_gradF(f, prob, cur) diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index 4c5809cfb..4a9ed3176 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -131,10 +131,10 @@ end f(x, p = nothing) = sum(distance(M, x, data2[i])^2 for i in 1:m) optf = OptimizationFunction(f, Optimization.AutoForwardDiff()) - prob = OptimizationProblem(optf, data2[1]; manifold = M) + prob = OptimizationProblem(optf, data2[1]; manifold = M, maxiters = 1000) opt = OptimizationManopt.GradientDescentOptimizer(M) @time sol = Optimization.solve(prob, opt) - @test sol.u ≈ q + @test sol.u ≈ q atol = 1e-2 end From 80d56718e2061bb13970f00c891b53525f5e84d4 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Fri, 5 Apr 2024 17:29:16 -0400 Subject: [PATCH 09/11] Update to caching interface and remove manifold from solver structs --- .../src/OptimizationManopt.jl | 367 +++++++++--------- lib/OptimizationManopt/test/runtests.jl | 40 +- 2 files changed, 201 insertions(+), 206 deletions(-) diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index b2e14a2fe..3507f7660 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -2,7 +2,7 @@ module OptimizationManopt using Reexport @reexport using Manopt -using Optimization, Manopt, ManifoldsBase, ManifoldDiff +using Optimization, Manopt, ManifoldsBase, ManifoldDiff, Optimization.SciMLBase """ abstract type AbstractManoptOptimizer end @@ -12,233 +12,191 @@ internal state. """ abstract type AbstractManoptOptimizer end -function stopping_criterion_to_kwarg(stopping_criterion::Nothing) - return NamedTuple() -end -function stopping_criterion_to_kwarg(stopping_criterion::StoppingCriterion) - return (; stopping_criterion = stopping_criterion) -end +SciMLBase.supports_opt_cache_interface(opt::AbstractManoptOptimizer) = true -## gradient descent +function __map_optimizer_args!(cache::OptimizationCache, + opt::AbstractManoptOptimizer; + callback = nothing, + maxiters::Union{Number, Nothing} = nothing, + maxtime::Union{Number, Nothing} = nothing, + abstol::Union{Number, Nothing} = nothing, + reltol::Union{Number, Nothing} = nothing, + kwargs...) -struct GradientDescentOptimizer{ - Teval <: AbstractEvaluationType, - TM <: AbstractManifold, - TLS <: Linesearch -} <: AbstractManoptOptimizer - M::TM - stepsize::TLS -end + solver_kwargs = (; kwargs...) -function GradientDescentOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = Manopt.AllocatingEvaluation(), - stepsize::Stepsize = ArmijoLinesearch(M)) - GradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, stepsize) + if !isnothing(maxiters) + solver_kwargs = (; solver_kwargs..., stopping_criterion = [Manopt.StopAfterIteration(maxiters)]) + end + + if !isnothing(maxtime) + if haskey(solver_kwargs, :stopping_criterion) + solver_kwargs = (; solver_kwargs..., stopping_criterion = push!(solver_kwargs.stopping_criterion, Manopt.StopAfterTime(maxtime))) + else + solver_kwargs = (; solver_kwargs..., stopping_criterion = [Manopt.StopAfter(maxtime)]) + end + end + + if !isnothing(abstol) + if haskey(solver_kwargs, :stopping_criterion) + solver_kwargs = (; solver_kwargs..., stopping_criterion = push!(solver_kwargs.stopping_criterion, Manopt.StopWhenChangeLess(abstol))) + else + solver_kwargs = (; solver_kwargs..., stopping_criterion = [Manopt.StopWhenChangeLess(abstol)]) + end + end + + if !isnothing(reltol) + @warn "common reltol is currently not used by $(typeof(opt).super)" + end + return solver_kwargs end -function call_manopt_optimizer(opt::GradientDescentOptimizer{Teval}, +## gradient descent +struct GradientDescentOptimizer <: AbstractManoptOptimizer end + +function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, opt::GradientDescentOptimizer, loss, gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType -} - sckwarg = stopping_criterion_to_kwarg(stopping_criterion) - opts = gradient_descent(opt.M, + x0; + stopping_criterion::Union{Manopt.StoppingCriterion, Manopt.StoppingCriterionSet}, + evaluation::AbstractEvaluationType = Manopt.AllocatingEvaluation(), + stepsize::Stepsize = ArmijoLinesearch(M), + kwargs...) + opts = gradient_descent(M, loss, gradF, x0; return_state = true, - evaluation = Teval(), - stepsize = opt.stepsize, - sckwarg...) + evaluation, + stepsize, + stopping_criterion) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) - return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + return (; minimizer = minimizer, minimum = loss(M, minimizer), options = opts) end ## Nelder-Mead +struct NelderMeadOptimizer <: AbstractManoptOptimizer end -struct NelderMeadOptimizer{ - TM <: AbstractManifold, -} <: AbstractManoptOptimizer - M::TM -end - -function call_manopt_optimizer(opt::NelderMeadOptimizer, +function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, opt::NelderMeadOptimizer, loss, gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) - sckwarg = stopping_criterion_to_kwarg(stopping_criterion) + x0; + stopping_criterion::Union{Manopt.StoppingCriterion, Manopt.StoppingCriterionSet}, + kwargs...) - opts = NelderMead(opt.M, + opts = NelderMead(M, loss; return_state = true, - sckwarg...) + stopping_criterion) minimizer = Manopt.get_solver_result(opts) - return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + return (; minimizer = minimizer, minimum = loss(M, minimizer), options = opts) end ## conjugate gradient descent +struct ConjugateGradientDescentOptimizer <: AbstractManoptOptimizer end -struct ConjugateGradientDescentOptimizer{Teval <: AbstractEvaluationType, - TM <: AbstractManifold, TLS <: Stepsize} <: - AbstractManoptOptimizer - M::TM - stepsize::TLS -end - -function ConjugateGradientDescentOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = InplaceEvaluation(), - stepsize::Stepsize = ArmijoLinesearch(M)) - ConjugateGradientDescentOptimizer{typeof(eval), typeof(M), typeof(stepsize)}(M, - stepsize) -end - -function call_manopt_optimizer(opt::ConjugateGradientDescentOptimizer{Teval}, +function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, + opt::ConjugateGradientDescentOptimizer, loss, gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType -} - sckwarg = stopping_criterion_to_kwarg(stopping_criterion) - opts = conjugate_gradient_descent(opt.M, + x0; + stopping_criterion::Union{Manopt.StoppingCriterion, Manopt.StoppingCriterionSet}, + evaluation::AbstractEvaluationType = InplaceEvaluation(), + stepsize::Stepsize = ArmijoLinesearch(M), + kwargs...) + + opts = conjugate_gradient_descent(M, loss, gradF, x0; return_state = true, - evaluation = Teval(), - stepsize = opt.stepsize, - sckwarg...) + evaluation, + stepsize, + stopping_criterion) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) - return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + return (; minimizer = minimizer, minimum = loss(M, minimizer), options = opts) end ## particle swarm +struct ParticleSwarmOptimizer <: AbstractManoptOptimizer end -struct ParticleSwarmOptimizer{Teval <: AbstractEvaluationType, - TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, - Tinvretr <: AbstractInverseRetractionMethod, - Tvt <: AbstractVectorTransportMethod} <: - AbstractManoptOptimizer - M::TM - retraction_method::Tretr - inverse_retraction_method::Tinvretr - vector_transport_method::Tvt - population_size::Int -end - -function ParticleSwarmOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = InplaceEvaluation(), +function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, + opt::ParticleSwarmOptimizer, + loss, + gradF, + x0; + stopping_criterion::Union{Manopt.StoppingCriterion, Manopt.StoppingCriterionSet}, + evaluation::AbstractEvaluationType = InplaceEvaluation(), population_size::Int = 100, retraction_method::AbstractRetractionMethod = default_retraction_method(M), inverse_retraction_method::AbstractInverseRetractionMethod = default_inverse_retraction_method(M), - vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M)) - ParticleSwarmOptimizer{typeof(eval), typeof(M), typeof(retraction_method), - typeof(inverse_retraction_method), - typeof(vector_transport_method)}(M, - retraction_method, - inverse_retraction_method, - vector_transport_method, - population_size) -end - -function call_manopt_optimizer(opt::ParticleSwarmOptimizer{Teval}, - loss, - gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType -} - sckwarg = stopping_criterion_to_kwarg(stopping_criterion) - initial_population = vcat([x0], [rand(opt.M) for _ in 1:(opt.population_size - 1)]) - opts = particle_swarm(opt.M, + vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M), + kwargs...) + + initial_population = vcat([x0], [rand(M) for _ in 1:(population_size - 1)]) + opts = particle_swarm(M, loss; x0 = initial_population, - n = opt.population_size, + n = population_size, return_state = true, - retraction_method = opt.retraction_method, - inverse_retraction_method = opt.inverse_retraction_method, - vector_transport_method = opt.vector_transport_method, - sckwarg...) + retraction_method, + inverse_retraction_method, + vector_transport_method, + stopping_criterion) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) - return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + return (; minimizer = minimizer, minimum = loss(M, minimizer), options = opts) end ## quasi Newton -struct QuasiNewtonOptimizer{Teval <: AbstractEvaluationType, - TM <: AbstractManifold, Tretr <: AbstractRetractionMethod, - Tvt <: AbstractVectorTransportMethod, TLS <: Stepsize} <: - AbstractManoptOptimizer - M::TM - retraction_method::Tretr - vector_transport_method::Tvt - stepsize::TLS -end +struct QuasiNewtonOptimizer <: AbstractManoptOptimizer end -function QuasiNewtonOptimizer(M::AbstractManifold; - eval::AbstractEvaluationType = InplaceEvaluation(), +function call_manopt_optimizer(M::Manopt.AbstractManifold, + opt::QuasiNewtonOptimizer, + loss, + gradF, + x0; + stopping_criterion::Union{Manopt.StoppingCriterion, Manopt.StoppingCriterionSet}, + evaluation::AbstractEvaluationType = InplaceEvaluation(), retraction_method::AbstractRetractionMethod = default_retraction_method(M), vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M), stepsize = WolfePowellLinesearch(M; retraction_method = retraction_method, vector_transport_method = vector_transport_method, - linesearch_stopsize = 1e-12)) - QuasiNewtonOptimizer{typeof(eval), typeof(M), typeof(retraction_method), - typeof(vector_transport_method), typeof(stepsize)}(M, - retraction_method, - vector_transport_method, - stepsize) -end - -function call_manopt_optimizer(opt::QuasiNewtonOptimizer{Teval}, - loss, - gradF, - x0, - stopping_criterion::Union{Nothing, Manopt.StoppingCriterion}) where { - Teval <: - AbstractEvaluationType -} - sckwarg = stopping_criterion_to_kwarg(stopping_criterion) - opts = quasi_Newton(opt.M, + linesearch_stopsize = 1e-12), + kwargs... + ) + + opts = quasi_Newton(M, loss, gradF, x0; return_state = true, - evaluation = Teval(), - retraction_method = opt.retraction_method, - vector_transport_method = opt.vector_transport_method, - stepsize = opt.stepsize, - sckwarg...) + evaluation, + retraction_method, + vector_transport_method, + stepsize, + stopping_criterion) # we unwrap DebugOptions here minimizer = Manopt.get_solver_result(opts) - return (; minimizer = minimizer, minimum = loss(opt.M, minimizer), options = opts), - :who_knows + return (; minimizer = minimizer, minimum = loss(M, minimizer), options = opts) end ## Optimization.jl stuff -function build_loss(f::OptimizationFunction, prob) +function build_loss(f::OptimizationFunction, prob, cb) function (::AbstractManifold, θ) - x = f.f(θ) + x = f.f(θ, prob.p) + cb(x, θ) __x = first(x) return prob.sense === Optimization.MaxSense ? -__x : __x end end -function build_gradF(f::OptimizationFunction{true}, prob, cur) +function build_gradF(f::OptimizationFunction{true}, cur) function g(M::AbstractManifold, G, θ) f.grad(G, θ, cur...) G .= riemannian_gradient(M, θ, G) @@ -255,50 +213,91 @@ end # 2) return convergence information # 3) add callbacks to Manopt.jl -function SciMLBase.__solve(prob::OptimizationProblem, - opt::AbstractManoptOptimizer, - data = Optimization.DEFAULT_DATA; - callback = (args...) -> (false), - maxiters::Union{Number, Nothing} = nothing, - maxtime::Union{Number, Nothing} = nothing, - abstol::Union{Number, Nothing} = nothing, - reltol::Union{Number, Nothing} = nothing, - progress = false, - kwargs...) +function SciMLBase.__solve(cache::OptimizationCache{ + F, + RC, + LB, + UB, + LC, + UC, + S, + O, + D, + P, + C +}) where { + F, + RC, + LB, + UB, + LC, + UC, + S, + O <: + AbstractManoptOptimizer, + D, + P, + C +} local x, cur, state - manifold = haskey(prob.kwargs, :manifold) ? prob.kwargs[:manifold] : nothing + manifold = haskey(cache.solver_args, :manifold) ? cache.solver_args[:manifold] : nothing - if manifold === nothing || manifold !== opt.M - throw(ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`")) + if manifold === nothing + throw(ArgumentError("Manifold not specified in the problem for e.g. `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))`.")) end - if data !== Optimization.DEFAULT_DATA - maxiters = length(data) + if cache.data !== Optimization.DEFAULT_DATA + maxiters = length(cache.data) + else + maxiters = cache.solver_args.maxiters end - cur, state = iterate(data) - - stopping_criterion = nothing - if maxiters !== nothing - stopping_criterion = StopAfterIteration(maxiters) + cur, state = iterate(cache.data) + + function _cb(x, θ) + opt_state = Optimization.OptimizationState(iter = 0, + u = θ, + objective = x[1]) + cb_call = cache.callback(opt_state, x...) + if !(cb_call isa Bool) + error("The callback should return a boolean `halt` for whether to stop the optimization process.") + end + nx_itr = iterate(cache.data, state) + if isnothing(nx_itr) + true + else + cur, state = nx_itr + cb_call + end end + solver_kwarg = __map_optimizer_args!(cache, cache.opt, callback = _cb, + maxiters = maxiters, + maxtime = cache.solver_args.maxtime, + abstol = cache.solver_args.abstol, + reltol = cache.solver_args.reltol; + ) - maxiters = Optimization._check_and_convert_maxiters(maxiters) - maxtime = Optimization._check_and_convert_maxtime(maxtime) + _loss = build_loss(cache.f, cache, _cb) - f = Optimization.instantiate_function(prob.f, prob.u0, prob.f.adtype, prob.p) + gradF = build_gradF(cache.f, cur) + + if haskey(solver_kwarg, :stopping_criterion) + stopping_criterion = Manopt.StopWhenAny(solver_kwarg.stopping_criterion...) + else + stopping_criterion = Manopt.StopAfterIteration(500) + end - _loss = build_loss(f, prob) + opt_res = call_manopt_optimizer(manifold, cache.opt, _loss, gradF, cache.u0; solver_kwarg..., stopping_criterion=stopping_criterion) - gradF = build_gradF(f, prob, cur) + asc = get_active_stopping_criteria(opt_res.options.stop) - opt_res, opt_ret = call_manopt_optimizer(opt, _loss, gradF, prob.u0, stopping_criterion) + opt_ret = any(Manopt.indicates_convergence, asc) ? ReturnCode.Success : ReturnCode.Failure - return SciMLBase.build_solution(SciMLBase.DefaultOptimizationCache(prob.f, prob.p), - opt, + return SciMLBase.build_solution(cache, + cache.opt, opt_res.minimizer, - prob.sense === Optimization.MaxSense ? + cache.sense === Optimization.MaxSense ? -opt_res.minimum : opt_res.minimum; original = opt_res.options, retcode = opt_ret) diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index 4a9ed3176..b309d9932 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -21,18 +21,12 @@ R2 = Euclidean(2) p = [1.0, 100.0] stepsize = Manopt.ArmijoLinesearch(R2) - opt = OptimizationManopt.GradientDescentOptimizer(R2, - stepsize = stepsize) + opt = OptimizationManopt.GradientDescentOptimizer() optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p) - @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve( + @test_throws ArgumentError("Manifold not specified in the problem for e.g. `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))`.") Optimization.solve( prob_forwarddiff, opt) - - optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) - prob = OptimizationProblem(optprob, x0, p; manifold = SymmetricPositiveDefinite(5)) - @test_throws ArgumentError("Either manifold not specified in the problem `OptimizationProblem(f, x, p; manifold = SymmetricPositiveDefinite(5))` or it doesn't match the manifold specified in the optimizer `$(opt.M)`") Optimization.solve( - prob, opt) end @testset "Gradient descent" begin @@ -40,16 +34,15 @@ end p = [1.0, 100.0] stepsize = Manopt.ArmijoLinesearch(R2) - opt = OptimizationManopt.GradientDescentOptimizer(R2, - stepsize = stepsize) + opt = OptimizationManopt.GradientDescentOptimizer() optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoEnzyme()) - prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p; manifold = R2) + prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p; manifold = R2, stepsize = stepsize) sol = Optimization.solve(prob_forwarddiff, opt) @test sol.minimum < 0.2 optprob_grad = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!) - prob_grad = OptimizationProblem(optprob_grad, x0, p; manifold = R2) + prob_grad = OptimizationProblem(optprob_grad, x0, p; manifold = R2, stepsize = stepsize) sol = Optimization.solve(prob_grad, opt) @test sol.minimum < 0.2 end @@ -58,7 +51,7 @@ end x0 = zeros(2) p = [1.0, 100.0] - opt = OptimizationManopt.NelderMeadOptimizer(R2) + opt = OptimizationManopt.NelderMeadOptimizer() optprob = OptimizationFunction(rosenbrock) prob = OptimizationProblem(optprob, x0, p; manifold = R2) @@ -72,13 +65,12 @@ end p = [1.0, 100.0] stepsize = Manopt.ArmijoLinesearch(R2) - opt = OptimizationManopt.ConjugateGradientDescentOptimizer(R2, - stepsize = stepsize) + opt = OptimizationManopt.ConjugateGradientDescentOptimizer() optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) prob = OptimizationProblem(optprob, x0, p; manifold = R2) - sol = Optimization.solve(prob, opt) + sol = Optimization.solve(prob, opt, stepsize = stepsize) @test sol.minimum < 0.5 end @@ -86,12 +78,16 @@ end x0 = zeros(2) p = [1.0, 100.0] - opt = OptimizationManopt.QuasiNewtonOptimizer(R2) - + opt = OptimizationManopt.QuasiNewtonOptimizer() + function callback(state, l) + println(state.u) + println(l) + return false + end optprob = OptimizationFunction(rosenbrock, Optimization.AutoForwardDiff()) prob = OptimizationProblem(optprob, x0, p; manifold = R2) - sol = Optimization.solve(prob, opt) + sol = Optimization.solve(prob, opt, callback = callback, maxiters = 30) @test sol.minimum < 1e-14 end @@ -99,7 +95,7 @@ end x0 = zeros(2) p = [1.0, 100.0] - opt = OptimizationManopt.ParticleSwarmOptimizer(R2) + opt = OptimizationManopt.ParticleSwarmOptimizer() optprob = OptimizationFunction(rosenbrock) prob = OptimizationProblem(optprob, x0, p; manifold = R2) @@ -113,7 +109,7 @@ end x0 = zeros(2) p = [1.0, 100.0] - opt = OptimizationManopt.GradientDescentOptimizer(R2) + opt = OptimizationManopt.GradientDescentOptimizer() optprob_cons = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!, cons = cons) prob_cons = OptimizationProblem(optprob_cons, x0, p) @@ -133,7 +129,7 @@ end optf = OptimizationFunction(f, Optimization.AutoForwardDiff()) prob = OptimizationProblem(optf, data2[1]; manifold = M, maxiters = 1000) - opt = OptimizationManopt.GradientDescentOptimizer(M) + opt = OptimizationManopt.GradientDescentOptimizer() @time sol = Optimization.solve(prob, opt) @test sol.u ≈ q atol = 1e-2 From 020c967623b6bde78ba641e47a47433ae31a24d1 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Fri, 5 Apr 2024 18:01:41 -0400 Subject: [PATCH 10/11] testing and format --- .github/workflows/CI.yml | 2 +- .../src/OptimizationManopt.jl | 91 ++++++++++--------- lib/OptimizationManopt/test/runtests.jl | 5 +- 3 files changed, 52 insertions(+), 46 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ef2e6850e..8cfed5ae4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -24,11 +24,11 @@ jobs: - OptimizationEvolutionary - OptimizationFlux - OptimizationGCMAES + - OptimizationManopt - OptimizationMetaheuristics - OptimizationMOI - OptimizationMultistartOptimization - OptimizationNLopt - #- OptimizationNonconvex - OptimizationNOMAD - OptimizationOptimJL - OptimizationOptimisers diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index 3507f7660..6f536d21a 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -22,26 +22,32 @@ function __map_optimizer_args!(cache::OptimizationCache, abstol::Union{Number, Nothing} = nothing, reltol::Union{Number, Nothing} = nothing, kwargs...) - solver_kwargs = (; kwargs...) if !isnothing(maxiters) - solver_kwargs = (; solver_kwargs..., stopping_criterion = [Manopt.StopAfterIteration(maxiters)]) + solver_kwargs = (; + solver_kwargs..., stopping_criterion = [Manopt.StopAfterIteration(maxiters)]) end if !isnothing(maxtime) if haskey(solver_kwargs, :stopping_criterion) - solver_kwargs = (; solver_kwargs..., stopping_criterion = push!(solver_kwargs.stopping_criterion, Manopt.StopAfterTime(maxtime))) + solver_kwargs = (; solver_kwargs..., + stopping_criterion = push!( + solver_kwargs.stopping_criterion, Manopt.StopAfterTime(maxtime))) else - solver_kwargs = (; solver_kwargs..., stopping_criterion = [Manopt.StopAfter(maxtime)]) + solver_kwargs = (; + solver_kwargs..., stopping_criterion = [Manopt.StopAfter(maxtime)]) end end if !isnothing(abstol) if haskey(solver_kwargs, :stopping_criterion) - solver_kwargs = (; solver_kwargs..., stopping_criterion = push!(solver_kwargs.stopping_criterion, Manopt.StopWhenChangeLess(abstol))) + solver_kwargs = (; solver_kwargs..., + stopping_criterion = push!( + solver_kwargs.stopping_criterion, Manopt.StopWhenChangeLess(abstol))) else - solver_kwargs = (; solver_kwargs..., stopping_criterion = [Manopt.StopWhenChangeLess(abstol)]) + solver_kwargs = (; + solver_kwargs..., stopping_criterion = [Manopt.StopWhenChangeLess(abstol)]) end end @@ -54,7 +60,8 @@ end ## gradient descent struct GradientDescentOptimizer <: AbstractManoptOptimizer end -function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, opt::GradientDescentOptimizer, +function call_manopt_optimizer( + M::ManifoldsBase.AbstractManifold, opt::GradientDescentOptimizer, loss, gradF, x0; @@ -84,7 +91,6 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, opt::NelderMea x0; stopping_criterion::Union{Manopt.StoppingCriterion, Manopt.StoppingCriterionSet}, kwargs...) - opts = NelderMead(M, loss; return_state = true, @@ -96,7 +102,7 @@ end ## conjugate gradient descent struct ConjugateGradientDescentOptimizer <: AbstractManoptOptimizer end -function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, +function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, opt::ConjugateGradientDescentOptimizer, loss, gradF, @@ -105,7 +111,6 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, evaluation::AbstractEvaluationType = InplaceEvaluation(), stepsize::Stepsize = ArmijoLinesearch(M), kwargs...) - opts = conjugate_gradient_descent(M, loss, gradF, @@ -134,7 +139,6 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, inverse_retraction_method::AbstractInverseRetractionMethod = default_inverse_retraction_method(M), vector_transport_method::AbstractVectorTransportMethod = default_vector_transport_method(M), kwargs...) - initial_population = vcat([x0], [rand(M) for _ in 1:(population_size - 1)]) opts = particle_swarm(M, loss; @@ -168,8 +172,7 @@ function call_manopt_optimizer(M::Manopt.AbstractManifold, vector_transport_method = vector_transport_method, linesearch_stopsize = 1e-12), kwargs... - ) - +) opts = quasi_Newton(M, loss, gradF, @@ -214,30 +217,30 @@ end # 3) add callbacks to Manopt.jl function SciMLBase.__solve(cache::OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C + F, + RC, + LB, + UB, + LC, + UC, + S, + O, + D, + P, + C }) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - AbstractManoptOptimizer, - D, - P, - C + F, + RC, + LB, + UB, + LC, + UC, + S, + O <: + AbstractManoptOptimizer, + D, + P, + C } local x, cur, state @@ -272,11 +275,11 @@ function SciMLBase.__solve(cache::OptimizationCache{ end end solver_kwarg = __map_optimizer_args!(cache, cache.opt, callback = _cb, - maxiters = maxiters, - maxtime = cache.solver_args.maxtime, - abstol = cache.solver_args.abstol, - reltol = cache.solver_args.reltol; - ) + maxiters = maxiters, + maxtime = cache.solver_args.maxtime, + abstol = cache.solver_args.abstol, + reltol = cache.solver_args.reltol; + ) _loss = build_loss(cache.f, cache, _cb) @@ -288,11 +291,13 @@ function SciMLBase.__solve(cache::OptimizationCache{ stopping_criterion = Manopt.StopAfterIteration(500) end - opt_res = call_manopt_optimizer(manifold, cache.opt, _loss, gradF, cache.u0; solver_kwarg..., stopping_criterion=stopping_criterion) + opt_res = call_manopt_optimizer(manifold, cache.opt, _loss, gradF, cache.u0; + solver_kwarg..., stopping_criterion = stopping_criterion) asc = get_active_stopping_criteria(opt_res.options.stop) - opt_ret = any(Manopt.indicates_convergence, asc) ? ReturnCode.Success : ReturnCode.Failure + opt_ret = any(Manopt.indicates_convergence, asc) ? ReturnCode.Success : + ReturnCode.Failure return SciMLBase.build_solution(cache, cache.opt, diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index b309d9932..ca9e5d083 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -37,7 +37,8 @@ end opt = OptimizationManopt.GradientDescentOptimizer() optprob_forwarddiff = OptimizationFunction(rosenbrock, Optimization.AutoEnzyme()) - prob_forwarddiff = OptimizationProblem(optprob_forwarddiff, x0, p; manifold = R2, stepsize = stepsize) + prob_forwarddiff = OptimizationProblem( + optprob_forwarddiff, x0, p; manifold = R2, stepsize = stepsize) sol = Optimization.solve(prob_forwarddiff, opt) @test sol.minimum < 0.2 @@ -132,5 +133,5 @@ end opt = OptimizationManopt.GradientDescentOptimizer() @time sol = Optimization.solve(prob, opt) - @test sol.u ≈ q atol = 1e-2 + @test sol.u≈q atol=1e-2 end From 5f2edfe090baeab87b2211733bef35c91e9abab0 Mon Sep 17 00:00:00 2001 From: Vaibhav Dixit Date: Fri, 5 Apr 2024 19:53:01 -0400 Subject: [PATCH 11/11] Add test deps --- lib/OptimizationManopt/Project.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/OptimizationManopt/Project.toml b/lib/OptimizationManopt/Project.toml index a8c083470..f60db4bb0 100644 --- a/lib/OptimizationManopt/Project.toml +++ b/lib/OptimizationManopt/Project.toml @@ -4,9 +4,20 @@ authors = ["Mateusz Baran "] version = "0.1.0" [deps] +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" ManifoldDiff = "af67fdf4-a580-4b9f-bbec-742ef357defd" Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e" ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb" Manopt = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5" Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" + +[extras] +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[targets] +test = ["Enzyme", "ForwardDiff", "Random", "Test", "Zygote"]