From f8631f73e12368a922e71829af91c5ad0116e487 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Tue, 27 Jun 2023 15:44:04 -0700 Subject: [PATCH 1/2] Add/expand a few docstrings New docstrings: - `objective_return` - `MutationStyle` - `InPlace` - `OutOfPlace` - `HessianApproximation` - `Static` Expanded docstrings: - `Backtracking` - `HZAW` - `ScalarObjective` --- src/NLSolvers.jl | 35 ++++++++++++++- .../linesearches/backtracking.jl | 45 ++++++++++--------- .../linesearches/hagerzhangline.jl | 26 +++++++++-- src/globalization/linesearches/static.jl | 5 +++ src/objectives.jl | 5 +++ 5 files changed, 88 insertions(+), 28 deletions(-) diff --git a/src/NLSolvers.jl b/src/NLSolvers.jl index 690e5e8..f5f5241 100644 --- a/src/NLSolvers.jl +++ b/src/NLSolvers.jl @@ -58,8 +58,12 @@ using Printf function solve end export solve + """ + objective_return(f, g, H=nothing) +Return the objective `f` or a tuple containing `f`, the gradient `g` if `g !== nothing`, +and the Hessian `H` if `H !== nothing`. """ function objective_return(f, g, H = nothing) if g isa Nothing && H isa Nothing @@ -70,16 +74,36 @@ function objective_return(f, g, H = nothing) return f, g, H end end + export objective_return + isallfinite(x) = mapreduce(isfinite, *, x) +""" + MutationStyle + +An abstract type with subtypes [`InPlace`](@ref) and [`OutOfPlace`](@ref) that permits +dispatching on whether an operation is mutating. +""" abstract type MutateStyle end -abstract type AbstractProblem end -abstract type AbstractOptions end +""" + InPlace +A [`MutationStyle`](@ref) for in-place (mutating) operations. +""" struct InPlace <: MutateStyle end + +""" + OutOfPlace + +A [`MutationStyle`](@ref) for out-of-place (non-mutating) operations. +""" struct OutOfPlace <: MutateStyle end + +abstract type AbstractProblem end +abstract type AbstractOptions end + include("precondition.jl") include("Manifolds.jl") include("objectives.jl") @@ -90,10 +114,17 @@ export NonDiffed, OnceDiffed, TwiceDiffed, ScalarObjective # make this struct that has scheme and approx abstract type QuasiNewton{T1} end +""" + HessianApproximation + +An abstract type that permits dispatching on particular methodologies for approximating +the Hessian of an objective function. +""" abstract type HessianApproximation end struct Inverse <: HessianApproximation end struct Direct <: HessianApproximation end export Inverse, Direct + # problem and options types include("optimize/problem_types.jl") export OptimizationProblem, OptimizationOptions diff --git a/src/globalization/linesearches/backtracking.jl b/src/globalization/linesearches/backtracking.jl index 21f4174..2e64cb2 100644 --- a/src/globalization/linesearches/backtracking.jl +++ b/src/globalization/linesearches/backtracking.jl @@ -35,33 +35,34 @@ function _safe_α(α_candidate, α_current, decrease = 0.1, ratio = 0.5) end """ - Backtracking(;) + Backtracking -Constructs an object to control the Backtracking line search algorithm. The +An object that controls the backtracking line search algorithm. The algorithm tries out an initial point and then it iteratively tries to find a good enough step length as measured by the improvement compared to a first order Taylor approximation around a step length of zero. -The Backtracking constructor takes the following keyword arguments - - ratio: the ratio between the current trial step length and the next - - decrease: a parameter that controls when descent is sufficient - - maxiter: maximum number of times to search for a better step length - - interp: which type of interpolation to use, if any - - steprange: the allowed range for steps lengths to be in given as a tuple - - verbose: the verbosity level - -`ratio` controls how much time to spend looking for a large step size. It -is very common to chose 1/2, but others can be chosen. It's - -When chosing parameters, it's important to note that it must be true that - 0 < decrease < ratio - -`maxiter` defaults to 26. With no interpolation and with a ratio of 1/2, this -means that we quit when the step size reaches sqrt(eps(Float64)). Provide another -value as appropriate. - - - + Backtracking(; kwargs...) + +The `Backtracking` constructor takes the following keyword arguments: + - `ratio`: the ratio between the current trial step length and the next. This controls + how much time is spent looking for a large step size. A common choice is 1/2, which + is the default here. + - `decrease`: a positive threshold less than `ratio` that determines when descent is + sufficient. The default value is 0.0001. + - `maxiter`: maximum number of times to search for a better step length. The default + value is 26. With no interpolation and a ratio of 1/2, this means that the search + is terminated when the step size reaches `sqrt(eps(Float64))`. + - `interp`: which type of interpolation to use, if any. Defaults to `FixedInterp()`, + i.e. no interpolation. + - `steprange`: the allowed range for steps lengths, given as a tuple or named tuple. + The default is `(; lower=0, upper=Inf)`. + - `verbose`: if `true`, information about the search is printed as the algorithm + progresses. Defaults to `false`. + +!!! note + When chosing parameters, it's important to note that it must be true that + `0 < decrease < ratio`. """ struct Backtracking{T1,T2,T3,TR} <: LineSearcher ratio::T1 diff --git a/src/globalization/linesearches/hagerzhangline.jl b/src/globalization/linesearches/hagerzhangline.jl index c3107c1..798bc6d 100644 --- a/src/globalization/linesearches/hagerzhangline.jl +++ b/src/globalization/linesearches/hagerzhangline.jl @@ -1,16 +1,34 @@ # fix eta k cchoice # fix quadstep? """ - HZAW(;) + HZAW -Implements the approximate Wolfe line search developed in [HZ2005]. +An object that controls the Hager-Zhang approximate Wolfe line search +algorithm.[^HZ2005] -theta controls the search in U3 of the paper. It defaults to 0.5, and as -a result uses bisection when (what is U3?) + HZAW(; kwargs...) + +The `HZAW` constructor takes the following keyword arguments. Default values +correspond to those used in section 5 of [^HZ2005]. + - `decrease`: parameter between 0 and 1, less than or equal to `curvature`, + specifying sufficient decrease in the objective per the Armijo rule. + Defaults to 0.1. + - `curvature`: parameter between 0 and 1, greater than or equal to `decrease`, + specifying sufficient decrease in the gradient per the curvature condition. + Defaults to 0.9. + - `theta`: parameter between 0 and 1 that controls the bracketing interval + update. Defaults to 1/2, which indicates bisection. (See step U3 of the + interval update procedure in section 4 of [^HZ2005].) + - `gamma`: factor by which the length of the bracketing interval decreases + at each iteration of the algorithm. Defaults to 2/3. We tweak the original algorithm slightly, by backtracking into a feasible region if the original step length results in function values that are not finite. + +[^HZ2005]: Hager, W. W., & Zhang, H. (2005). A New Conjugate Gradient Method + with Guaranteed Descent and an Efficient Line Search. SIAM Journal + on Optimization, 16(1), 170–192. doi:10.1137/030601880 """ struct HZAW{T} <: LineSearcher decrease::T diff --git a/src/globalization/linesearches/static.jl b/src/globalization/linesearches/static.jl index 8a6e333..50db037 100644 --- a/src/globalization/linesearches/static.jl +++ b/src/globalization/linesearches/static.jl @@ -1,3 +1,8 @@ +""" + Static + +An object that specifies a constant step length. +""" struct Static{T} <: LineSearcher α::T end diff --git a/src/objectives.jl b/src/objectives.jl index ffff7ce..b2d59f4 100644 --- a/src/objectives.jl +++ b/src/objectives.jl @@ -3,6 +3,11 @@ abstract type ObjWrapper end """ ScalarObjective +Type for objective functions that return a scalar value. +The type stores the objective function `f` and, if applicable, functions that compute +its gradient `g`, Hessian `h`, combinations thereof (`fg`, `fgh`), and a "batched" version +of the objective `batched_f` that applies `f` to all elements of an array. +It may also store an addition parameter `param` passed to these functions. """ struct ScalarObjective{Tf,Tg,Tfg,Tfgh,Th,Thv,Tbf,P} f::Tf From 4976a8eba2dd91a9e2ba674b5618136ce3c2d482 Mon Sep 17 00:00:00 2001 From: Patrick Kofod Mogensen Date: Thu, 27 Jul 2023 09:12:30 -0400 Subject: [PATCH 2/2] Update mkdocs.yml --- docs/mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index c32b2be..de04624 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -24,8 +24,8 @@ docs_dir: 'build' nav: - Home: 'index.md' -# - Tutorials: -# - Minimizing a function: 'user/minimization.md' + - Tutorials: + - Minimizing a function: 'optimization.md' # - Gradients and Hessians: 'user/gradientsandhessians.md' # - Configurable Options: 'user/config.md' # - Linesearch: 'algo/linesearch.md'