Optimizers
An Optimizer stores an OptimizerState, a OptimizerProblem, the SimpleSolvers.OptimizerResult and a SimpleSolvers.NonlinearMethod. Its purposes are:
using SimpleSolvers
using LinearAlgebra: norm
x = rand(3)
obj = OptimizerProblem(x -> sum((x - [0., 0., 1.]) .^ 2), x)
bt = Backtracking()
alg = Newton()
opt = Optimizer(x, obj; algorithm = alg, linesearch = bt)Optimizer{Float64, Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}, Linesearch{Float64, Backtracking{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}}(Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90710054772846e-310,5.0e-324,6.90709933784193e-310,6.9069764928064e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9071090642451e-310,6.9071075797067e-310,6.9071125655295e-310,6.9070993285535e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90709933781505e-310,6.9070680002707e-310,6.9070680014509e-310,6.90709933781505e-310)])), HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}(ForwardDiff.JacobianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,4.243991582e-314,0.0,1.5e-323), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(8.487983164e-314,0.0,2.5e-323,1.2731974746e-313), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,3.5e-323,1.69759663277e-313,0.0)]), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}}((Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0))), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90703353674366e-310,6.90703353674524e-310,6.9070335367468e-310,6.9070335462582e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90703354619336e-310,6.9069775150725e-310,6.90703353656026e-310,6.90703353517846e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90703353675473e-310,6.9070335367563e-310,6.9070335367579e-310,6.90703353675947e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90703353676105e-310,6.90703353676263e-310,6.90703353618873e-310,1.91105e-319)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.9889e-319,6.9070335352433e-310,6.90703353524486e-310,6.90703353524644e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90703353524803e-310,6.9070335352496e-310,6.90710054774076e-310,6.90703353521324e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90703353518637e-310,6.90703353518637e-310,6.9069775984281e-310,6.90703353525435e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9070335352512e-310,6.90703353525593e-310,6.9070335352417e-310,5.0e-324)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9070335352575e-310,6.9070335352591e-310,6.90703353526067e-310,6.90703353526225e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90703353526384e-310,6.9070335352654e-310,6.90703353525277e-310,6.907033535267e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9070335352686e-310,6.90703353527016e-310,6.90703353527174e-310,6.9070335352733e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9070335352749e-310,6.9070335352828e-310,3.5e-323,3.5e-323))]))), x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([NaN, NaN, NaN], [NaN, NaN, NaN], [NaN, NaN, NaN], [2.08e-322, 6.9069743752426e-310, 6.90703217056915e-310], [NaN, NaN, NaN], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]), Linesearch{Float64, Backtracking{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}(LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}(SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([NaN, NaN, NaN], [NaN, NaN, NaN], [NaN, NaN, NaN], [2.08e-322, 6.9069743752426e-310, 6.90703217056915e-310], [NaN, NaN, NaN], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN])), SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90710054772846e-310,5.0e-324,6.90709933784193e-310,6.9069764928064e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9071090642451e-310,6.9071075797067e-310,6.9071125655295e-310,6.9070993285535e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90709933781505e-310,6.9070680002707e-310,6.9070680014509e-310,6.90709933781505e-310)])), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([NaN, NaN, NaN], [NaN, NaN, NaN], [NaN, NaN, NaN], [2.08e-322, 6.9069743752426e-310, 6.90703217056915e-310], [NaN, NaN, NaN], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]))), Backtracking with α₀ = 1.0 c₁ = 0.0001, c₂ = 0.9 and p = 0.5., x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
))Optimizer Constructor
Internally the constructor for Optimizer calls SimpleSolvers.OptimizerResult and SimpleSolvers.NewtonOptimizerState and Hessian. We can also allocate these objects manually and then call a different constructor for Optimizer:
using SimpleSolvers: NewtonOptimizerState, NewtonOptimizerCache, initialize!, linesearch_problem, Linesearch
_cache = NewtonOptimizerCache(x)
hes = Hessian(alg, obj, x)
grad = GradientAutodiff(obj.F, x)
state = NewtonOptimizerState(x)
update!(_cache, state, grad, x)
ls_obj = linesearch_problem(obj, grad, _cache)
ls_method = Static(.1)
ls = Linesearch(ls_obj, ls_method)
opt₂ = Optimizer(alg, obj, hes, _cache, ls_method)Optimizer{Float64, Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}, Linesearch{Float64, Static{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}}(Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069774219913e-310,6.9069752052666e-310,6.90703288896747e-310,6.90703288897143e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069753541756e-310,6.90697535417717e-310,6.9070367390274e-310,6.9069753564396e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069753564459e-310,6.9069753564475e-310,6.90703673902817e-310,6.9069753564491e-310)])), HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}(ForwardDiff.JacobianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,3.5e-323,6.90709765243935e-310,6.90709765244093e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(4.0e-323,5.0e-323,6.9070976524425e-310,6.9070976524441e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.4e-323,1.04e-322,6.90709765244567e-310,6.90709765244726e-310)]), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}}((Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0))), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069720628968e-310,6.9069720628984e-310,6.90697206290156e-310,6.90697206561774e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9071074048051e-310,6.9069756501984e-310,6.90697206274504e-310,6.90697206293793e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90697206291263e-310,6.9069720629142e-310,6.9069720629158e-310,6.90697206291737e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90697206291895e-310,6.90697206292054e-310,6.90697206569047e-310,1.94474e-319)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.9889e-319,6.90697206301223e-310,6.9069720630154e-310,6.90697206301856e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90697206302014e-310,6.9069720630233e-310,6.90710054774076e-310,6.9069720629743e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069720629474e-310,6.9069720629474e-310,6.9069735546747e-310,6.9069720630312e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90697206302646e-310,6.9069720630328e-310,6.90697206300907e-310,5.0e-324)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90697206303437e-310,6.90697206303595e-310,6.90697206303753e-310,6.9069720630391e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069720630407e-310,6.90697206304227e-310,6.90697206302804e-310,6.90697206304385e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90697206304544e-310,6.906972063047e-310,6.9069720630486e-310,6.9069720630502e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.90697206305176e-310,6.90697206305334e-310,1.0e-323,1.0e-323))]))), x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([0.521213795535383, 0.5868067574533484, 0.8908786980927811], [0.0, 0.0, 0.0], [1.042427591070766, 1.1736135149066969, -0.21824260381443783], [1.5e-323, 1.0e-323, 0.0], [-1.042427591070766, -1.1736135149066969, 0.21824260381443783], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]), Linesearch{Float64, Static{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}(LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}(SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([0.521213795535383, 0.5868067574533484, 0.8908786980927811], [0.0, 0.0, 0.0], [1.042427591070766, 1.1736135149066969, -0.21824260381443783], [1.5e-323, 1.0e-323, 0.0], [-1.042427591070766, -1.1736135149066969, 0.21824260381443783], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN])), SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069774219913e-310,6.9069752052666e-310,6.90703288896747e-310,6.90703288897143e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069753541756e-310,6.90697535417717e-310,6.9070367390274e-310,6.9069753564396e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9069753564459e-310,6.9069753564475e-310,6.90703673902817e-310,6.9069753564491e-310)])), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([0.521213795535383, 0.5868067574533484, 0.8908786980927811], [0.0, 0.0, 0.0], [1.042427591070766, 1.1736135149066969, -0.21824260381443783], [1.5e-323, 1.0e-323, 0.0], [-1.042427591070766, -1.1736135149066969, 0.21824260381443783], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]))), Static with α = 0.1., x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
))If we want to solve the problem, we can call solve! on the Optimizer instance:
x₀ = copy(x)
state = NewtonOptimizerState(x)
solve!(x₀, state, opt)SimpleSolvers.OptimizerResult{Float64, Float64, Vector{Float64}, SimpleSolvers.OptimizerStatus{Float64, Float64}}(
* Convergence measures
|x - x'| = 7.92e-01
|x - x'|/|x'| = 7.92e-01
|f(x) - f(x')| = 0.00e+00
|f(x) - f(x')|/|f(x')| = NaN
|g(x) - g(x')| = 0.00e+00
|g(x)| = 0.00e+00
, [0.0, 0.0, 1.0], 0.0)Internally SimpleSolvers.solve! repeatedly calls SimpleSolvers.solver_step! until SimpleSolvers.meets_stopping_criteria is satisfied.
using SimpleSolvers: solver_step!
x = rand(3)
solver_step!(x, state, opt)3-element Vector{Float64}:
0.0
0.0
1.0The function SimpleSolvers.solver_step! in turn does the following:
# update problem, hessian, state and result
update!(opt, state, x)
# solve H δx = - ∇f
# rhs is -g
ldiv!(direction(opt), hessian(opt), rhs(opt))
# apply line search
α = linesearch(opt)(linesearch_problem(problem(opt), gradient(opt), cache(opt), state))
# compute new minimizer
x .= compute_new_iterate(x, α, direction(opt))
cache(opt).x .= xSolving the Line Search Problem with Backtracking
Calling solve together with Linesearch (in this case SimpleSolvers.Backtracking) on an SimpleSolvers.LinesearchProblem in turn does:
α *= ls.pas long as the SimpleSolvers.SufficientDecreaseCondition isn't satisfied. This condition checks the following:
fₖ₊₁ ≤ sdc.fₖ + sdc.c₁ * αₖ * sdc.pₖ' * sdc.gradₖsdc is first allocated as:
ls = linesearch(opt)
α = method(ls).α₀
x₀ = zero(α)
grad = GradientAutodiff{Float64}(problem(opt).F, length(x))
params = (x = state.x̄, parameters = NullParameters())
sdc = SufficientDecreaseCondition(method(ls).c₁, ls_obj.F(0., params), ls_obj.D(0., params), alpha -> ls_obj.F(alpha, params))SimpleSolvers.SufficientDecreaseCondition{Float64, Main.var"#5#6"}(0.0001, 0.6279134497792394, 0.0, Main.var"#5#6"())