Optimizers
An Optimizer stores an OptimizerState, a OptimizerProblem, the SimpleSolvers.OptimizerResult and a SimpleSolvers.NonlinearMethod. Its purposes are:
using SimpleSolvers
using LinearAlgebra: norm
x = rand(3)
obj = OptimizerProblem(x -> sum((x - [0., 0., 1.]) .^ 2), x)
bt = Backtracking()
alg = Newton()
opt = Optimizer(x, obj; algorithm = alg, linesearch = bt)Optimizer{Float64, Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}, Linesearch{Float64, Backtracking{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}}(Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,6.89860750091544e-310,5.0e-324,6.898607500917e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,6.8986075009186e-310,5.0e-324,6.8986075009202e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,6.89860750092176e-310,5.0e-324,6.89860750092335e-310)])), HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}(ForwardDiff.JacobianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,5.0e-324,6.8986077719322e-310,6.89860777193535e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0e-323,1.0e-323,6.89860777193693e-310,6.8986077719401e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.5e-323,2.0e-323,6.89860777194167e-310,6.89860777194325e-310)]), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}}((Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0))), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987056141773e-310,6.8987056141789e-310,6.89870561418046e-310,6.89870557764964e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987441435602e-310,6.8986705309505e-310,6.89870561414094e-310,6.89870561422e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89870561418837e-310,6.89870561418995e-310,6.89870561419153e-310,6.8987056141947e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89870561419627e-310,6.89870561419785e-310,6.8987056074833e-310,3.18e-321)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.9891e-319,6.89870561434015e-310,6.89870561434173e-310,6.8987056143433e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987056143449e-310,6.89870561434805e-310,6.8987550552087e-310,6.89870561431485e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.898705614337e-310,6.898705614337e-310,6.89873710748163e-310,6.8987056143528e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89870561434963e-310,6.89870561435437e-310,6.89870561433856e-310,5.0e-324)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89870561435596e-310,6.89870561435754e-310,6.8987056143591e-310,6.8987056143607e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987056143623e-310,6.89870561436386e-310,6.8987056143512e-310,6.89870561436544e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.898705614367e-310,6.8987056143686e-310,6.8987056143702e-310,6.89870561437177e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89870561437335e-310,6.89870561437493e-310,1.5e-323,2.5e-323))]))), x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([NaN, NaN, NaN], [NaN, NaN, NaN], [NaN, NaN, NaN], [1.4e-322, 1.53e-322, 1.7e-322], [NaN, NaN, NaN], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]), Linesearch{Float64, Backtracking{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}(LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}(SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([NaN, NaN, NaN], [NaN, NaN, NaN], [NaN, NaN, NaN], [1.4e-322, 1.53e-322, 1.7e-322], [NaN, NaN, NaN], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN])), SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,6.89860750091544e-310,5.0e-324,6.898607500917e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,6.8986075009186e-310,5.0e-324,6.8986075009202e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,6.89860750092176e-310,5.0e-324,6.89860750092335e-310)])), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([NaN, NaN, NaN], [NaN, NaN, NaN], [NaN, NaN, NaN], [1.4e-322, 1.53e-322, 1.7e-322], [NaN, NaN, NaN], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]))), Backtracking with α₀ = 1.0 c₁ = 0.0001, c₂ = 0.9 and p = 0.5., x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
))Optimizer Constructor
Internally the constructor for Optimizer calls SimpleSolvers.OptimizerResult and SimpleSolvers.NewtonOptimizerState and Hessian. We can also allocate these objects manually and then call a different constructor for Optimizer:
using SimpleSolvers: NewtonOptimizerState, NewtonOptimizerCache, initialize!, linesearch_problem, Linesearch
_cache = NewtonOptimizerCache(x)
hes = Hessian(alg, obj, x)
grad = GradientAutodiff(obj.F, x)
state = NewtonOptimizerState(x)
update!(_cache, state, grad, x)
ls_obj = linesearch_problem(obj, grad, _cache)
ls_method = Static(.1)
ls = Linesearch(ls_obj, ls_method)
opt₂ = Optimizer(alg, obj, hes, _cache, ls_method)Optimizer{Float64, Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}, Linesearch{Float64, Static{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}}(Newton, OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710748163e-310,6.89873710748163e-310,6.89873710773815e-310,6.89873710748163e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710748163e-310,6.89873710999583e-310,6.89873710748163e-310,6.89873710748163e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710748163e-310,6.89873710748163e-310,6.89873710748163e-310,6.89873711019504e-310)])), HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}(ForwardDiff.JacobianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,5.0e-324,5.0e-324,1.0e-323), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,5.0e-324,5.0e-324,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)]), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}}((Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0))), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987371078267e-310,6.8987371078488e-310,6.89873710787095e-310,6.8987371078931e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987371079152e-310,6.89873710793735e-310,6.8987371079595e-310,6.8987371079816e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710800376e-310,6.8987371080259e-310,6.89873710804803e-310,6.89873710807016e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987371080923e-310,6.89873710811443e-310,6.89873710813656e-310,6.8987371081587e-310)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710818083e-310,6.89873710820296e-310,6.8987371082251e-310,6.89873710824723e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710826937e-310,6.8987371082915e-310,6.89873710831364e-310,6.89873710833577e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987371083579e-310,6.89873710838004e-310,6.89873710840217e-310,6.8987371084243e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710844644e-310,6.89873710846857e-310,6.8987371084907e-310,6.89873710851284e-310)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.898737108535e-310,6.8987371085571e-310,6.89873710857925e-310,6.8987371086014e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987371086235e-310,6.89873710864565e-310,6.8987371086678e-310,6.8987371086899e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710871205e-310,6.8987371087342e-310,6.8987371087563e-310,6.89873710877845e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.8987371088006e-310,6.8987371088227e-310,6.89873710884485e-310,6.898737108867e-310))]))), x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([0.521213795535383, 0.5868067574533484, 0.8908786980927811], [0.0, 0.0, 0.0], [1.042427591070766, 1.1736135149066969, -0.21824260381443783], [6.8986100332983e-310, 0.0, 0.0], [-1.042427591070766, -1.1736135149066969, 0.21824260381443783], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]), Linesearch{Float64, Static{Float64}, LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}, Options{Float64}}(LinesearchProblem{Float64, SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}, SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}}(SimpleSolvers.var"#f#linesearch_problem##2"{OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(OptimizerProblem{Float64, Main.var"#2#3", Missing, Missing}(Main.var"#2#3"(), missing, missing), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([0.521213795535383, 0.5868067574533484, 0.8908786980927811], [0.0, 0.0, 0.0], [1.042427591070766, 1.1736135149066969, -0.21824260381443783], [6.8986100332983e-310, 0.0, 0.0], [-1.042427591070766, -1.1736135149066969, 0.21824260381443783], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN])), SimpleSolvers.var"#d#linesearch_problem##3"{GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}, SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}}(GradientAutodiff{Float64, Main.var"#2#3", ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710748163e-310,6.89873710748163e-310,6.89873710773815e-310,6.89873710748163e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710748163e-310,6.89873710999583e-310,6.89873710748163e-310,6.89873710748163e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.89873710748163e-310,6.89873710748163e-310,6.89873710748163e-310,6.89873711019504e-310)])), SimpleSolvers.NewtonOptimizerCache{Float64, Vector{Float64}, Matrix{Float64}}([0.521213795535383, 0.5868067574533484, 0.8908786980927811], [0.0, 0.0, 0.0], [1.042427591070766, 1.1736135149066969, -0.21824260381443783], [6.8986100332983e-310, 0.0, 0.0], [-1.042427591070766, -1.1736135149066969, 0.21824260381443783], [NaN NaN NaN; NaN NaN NaN; NaN NaN NaN]))), Static with α = 0.1., x_abstol = 4.440892098500626e-16
x_reltol = 4.440892098500626e-16
x_suctol = 4.440892098500626e-16
f_abstol = 0.0
f_reltol = 4.440892098500626e-16
f_suctol = 4.440892098500626e-16
f_mindec = 0.0001
g_restol = 1.4901161193847656e-8
x_abstol_break = Inf
x_reltol_break = Inf
f_abstol_break = Inf
f_reltol_break = Inf
g_restol_break = Inf
allow_f_increases = true
min_iterations = 0
max_iterations = 1000
warn_iterations = 1000
show_trace = false
store_trace = false
extended_trace = false
show_every = 1
verbosity = 1
nan_max_iterations = 10
nan_factor = 0.5
regularization_factor = 0.0
))If we want to solve the problem, we can call solve! on the Optimizer instance:
x₀ = copy(x)
state = NewtonOptimizerState(x)
solve!(x₀, state, opt)SimpleSolvers.OptimizerResult{Float64, Float64, Vector{Float64}, SimpleSolvers.OptimizerStatus{Float64, Float64}}(
* Convergence measures
|x - x'| = 7.92e-01
|x - x'|/|x'| = 7.92e-01
|f(x) - f(x')| = 0.00e+00
|f(x) - f(x')|/|f(x')| = NaN
|g(x) - g(x')| = 0.00e+00
|g(x)| = 0.00e+00
, [0.0, 0.0, 1.0], 0.0)Internally SimpleSolvers.solve! repeatedly calls SimpleSolvers.solver_step! until SimpleSolvers.meets_stopping_criteria is satisfied.
using SimpleSolvers: solver_step!
x = rand(3)
solver_step!(x, state, opt)3-element Vector{Float64}:
0.0
0.0
1.0The function SimpleSolvers.solver_step! in turn does the following:
# update problem, hessian, state and result
update!(opt, state, x)
# solve H δx = - ∇f
# rhs is -g
ldiv!(direction(opt), hessian(opt), rhs(opt))
# apply line search
α = linesearch(opt)(linesearch_problem(problem(opt), gradient(opt), cache(opt), state))
# compute new minimizer
x .= compute_new_iterate(x, α, direction(opt))
cache(opt).x .= xSolving the Line Search Problem with Backtracking
Calling solve together with Linesearch (in this case SimpleSolvers.Backtracking) on an SimpleSolvers.LinesearchProblem in turn does:
α *= ls.pas long as the SimpleSolvers.SufficientDecreaseCondition isn't satisfied. This condition checks the following:
fₖ₊₁ ≤ sdc.fₖ + sdc.c₁ * αₖ * sdc.pₖ' * sdc.gradₖsdc is first allocated as:
ls = linesearch(opt)
α = method(ls).α₀
x₀ = zero(α)
grad = GradientAutodiff{Float64}(problem(opt).F, length(x))
params = (x = state.x̄, parameters = NullParameters())
sdc = SufficientDecreaseCondition(method(ls).c₁, ls_obj.F(0., params), ls_obj.D(0., params), alpha -> ls_obj.F(alpha, params))SimpleSolvers.SufficientDecreaseCondition{Float64, Main.var"#5#6"}(0.0001, 0.6279134497792394, 0.0, Main.var"#5#6"())