Hessians

Hessians are a crucial ingredient in NewtonSolvers and SimpleSolvers.NewtonOptimizerStates.

using SimpleSolvers
using LinearAlgebra: norm

x = rand(3)
obj = OptimizerProblem(x -> norm(x - vcat(0., 0., 1.))  ^ 2, x)
hes = HessianAutodiff(obj, x)

HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}(ForwardDiff.JacobianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9282796820863e-310,6.92828673844052e-310,6.92828674081994e-310,6.92823458887486e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.928234588878e-310,6.9282345888812e-310,6.92823458888592e-310,6.92828673832985e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9282867395994e-310,6.92823458889067e-310,6.9282796824756e-310,6.92823458889383e-310)]), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}}((Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0))), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92824905445927e-310,6.92824905446085e-310,6.92824905446243e-310,6.92824904717555e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92828635997913e-310,6.9282491716058e-310,6.92824905434227e-310,6.9282490545162e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92824905447033e-310,6.9282490544719e-310,6.9282490544735e-310,6.9282490544751e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92824905447666e-310,6.92824905447824e-310,6.9282490512577e-310,1.91075e-319)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.9341e-319,6.92824905452093e-310,6.9282490545225e-310,6.9282490545241e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92824905452567e-310,6.92824905452725e-310,6.92828094487024e-310,6.92824905434227e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92824905451777e-310,6.92824905451777e-310,6.9282796820863e-310,6.928249054532e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92824905452883e-310,6.92824905453358e-310,6.92824905451935e-310,5.0e-324)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92824905453516e-310,6.92824905453674e-310,6.9282490545383e-310,6.9282490545399e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9282490545415e-310,6.92824905454306e-310,6.9282490545304e-310,6.92824905454464e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9282490545494e-310,6.92824905455097e-310,6.92824905455413e-310,6.9282490545557e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9282490545573e-310,6.92824905455887e-310,4.4e-323,4.4e-323))])))

The instance of HessianAutodiff can be called:

hes(x)

3×3 Matrix{Float64}:
 2.0          0.0           0.0
 0.0          2.0          -1.11022e-16
 5.55112e-17  1.11022e-16   2.0

Or alternative in-place:

H = SimpleSolvers.alloc_h(x)
hes(H, x)
H

3×3 Matrix{Float64}:
 2.0          0.0           0.0
 0.0          2.0          -1.11022e-16
 5.55112e-17  1.11022e-16   2.0