Hessians

Hessians are a crucial ingredient in NewtonSolvers and SimpleSolvers.NewtonOptimizerStates.

using SimpleSolvers
using LinearAlgebra: norm

x = rand(3)
obj = OptimizerProblem(x -> norm(x - vcat(0., 0., 1.))  ^ 2, x)
hes = HessianAutodiff(obj, x)

HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}(ForwardDiff.JacobianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92235717677454e-310,6.92235717680576e-310,6.9223802036735e-310,6.92238022004527e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92235717678916e-310,6.9223571767868e-310,6.92238020333637e-310,6.9223571768101e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92235717680774e-310,6.92238019212305e-310,6.9223571768014e-310,6.9223571768038e-310)]), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}}((Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0))), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.9223887064737e-310,6.9223844309237e-310,6.9223887064737e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.9223887064737e-310,6.9223801897456e-310,6.9223847984184e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223482040788e-310,6.9223887064737e-310,6.9223844309237e-310,6.9223887064737e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.92238745139125e-310,6.9223337852148e-310,6.9223887064737e-310)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.9223887064737e-310,6.9223844309237e-310,6.9223887064737e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.9223887064737e-310,6.9223801723798e-310,6.9223847984184e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92234820408827e-310,6.9223887064737e-310,6.9223844309237e-310,6.9223887064737e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.92238745139125e-310,6.9223337852148e-310,6.9223887064737e-310)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.9223887064737e-310,6.9223844309237e-310,6.9223887064737e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9223844309237e-310,6.9223887064737e-310,6.9223801723798e-310,6.9223847984184e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92234820409775e-310,6.9223887064737e-310,6.92238770431096e-310,6.9223887064737e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.92238770431096e-310,6.9223887064737e-310,6.92238770431096e-310,6.9223887064737e-310))])))

The instance of HessianAutodiff can be called:

hes(x)

3×3 Matrix{Float64}:
 2.0          0.0           0.0
 0.0          2.0          -1.11022e-16
 5.55112e-17  1.11022e-16   2.0

Or alternative in-place:

H = SimpleSolvers.alloc_h(x)
hes(H, x)
H

3×3 Matrix{Float64}:
 2.0          0.0           0.0
 0.0          2.0          -1.11022e-16
 5.55112e-17  1.11022e-16   2.0