Hessians

Hessians are a crucial ingredient in NewtonSolvers and SimpleSolvers.NewtonOptimizerStates.

using SimpleSolvers
using LinearAlgebra: norm

x = rand(3)
obj = OptimizerProblem(x -> norm(x - vcat(0., 0., 1.))  ^ 2, x)
hes = HessianAutodiff(obj, x)

HessianAutodiff{Float64, Main.var"#2#3", ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}}(Main.var"#2#3"(), ForwardDiff.HessianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}(ForwardDiff.JacobianConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}}}((Partials(1.0, 0.0, 0.0), Partials(0.0, 1.0, 0.0), Partials(0.0, 0.0, 1.0)), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94578438593945e-310,6.94575216133576e-310,6.94579640384953e-310,6.94575174143766e-310), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(5.0e-324,1.9324e-319,2.130317216e-314,7.746817145885188e-304), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94573728732037e-310,6.945788661946e-310,6.9457484062985e-310,6.94575174143055e-310)]), ForwardDiff.GradientConfig{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}}}((Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0)), Partials(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,0.0,0.0,0.0), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(1.0,0.0,0.0,0.0))), ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, ForwardDiff.Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}, Float64, 3}, 3}[Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94578447602145e-310,6.94578447632343e-310,6.94578447632105e-310,6.94578447601473e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9457844760167e-310,6.9457844761444e-310,6.94578447614635e-310,6.94578447612066e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94578447612264e-310,6.94578447693883e-310,6.9457844769412e-310,6.9457844769345e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94574944567745e-310,6.94579127279203e-310,1.4e-322,6.9457387041346e-310)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.945784475793e-310,6.945784386126e-310,6.94578438612404e-310,6.9457844758021e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94578447579735e-310,6.94578447308473e-310,6.94578447589695e-310,6.9457844759183e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9457844759163e-310,6.9457844681575e-310,6.94578447591434e-310,6.94578447591236e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94578447591e-310,6.9457844759076e-310,6.94578447590446e-310,6.9457844759017e-310)), Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94578447589893e-310,6.94578447577205e-310,6.945784415564e-310,6.9457844758946e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.94578441256366e-310,6.9457844758922e-310,6.94578447588983e-310,6.94578447588746e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(6.9457844758855e-310,6.9457844758835e-310,6.94578441646125e-310,6.94578447588153e-310),Dual{ForwardDiff.Tag{Main.var"#2#3", Float64}}(0.0,6.94579127279203e-310,1.4e-322,6.94573870414723e-310))])))

The instance of HessianAutodiff can be called:

hes(x)

3×3 Matrix{Float64}:
 2.0          0.0           0.0
 0.0          2.0          -1.11022e-16
 5.55112e-17  1.11022e-16   2.0

Or alternative in-place:

H = SimpleSolvers.alloc_h(x)
hes(H, x)
H

3×3 Matrix{Float64}:
 2.0          0.0           0.0
 0.0          2.0          -1.11022e-16
 5.55112e-17  1.11022e-16   2.0