Comparison of Optimizers

using GeometricMachineLearning
using CairoMakie
import Random
Random.seed!(123)

f(x::Number, y::Number) = x ^ 2 + y ^ 2
function make_surface()
    n = 100
    r = √2
    u = range(-π, π; length = n)
    v = range(0, π; length = n)
    x = r * cos.(u) * sin.(v)'
    y = r * sin.(u) * sin.(v)'
    z = f.(x, y)
    x, y, z
end

fig = Figure()
ax = Axis3(fig[1, 1])
surface!(ax, make_surface()...; alpha = .3, transparency = true)

init_con = rand(2, 1)
init_cont = Tuple(init_con)
scatter!(ax, init_cont..., f(init_cont...); color = mred, marker = :star5)

weights = (xy = init_con, )
η = 1e-3
method1 = GradientMethod()
method2 = Adam()
optimizer1 = Optimizer(method1, weights; step_size = η)
optimizer2 = Optimizer(method2, weights; step_size = η)

Optimizer{Adam{Float64}, GeometricOptimizers.AdamCache{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}}, AdamState{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}, @NamedTuple{xy::Matrix{Float64}}}, typeof(cayley), Float64}(Adam{Float64}(0.9, 0.99, 1.0e-8), GeometricOptimizers.AdamCache{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}}((xy = [0.521213795535383; 0.5868067574533484;;],), (xy = [0.0; 0.0;;],), (xy = [0.0; 0.0;;],), (xy = [NaN; NaN;;],), (xy = [NaN; NaN;;],), Base.RefValue{Bool}(false), (xy = [0.0; 6.91982827433775e-310;;],), (xy = [6.91989733654833e-310; 6.9199015757375e-310;;],), (xy = [6.91975894202346e-310; 6.91976984373687e-310;;],), (xy = GlobalSection{Float64, Matrix{Float64}, Nothing}([0.521213795535383; 0.5868067574533484;;], nothing),)), AdamState{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}, @NamedTuple{xy::Matrix{Float64}}}((xy = GlobalSection{Float64, Matrix{Float64}, Nothing}([0.521213795535383; 0.5868067574533484;;], nothing),), 0, (xy = [0.521213795535383; 0.5868067574533484;;],), (xy = [6.91990027046903e-310; 6.9197721547807e-310;;],), (xy = [0.0; 0.0;;],), (xy = [6.9198996160178e-310; 6.91977215478703e-310;;],), (xy = [0.0; 0.0;;],), (xy = [0.0; 0.0;;],), (xy = [0.0; 0.0;;],), NaN, NaN), GeometricOptimizers.cayley, 0.001, 0)