Comparison of Optimizers

using GeometricMachineLearning
using CairoMakie
import Random
Random.seed!(123)

f(x::Number, y::Number) = x ^ 2 + y ^ 2
function make_surface()
    n = 100
    r = √2
    u = range(-π, π; length = n)
    v = range(0, π; length = n)
    x = r * cos.(u) * sin.(v)'
    y = r * sin.(u) * sin.(v)'
    z = f.(x, y)
    x, y, z
end

fig = Figure()
ax = Axis3(fig[1, 1])
surface!(ax, make_surface()...; alpha = .3, transparency = true)

init_con = rand(2, 1)
init_cont = Tuple(init_con)
scatter!(ax, init_cont..., f(init_cont...); color = mred, marker = :star5)

weights = (xy = init_con, )
η = 1e-3
method1 = GradientMethod()
method2 = Adam()
optimizer1 = Optimizer(method1, weights; step_size = η)
optimizer2 = Optimizer(method2, weights; step_size = η)

Optimizer{Adam{Float64}, GeometricOptimizers.AdamCache{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}}, AdamState{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}, @NamedTuple{xy::Matrix{Float64}}}, typeof(cayley), Float64}(Adam{Float64}(0.9, 0.99, 1.0e-8), GeometricOptimizers.AdamCache{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}}((xy = [0.521213795535383; 0.5868067574533484;;],), (xy = [0.0; 0.0;;],), (xy = [0.0; 0.0;;],), (xy = [NaN; NaN;;],), (xy = [NaN; NaN;;],), Base.RefValue{Bool}(false), (xy = [6.92346037419824e-310; 6.92342657853733e-310;;],), (xy = [6.92333583893963e-310; 6.9233358389404e-310;;],), (xy = [6.923422636631e-310; 6.92333583895227e-310;;],), (xy = GlobalSection{Float64, Matrix{Float64}, Nothing}([0.521213795535383; 0.5868067574533484;;], nothing),)), AdamState{Float64, @NamedTuple{xy::Matrix{Float64}}, @NamedTuple{xy::GlobalSection{Float64, Matrix{Float64}, Nothing}}, @NamedTuple{xy::Matrix{Float64}}}((xy = GlobalSection{Float64, Matrix{Float64}, Nothing}([0.521213795535383; 0.5868067574533484;;], nothing),), 0, (xy = [0.521213795535383; 0.5868067574533484;;],), (xy = [6.92346037453104e-310; 1.0e-323;;],), (xy = [0.0; 0.0;;],), (xy = [6.9234077222279e-310; 6.92333698692447e-310;;],), (xy = [0.0; 0.0;;],), (xy = [0.0; 0.0;;],), (xy = [0.0; 0.0;;],), NaN, NaN), GeometricOptimizers.cayley, 0.001, 0)