nonmonotone DRLS

Author: aldma

src/algorithms/drls.jl

@@ -43,6 +43,7 @@ See also: [`DRLS`](@ref).
 - `gamma`: stepsize to use, chosen appropriately based on Lf and mf by defaults.
 - `max_backtracks=20`: maximum number of line-search backtracks.
 - `directions=LBFGS(5)`: strategy to use to compute line-search directions.
+- `monotonicity=1`: parameter controlling the averaging scheme for nonmonotone linesearch; monotonicity ∈ (0,1], monotone scheme by default.
 
 # References
 1. Themelis, Stella, Patrinos, "Douglas-Rachford splitting and ADMM for nonconvex optimization: Accelerated and Newton-type linesearch algorithms", Computational Optimization and Applications, vol. 82, no. 2, pp. 395-440 (2022).
@@ -61,6 +62,7 @@ Base.@kwdef struct DRLSIteration{R,Tx,Tf,Tg,Tmf,TLf,D}
     dre_sign::Int = mf === nothing || mf <= 0 ? 1 : -1
     max_backtracks::Int = 20
     directions::D = LBFGS(5)
+    monotonicity::R = real(eltype(x0))(1)
 end
 
 Base.IteratorSize(::Type{<:DRLSIteration}) = Base.IsInfinite()
@@ -80,6 +82,7 @@ Base.@kwdef mutable struct DRLSState{R,Tx,TH}
     f_u::R
     g_v::R
     H::TH
+    merit::R = zero(gamma)
     tau::R = zero(gamma)
     u0::Tx = similar(x)
     u1::Tx = similar(x)
@@ -116,6 +119,8 @@ function Base.iterate(iter::DRLSIteration)
         g_v = g_v,
         H = initialize(iter.directions, x),
     )
+    # initialize merit
+    state.merit = DRE(state)
     return state, state
 end
 
@@ -141,7 +146,8 @@ update_direction_state!(iter::DRLSIteration, state::DRLSState) =
     update_direction_state!(acceleration_style(typeof(iter.directions)), iter, state)
 
 function Base.iterate(iter::DRLSIteration{R,Tx,Tf}, state::DRLSState) where {R,Tx,Tf}
-    DRE_curr = DRE(state)
+    # retrieve merit and set threshold
+    DRE_curr = state.merit
     threshold = iter.dre_sign * DRE_curr - iter.c / iter.gamma * norm(state.res)^2
 
     set_next_direction!(iter, state)
@@ -190,6 +196,8 @@ function Base.iterate(iter::DRLSIteration{R,Tx,Tf}, state::DRLSState) where {R,T
         state.res .= state.u .- state.v
         state.xbar .= state.x .- iter.lambda * state.res
     end
+    # update merit with averaging rule
+    state.merit = (1 - iter.monotonicity) * state.merit + iter.monotonicity * DRE(state)
 
     return state, state
 end

test/problems/test_lasso_small.jl

@@ -241,6 +241,23 @@ using ProximalAlgorithms:
         @test x0 == x0_backup
     end
 
+    @testset "DouglasRachford line search ($acc) (nonmonotone)" for (acc, maxit) in [
+        (LBFGS(5), 25),
+        (Broyden(), 20),
+        (AndersonAcceleration(5), 12),
+        (NesterovExtrapolation(FixedNesterovSequence), 60),
+        (NesterovExtrapolation(SimpleNesterovSequence), 50),
+    ]
+        x0 = zeros(T, n)
+        x0_backup = copy(x0)
+        solver = ProximalAlgorithms.DRLS(tol=10 * TOL, directions=acc, monotonicity=R(0.5))
+        z, it = @inferred solver(x0=x0, f=fA_prox, g=g, Lf=Lf)
+        @test eltype(z) == T
+        @test norm(z - x_star, Inf) <= 10 * TOL
+        @test it < maxit
+        @test x0 == x0_backup
+    end
+
     @testset "AFBA" begin
         x0 = zeros(T, n)
         x0_backup = copy(x0)

test/problems/test_lasso_small_strongly_convex.jl

@@ -152,6 +152,15 @@ using ProximalAlgorithms
         @test x0 == x0_backup
     end
 
+    @testset "DRLS (nonmonotone)" begin
+        solver = ProximalAlgorithms.DRLS(tol=TOL, monotonicity=T(0.1))
+        v, it = solver(x0=x0, f=fA_prox, g=g, mf=mf)
+        @test eltype(v) == T
+        @test norm(v - x_star, Inf) <= TOL
+        @test it < 14
+        @test x0 == x0_backup
+    end
+
     @testset "PANOC" begin
         solver = ProximalAlgorithms.PANOC(tol=TOL)
         y, it = solver(x0=x0, f=fA_autodiff, g=g, Lf=Lf)