Skip to content

Commit 9af4c2c

Browse files
aldmaaldma
committed
nonmonotone PANOCplus
1 parent 01f4995 commit 9af4c2c

File tree

5 files changed

+109
-41
lines changed

5 files changed

+109
-41
lines changed

‎src/algorithms/panocplus.jl‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ See also: [`PANOCplus`](@ref).
3232
- `minimum_gamma=1e-7`: lower bound to `gamma` in case `adaptive == true`.
3333
- `max_backtracks=20`: maximum number of line-search backtracks.
3434
- `directions=LBFGS(5)`: strategy to use to compute line-search directions.
35+
- `monotonicity=1`: parameter controlling the averaging scheme for nonmonotone linesearch; monotonicity ∈ (0,1], monotone scheme by default.
3536
3637
# References
3738
1. De Marchi, Themelis, "Proximal Gradient Algorithms under Local Lipschitz Gradient Continuity", Journal of Optimization Theory and Applications, vol. 194, no. 3, pp. 771-794 (2022).
@@ -49,6 +50,7 @@ Base.@kwdef struct PANOCplusIteration{R,Tx,Tf,TA,Tg,TLf,Tgamma,D}
4950
minimum_gamma::R = real(eltype(x0))(1e-7)
5051
max_backtracks::Int = 20
5152
directions::D = LBFGS(5)
53+
monotonicity::R = real(eltype(x0))(1)
5254
end
5355

5456
Base.IteratorSize(::Type{<:PANOCplusIteration}) = Base.IsInfinite()
@@ -65,6 +67,7 @@ Base.@kwdef mutable struct PANOCplusState{R,Tx,TAx,TH}
6567
g_z::R # value of nonsmooth term (at z)
6668
res::Tx # fixed-point residual at iterate (= x - z)
6769
H::TH # variable metric
70+
merit::R = zero(gamma)
6871
tau::R = zero(gamma)
6972
x_prev::Tx = similar(x)
7073
res_prev::Tx = similar(x)
@@ -125,6 +128,8 @@ function Base.iterate(iter::PANOCplusIteration{R}) where {R}
125128
state.grad_f_Az = grad_f_Az
126129
end
127130
mul!(state.At_grad_f_Az, adjoint(iter.A), state.grad_f_Az)
131+
# initialize merit
132+
state.merit = f_model(iter, state) + state.g_z
128133
return state, state
129134
end
130135

@@ -170,9 +175,8 @@ function Base.iterate(iter::PANOCplusIteration{R}, state::PANOCplusState) where
170175
state.x_prev .= state.x
171176
state.res_prev .= state.res
172177

173-
# compute FBE
174-
FBE_x = f_model(iter, state) + state.g_z
175-
178+
# retrieve merit and set threshold
179+
FBE_x = state.merit
176180
sigma = iter.beta * (0.5 / state.gamma) * (1 - iter.alpha)
177181
tol = 10 * eps(R) * (1 + abs(FBE_x))
178182
threshold = FBE_x - sigma * norm(state.res)^2 + tol
@@ -226,6 +230,8 @@ function Base.iterate(iter::PANOCplusIteration{R}, state::PANOCplusState) where
226230

227231
FBE_x_new = f_Az_upp + state.g_z
228232
if FBE_x_new <= threshold || tau_backtracks >= iter.max_backtracks
233+
# update merit with averaging rule
234+
state.merit = (1 - iter.monotonicity) * state.merit + iter.monotonicity * FBE_x_new
229235
break
230236
end
231237
state.tau = tau_backtracks >= iter.max_backtracks - 1 ? R(0) : state.tau / 2
@@ -280,13 +286,13 @@ See also: [`PANOCplusIteration`](@ref), [`IterativeAlgorithm`](@ref).
280286
1. De Marchi, Themelis, "Proximal Gradient Algorithms under Local Lipschitz Gradient Continuity", Journal of Optimization Theory and Applications, vol. 194, no. 3, pp. 771-794 (2022).
281287
"""
282288
PANOCplus(;
283-
maxit = 1_000,
284-
tol = 1e-8,
285-
stop = (iter, state) -> default_stopping_criterion(tol, iter, state),
286-
solution = default_solution,
287-
verbose = false,
288-
freq = 10,
289-
display = default_display,
289+
maxit=1_000,
290+
tol=1e-8,
291+
stop=(iter, state) -> default_stopping_criterion(tol, iter, state),
292+
solution=default_solution,
293+
verbose=false,
294+
freq=10,
295+
display=default_display,
290296
kwargs...,
291297
) = IterativeAlgorithm(
292298
PANOCplusIteration;

‎test/problems/test_lasso_small.jl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,17 @@ using ProximalAlgorithms:
202202
@test x0 == x0_backup
203203
end
204204

205+
@testset "PANOCplus (adaptive step, nonmonotone)" begin
206+
x0 = zeros(T, n)
207+
x0_backup = copy(x0)
208+
solver = ProximalAlgorithms.PANOCplus(adaptive = true, tol = TOL, monotonicity=R(0.1))
209+
x, it = @inferred solver(x0 = x0, f = f_autodiff, A = A, g = g)
210+
@test eltype(x) == T
211+
@test norm(x - x_star, Inf) <= TOL
212+
@test it < 40
213+
@test x0 == x0_backup
214+
end
215+
205216
@testset "DouglasRachford" begin
206217
x0 = zeros(T, n)
207218
x0_backup = copy(x0)

‎test/problems/test_lasso_small_strongly_convex.jl‎

Lines changed: 40 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -54,26 +54,26 @@ using ProximalAlgorithms
5454
x0_backup = copy(x0)
5555

5656
@testset "SFISTA" begin
57-
solver = ProximalAlgorithms.SFISTA(tol = TOL)
58-
y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf, mf = mf)
57+
solver = ProximalAlgorithms.SFISTA(tol=TOL)
58+
y, it = solver(x0=x0, f=fA_autodiff, g=g, Lf=Lf, mf=mf)
5959
@test eltype(y) == T
6060
@test norm(y - x_star) <= TOL
6161
@test it < 40
6262
@test x0 == x0_backup
6363
end
6464

6565
@testset "ForwardBackward" begin
66-
solver = ProximalAlgorithms.ForwardBackward(tol = TOL)
67-
y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf)
66+
solver = ProximalAlgorithms.ForwardBackward(tol=TOL)
67+
y, it = solver(x0=x0, f=fA_autodiff, g=g, Lf=Lf)
6868
@test eltype(y) == T
6969
@test norm(y - x_star, Inf) <= TOL
7070
@test it < 110
7171
@test x0 == x0_backup
7272
end
7373

7474
@testset "ForwardBackward (adaptive step)" begin
75-
solver = ProximalAlgorithms.ForwardBackward(tol = TOL, adaptive = true)
76-
y, it = solver(x0 = x0, f = fA_autodiff, g = g)
75+
solver = ProximalAlgorithms.ForwardBackward(tol=TOL, adaptive=true)
76+
y, it = solver(x0=x0, f=fA_autodiff, g=g)
7777
@test eltype(y) == T
7878
@test norm(y - x_star, Inf) <= TOL
7979
@test it < 300
@@ -82,29 +82,29 @@ using ProximalAlgorithms
8282

8383
@testset "ForwardBackward (adaptive step, regret)" begin
8484
solver = ProximalAlgorithms.ForwardBackward(
85-
tol = TOL,
86-
adaptive = true,
87-
increase_gamma = T(1.01),
85+
tol=TOL,
86+
adaptive=true,
87+
increase_gamma=T(1.01),
8888
)
89-
y, it = solver(x0 = x0, f = fA_autodiff, g = g)
89+
y, it = solver(x0=x0, f=fA_autodiff, g=g)
9090
@test eltype(y) == T
9191
@test norm(y - x_star, Inf) <= TOL
9292
@test it < 80
9393
@test x0 == x0_backup
9494
end
9595

9696
@testset "FastForwardBackward" begin
97-
solver = ProximalAlgorithms.FastForwardBackward(tol = TOL)
98-
y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf, mf = mf)
97+
solver = ProximalAlgorithms.FastForwardBackward(tol=TOL)
98+
y, it = solver(x0=x0, f=fA_autodiff, g=g, Lf=Lf, mf=mf)
9999
@test eltype(y) == T
100100
@test norm(y - x_star, Inf) <= TOL
101101
@test it < 35
102102
@test x0 == x0_backup
103103
end
104104

105105
@testset "FastForwardBackward (adaptive step)" begin
106-
solver = ProximalAlgorithms.FastForwardBackward(tol = TOL, adaptive = true)
107-
y, it = solver(x0 = x0, f = fA_autodiff, g = g)
106+
solver = ProximalAlgorithms.FastForwardBackward(tol=TOL, adaptive=true)
107+
y, it = solver(x0=x0, f=fA_autodiff, g=g)
108108
@test eltype(y) == T
109109
@test norm(y - x_star, Inf) <= TOL
110110
@test it < 100
@@ -113,26 +113,26 @@ using ProximalAlgorithms
113113

114114
@testset "FastForwardBackward (adaptive step, regret)" begin
115115
solver = ProximalAlgorithms.FastForwardBackward(
116-
tol = TOL,
117-
adaptive = true,
118-
increase_gamma = T(1.01),
116+
tol=TOL,
117+
adaptive=true,
118+
increase_gamma=T(1.01),
119119
)
120-
y, it = solver(x0 = x0, f = fA_autodiff, g = g)
120+
y, it = solver(x0=x0, f=fA_autodiff, g=g)
121121
@test eltype(y) == T
122122
@test norm(y - x_star, Inf) <= TOL
123123
@test it < 100
124124
@test x0 == x0_backup
125125
end
126126

127127
@testset "FastForwardBackward (custom extrapolation)" begin
128-
solver = ProximalAlgorithms.FastForwardBackward(tol = TOL)
128+
solver = ProximalAlgorithms.FastForwardBackward(tol=TOL)
129129
y, it = solver(
130-
x0 = x0,
131-
f = fA_autodiff,
132-
g = g,
133-
gamma = 1 / Lf,
134-
mf = mf,
135-
extrapolation_sequence = ProximalAlgorithms.ConstantNesterovSequence(
130+
x0=x0,
131+
f=fA_autodiff,
132+
g=g,
133+
gamma=1 / Lf,
134+
mf=mf,
135+
extrapolation_sequence=ProximalAlgorithms.ConstantNesterovSequence(
136136
mf,
137137
1 / Lf,
138138
),
@@ -144,26 +144,35 @@ using ProximalAlgorithms
144144
end
145145

146146
@testset "DRLS" begin
147-
solver = ProximalAlgorithms.DRLS(tol = TOL)
148-
v, it = solver(x0 = x0, f = fA_prox, g = g, mf = mf)
147+
solver = ProximalAlgorithms.DRLS(tol=TOL)
148+
v, it = solver(x0=x0, f=fA_prox, g=g, mf=mf)
149149
@test eltype(v) == T
150150
@test norm(v - x_star, Inf) <= TOL
151151
@test it < 14
152152
@test x0 == x0_backup
153153
end
154154

155155
@testset "PANOC" begin
156-
solver = ProximalAlgorithms.PANOC(tol = TOL)
157-
y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf)
156+
solver = ProximalAlgorithms.PANOC(tol=TOL)
157+
y, it = solver(x0=x0, f=fA_autodiff, g=g, Lf=Lf)
158158
@test eltype(y) == T
159159
@test norm(y - x_star, Inf) <= TOL
160160
@test it < 45
161161
@test x0 == x0_backup
162162
end
163163

164164
@testset "PANOCplus" begin
165-
solver = ProximalAlgorithms.PANOCplus(tol = TOL)
166-
y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf)
165+
solver = ProximalAlgorithms.PANOCplus(tol=TOL)
166+
y, it = solver(x0=x0, f=fA_autodiff, g=g, Lf=Lf)
167+
@test eltype(y) == T
168+
@test norm(y - x_star, Inf) <= TOL
169+
@test it < 45
170+
@test x0 == x0_backup
171+
end
172+
173+
@testset "PANOCplus (nonmonotone)" begin
174+
solver = ProximalAlgorithms.PANOCplus(tol=TOL, monotonicity=T(0.1))
175+
y, it = solver(x0=x0, f=fA_autodiff, g=g, Lf=Lf)
167176
@test eltype(y) == T
168177
@test norm(y - x_star, Inf) <= TOL
169178
@test it < 45

‎test/problems/test_nonconvex_qp.jl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,16 @@ using Test
4545
@test x0 == x0_backup
4646
end
4747

48+
@testset "PANOCplus (nonmonotone)" begin
49+
x0 = zeros(T, n)
50+
x0_backup = copy(x0)
51+
solver = ProximalAlgorithms.PANOCplus(tol = TOL, monotonicity=T(0.1))
52+
x, it = solver(x0 = x0, f = f, g = g)
53+
z = min.(upp, max.(low, x .- gamma .* (Q * x + q)))
54+
@test norm(x - z, Inf) / gamma <= TOL
55+
@test x0 == x0_backup
56+
end
57+
4858
@testset "ZeroFPR" begin
4959
x0 = zeros(T, n)
5060
x0_backup = copy(x0)
@@ -112,6 +122,16 @@ end
112122
@test x0 == x0_backup
113123
end
114124

125+
@testset "PANOCplus (nonmonotone)" begin
126+
x0 = zeros(T, n)
127+
x0_backup = copy(x0)
128+
solver = ProximalAlgorithms.PANOCplus(tol = TOL, monotonicity=T(0.1))
129+
x, it = solver(x0 = x0, f = f, g = g)
130+
z = min.(upp, max.(low, x .- gamma .* (Q * x + q)))
131+
@test norm(x - z, Inf) / gamma <= TOL
132+
@test x0 == x0_backup
133+
end
134+
115135
@testset "ZeroFPR" begin
116136
x0 = zeros(T, n)
117137
x0_backup = copy(x0)

‎test/problems/test_sparse_logistic_small.jl‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,4 +120,26 @@ using LinearAlgebra
120120
@test x0 == x0_backup
121121
end
122122

123+
@testset "PANOCplus (adaptive step, nonmonotone)" begin
124+
x0 = zeros(T, n)
125+
x0_backup = copy(x0)
126+
solver = ProximalAlgorithms.PANOCplus(adaptive = true, tol = TOL, monotonicity=R(0.9))
127+
x, it = solver(x0 = x0, f = f_autodiff, A = A, g = g)
128+
@test eltype(x) == T
129+
@test norm(x - x_star, Inf) <= 1e-4
130+
@test it < 50
131+
@test x0 == x0_backup
132+
end
133+
134+
@testset "PANOCplus (adaptive step, very nonmonotone)" begin
135+
x0 = zeros(T, n)
136+
x0_backup = copy(x0)
137+
solver = ProximalAlgorithms.PANOCplus(adaptive = true, tol = TOL, monotonicity=R(0.1))
138+
x, it = solver(x0 = x0, f = f_autodiff, A = A, g = g)
139+
@test eltype(x) == T
140+
@test norm(x - x_star, Inf) <= 1e-4
141+
@test it < 110
142+
@test x0 == x0_backup
143+
end
144+
123145
end

0 commit comments

Comments
 (0)