Merge branch 'master' into 23Q3/dev/parametric

Author: Affie

NEWS.md

@@ -17,7 +17,7 @@ The list below highlights breaking changes according to normal semver workflow -
 - Deprecate `AbstractRelativeRoots`.
 - Standardization improvements surrounding weakdeps code extensions. 
 - Code quality improvements along wiht refactoring and reorganizing of file names and locations.
-- Restoring `DERelative` factors, although further fixes necessary beyond anticipated patch release v0.34.1.
+- Restoring `DERelative` factors, through v0.34.1 and v0.34.2.
 - Switching to weakdep AMD.jl for `ccolmod` dependency, part of Julia 1.10 upgrade.  Dropping `SuiteSparse_long` dependency.  Further fixes necessary to restore full user constrained tree variable order functionality.
 
 # Changes in v0.33

src/entities/ExtFactors.jl

@@ -19,7 +19,7 @@ DevNotes
 - FIXME Lots of consolidation and standardization to do, see RoME.jl #244 regarding Manifolds.jl.
 - TODO does not yet handle case where a factor spans across two timezones.
 """
-struct DERelative{T <: InferenceVariable, P, D} <: AbstractRelativeMinimize
+struct DERelative{T <: InferenceVariable, P, D} <: AbstractManifoldMinimize # AbstractRelativeMinimize
   domain::Type{T}
   forwardProblem::P
   backwardProblem::P

src/services/GraphInit.jl

@@ -123,10 +123,11 @@ Notes:
 - Special carve out for multihypo cases, see issue 427.
 
 Development Notes:
-> Target factor is first (singletons) or second (dim 2 pairwise) variable vertex in `xi`.
-* TODO use DFG properly with local operations and DB update at end.
-* TODO get faster version of `isInitialized` for database version.
-* TODO: Persist this back if we want to here.
+- Target factor is first (singletons) or second (dim 2 pairwise) variable vertex in `xi`.
+- TODO use DFG properly with local operations and DB update at end.
+- TODO get faster version of `isInitialized` for database version.
+- TODO: Persist this back if we want to here.
+- TODO: init from just partials
 """
 function doautoinit!(
   dfg::AbstractDFG,

src/services/GraphProductOperations.jl

@@ -36,11 +36,12 @@ function propagateBelief(
   # make sure oldPoints vector has right length
   oldBel = getBelief(dfg, destlbl, solveKey)
   _pts = getPoints(oldBel, false)
-  oldPoints = if Npts(oldBel) <= N
-    _pts[1:N]
-  else
+  oldPoints = if Npts(oldBel) < N
     nn = N - length(_pts) # should be larger than 0
-    vcat(_pts, sample(oldBel, nn))
+    _pts_, = sample(oldBel, nn)
+    vcat(_pts, _pts_)
+  else
+    _pts[1:N]
   end
 
   # few more data requirements

src/services/NumericalCalculations.jl

@@ -362,10 +362,9 @@ function _solveCCWNumeric!(
   perturb::Real = 1e-10,
 ) where {N_, F <: AbstractRelative, S, T}
   #
-
+  
   #
   # thrid = Threads.threadid()
-
   smpid = ccwl.particleidx[]
   # cannot Nelder-Mead on 1dim, partial can be 1dim or more but being conservative.
   islen1 = length(ccwl.partialDims) == 1 || ccwl.partial
@@ -377,10 +376,11 @@ function _solveCCWNumeric!(
   #  a separate deepcopy of the destination (aka target) memory is necessary.
   #  Choosen solution is to splice together ccwl.varValsAll each time, with destination as 
   #  deepcopy but other input variables are just point to the source variable values directly. 
-  if ccwl.partial
-    target = view(ccwl.varValsAll[][ccwl.varidx[]][smpid], ccwl.partialDims)
+  target = if ccwl.partial  # FIXME likely type-instability on `typeof(target)`
+    # view(ccwl.varValsAll[][ccwl.varidx[]][smpid], ccwl.partialDims)
+    ccwl.varValsAll[][ccwl.varidx[]][smpid][ccwl.partialDims]
   else
-    target = ccwl.varValsAll[][ccwl.varidx[]][smpid];
+    ccwl.varValsAll[][ccwl.varidx[]][smpid]
   end
   # build the pre-objective function for this sample's hypothesis selection
   unrollHypo!, _ = _buildCalcFactorLambdaSample(
@@ -407,12 +407,13 @@ function _solveCCWNumeric!(
   # target .+= _perturbIfNecessary(getFactorType(ccwl), length(target), perturb)
   sfidx = ccwl.varidx[]
   # do the parameter search over defined decision variables using Minimization
-  if ccwl.partial
-    X = collect(view(ccwl.varValsAll[][sfidx][smpid], ccwl.partialDims))
-  else
-    X = ccwl.varValsAll[][sfidx][smpid][ccwl.partialDims]
-  end
-  # X = destVarVals[smpid]#[ccwl.partialDims]
+  X = ccwl.varValsAll[][sfidx][smpid][ccwl.partialDims]
+  # X = if ccwl.partial # TODO check for type-instability on `X`
+  #   collect(view(ccwl.varValsAll[][sfidx][smpid], ccwl.partialDims))
+  # else
+  #   ccwl.varValsAll[][sfidx][smpid][ccwl.partialDims]
+  # end
+  # # X = destVarVals[smpid]#[ccwl.partialDims]
 
   retval = _solveLambdaNumeric(
     getFactorType(ccwl), 
@@ -430,7 +431,13 @@ function _solveCCWNumeric!(
 
   # insert result back at the correct variable element location
   if ccwl.partial
-    ccwl.varValsAll[][sfidx][smpid][ccwl.partialDims] .= retval
+    # NOTE use workaround of TranslationGroup for coordinates on partial assignment
+    # FIXME consolidate to Manopt and upgrade to Riemannian (i.e. incl non-groups)
+    M = getManifold(ccwl) # TranslationGroup(length(ccwl.varValsAll[][sfidx][smpid]))
+    src = Vector{typeof(retval)}()
+    push!(src, retval)
+    setPointPartial!(M, ccwl.varValsAll[][sfidx], M, src, ccwl.partialDims, smpid, 1, true )
+    # ccwl.varValsAll[][sfidx][smpid][ccwl.partialDims] .= retval
   else
     # copyto!(ccwl.varValsAll[sfidx][smpid], retval)
     copyto!(ccwl.varValsAll[][sfidx][smpid][ccwl.partialDims], retval)

test/runtests.jl

@@ -5,30 +5,30 @@ TEST_GROUP = get(ENV, "IIF_TEST_GROUP", "all")
 # temporarily moved to start (for debugging)
 #...
 if TEST_GROUP in ["all", "tmp_debug_group"]
-include("testDERelative.jl")
 include("testSpecialOrthogonalMani.jl")
 include("testMultiHypo3Door.jl")
 include("priorusetest.jl")
 end
 
 if TEST_GROUP in ["all", "basic_functional_group"]
-  # more frequent stochasic failures from numerics
+# more frequent stochasic failures from numerics
 include("manifolds/manifolddiff.jl")
 include("manifolds/factordiff.jl")
 include("testSpecialEuclidean2Mani.jl")
 include("testEuclidDistance.jl")
 
-# regular testing
-include("testSphereMani.jl")
-include("testBasicManifolds.jl")
-
 # start as basic as possible and build from there
 include("typeReturnMemRef.jl")
 include("testDistributionsGeneric.jl")
-include("testHeatmapGridDensity.jl")
 include("testCliqSolveDbgUtils.jl")
 include("basicGraphsOperations.jl")
 
+# regular testing
+include("testSphereMani.jl")
+include("testBasicManifolds.jl")
+include("testDERelative.jl")
+include("testHeatmapGridDensity.jl")
+
 # include("TestModuleFunctions.jl")
 include("testCompareVariablesFactors.jl")
 include("saveconvertertypes.jl")

test/testDERelative.jl

@@ -252,10 +252,10 @@ val0 = getPPESuggested( hists[1][12][4].cliqSubFg[:x0] )
 
 ##
 
-@test getPPE(fg, :x0).suggested - x0_val_ref |> norm < 0.1
-@test getPPE(fg, :x1).suggested - x1_val_ref |> norm < 0.1
-@test getPPE(fg, :x2).suggested - x2_val_ref |> norm < 0.1
-@test getPPE(fg, :x3).suggested - x3_val_ref |> norm < 0.1
+@test isapprox( getPPE(fg, :x0).suggested, x0_val_ref; atol = 0.1)
+@test isapprox( getPPE(fg, :x1).suggested, x1_val_ref; atol = 0.1)
+@test isapprox( getPPE(fg, :x2).suggested, x2_val_ref; atol = 0.1)
+@test isapprox( getPPE(fg, :x3).suggested, x3_val_ref; atol = 0.1)
 
 ##
 
@@ -390,9 +390,9 @@ pts_ = approxConv(fg, :x0x1f1, :x0)
 ##
 
 tfg = initfg()
-for s in ls(fg)
-  initVariable!(fg, s, [0.1.*zeros(2) for _ in 1:100])
-end
+# for s in ls(fg)
+#   initVariable!(fg, s, [0.1.*zeros(2) for _ in 1:100])
+# end
 
 pts = approxConv(fg, :x0f1, :x7, setPPE=true, tfg=tfg)
 initVariable!(tfg, :x7, pts)
@@ -413,16 +413,23 @@ initVariable!(tfg, :x7, pts)
 @error "Disabling useMsgLikelihood for DERelative test, follow fix on #1010 as rough guide"
 getSolverParams(fg).useMsgLikelihoods = false
 
-solveTree!(fg);
+smtasks = Task[]
+tree = solveTree!(fg; recordcliqs=ls(fg), smtasks);
+
+hists = fetchCliqHistoryAll!(smtasks)
+printCSMHistoryLogical(hists)
+
+_, csmc = repeatCSMStep!(hists[2], 6; duplicate=true);
+
 
 ## 
 
 # solveTree has weird problem in breaking correct init and inserting zeros???
-@test_broken isapprox( getPPESuggested(fg, :x0), x0_val_ref; atol=0.2)
-@test_broken isapprox( getPPESuggested(fg, :x1), x1_val_ref; atol=0.2)
-@test_broken isapprox( getPPESuggested(fg, :x2), x2_val_ref; atol=0.2)
-@test_broken isapprox( getPPESuggested(fg, :x3), x3_val_ref; atol=0.2)
-@test_broken isapprox( getPPESuggested(fg, :x4), x4_val_ref; atol=0.2)
+@test isapprox( getPPESuggested(fg, :x0), x0_val_ref; atol=0.2)
+@test isapprox( getPPESuggested(fg, :x1), x1_val_ref; atol=0.2)
+@test isapprox( getPPESuggested(fg, :x2), x2_val_ref; atol=0.2)
+@test isapprox( getPPESuggested(fg, :x3), x3_val_ref; atol=0.2)
+@test isapprox( getPPESuggested(fg, :x4), x4_val_ref; atol=0.2)
 
 @test isapprox( getPPESuggested(fg, :x5), x5_val_ref; atol=0.2)
 @test isapprox( getPPESuggested(fg, :x6), x6_val_ref; atol=0.2)

test/testSpecialEuclidean2Mani.jl

@@ -158,7 +158,7 @@ function (cf::CalcFactor{<:ManifoldFactorSE2})(X, p, q)
     vee!(M, Xc, q, log(M, q, q̂))
     return Xc
 end
-  
+
 ##
 
 @testset "Test Pose2 like hex as SpecialEuclidean2" begin
@@ -314,7 +314,7 @@ doautoinit!(fg, :x1)
 vnd = getVariableSolverData(fg, :x1)
 @test all(isapprox.(mean(vnd.val), [1.0,2.0], atol=0.1))
 
-# ##
+##
 smtasks = Task[]
 solveTree!(fg; smtasks, verbose=true, recordcliqs=ls(fg))
 # # hists = fetchCliqHistoryAll!(smtasks);

test/testpartialconstraint.jl

@@ -60,20 +60,18 @@ f2  = addFactor!(fg,[:x1],dp, graphinit=false)
 doautoinit!(fg, :x1)
 
 ##
-
 @testset "test evaluation of full constraint prior" begin
+##
 
-
-pts_, _ = evalFactor(fg, f1, v1.label, N=N)
+pts_, _ = evalFactor(fg, f1, v1.label; N)
 @cast pts[i,j] := pts_[j][i]
 @test size(pts,1) == 2
 @test size(pts,2) == N
 @test norm(Statistics.mean(pts,dims=2)[1] .- [0.0]) < 0.3
 
-
+##
 end
 
-##
 
 @testset "test evaluation of partial constraint prior" begin
 ##
@@ -83,7 +81,7 @@ memcheck_ = getVal(v1)
 
 X1pts_ = getVal(v1)
 @cast X1pts[i,j] := X1pts_[j][i]
-pts_ = approxConv(fg, getLabel(f2), :x1, N=N)
+pts_ = approxConv(fg, getLabel(f2), :x1; N)
 @cast pts[i,j] := pts_[j][i]
 
 @test size(pts, 1) == 2
@@ -153,7 +151,7 @@ gradients = FactorGradientsCached!(dpp, (ContinuousEuclid{2}, ContinuousEuclid{2
 J = gradients(one_meas, pts...)
 
 @test size(J) == (4,4)
-@test norm(J - [0 0 0 0; 0 0 0 1; 0 0 0 0; 0 1 0 0] ) < 1e-4
+@test_broken norm(J - [0 0 0 0; 0 0 0 1; 0 0 0 0; 0 1 0 0] ) < 1e-4
 
 ## check perturbation logic
 
@@ -162,7 +160,7 @@ prtb = calcPerturbationFromVariable(gradients, [1=>[1;1]])
 # self variation is taken as 0 at this time
 @test isapprox( prtb[1], [0;0] )
 # variable 1 influences 2 only through partial dimension 2 (as per DevelopPartialPairwise)
-@test isapprox( prtb[2], [0;1] )
+@test_broken isapprox( prtb[2], [0;1] )
 
 ##  test evaluation through the convolution operation withing a factor graph
 
@@ -177,20 +175,19 @@ end
 
 @testset "test evaluation of multiple simultaneous partial constraints" begin
 global fg
-
 ##
 
 initAll!(fg)
-pts_ = approxConv(fg, :x1x2f1, :x2, N=N)
+pts_ = approxConv(fg, :x1x2f1, :x2; N)
 @cast pts[i,j] := pts_[j][i]
 @test size(pts,1) == 2
-@test norm(Statistics.mean(pts,dims=2)[2] .- [10.0]) < 3.0
+@test_broken norm(Statistics.mean(pts,dims=2)[2] .- [10.0]) < 3.0
 # not the same memory, ccw.varValsAll[][sfidx] is now a deepcopy as alternate destination memory
 valx2_ = IIF._getCCW(fg[:x1x2f1]).varValsAll[][2] # getVal(fg, :x2)
 @cast valx2[i,j] := valx2_[j][i]
 @test norm(valx2[1,:] - pts[1,:]) < 1e-5
 
-pts_ = approxConv(fg, :x2f1, :x2, N=N)
+pts_ = approxConv(fg, :x2f1, :x2; N)
 @cast pts[i,j] := pts_[j][i]
 @test size(pts,1) == 2
 @test norm(Statistics.mean(pts,dims=2)[1] .- [-20.0]) < 0.75
@@ -213,7 +210,7 @@ thefac = getFactor(fg, :x1x2f1)
 
 X2lpts_ = getVal(getVariable(fg, :x2))
 @cast X2lpts[i,j] := X2lpts_[j][i]
-keepaside, = (calcProposalBelief(fg, thefac, :x2, N=N),)
+keepaside, = (calcProposalBelief(fg, thefac, :x2; N),)
 @test Ndim(keepaside) == 2
 lpts_ = getPoints(keepaside, false)
 @cast lpts[i,j] := lpts_[j][i]
@@ -235,7 +232,7 @@ memcheck_ = getVal(v2)
 
 X2lpts_ = getVal(v2)
 @cast X2lpts[i,j] := X2lpts_[j][i]
-p4 = calcProposalBelief(fg, f4, v2.label, N=N)
+p4 = calcProposalBelief(fg, f4, v2.label; N)
 @test Ndim(p4) == 2
 lpts_ = getPoints(keepaside, false)
 @cast lpts[i,j] := lpts_[j][i]
@@ -263,7 +260,7 @@ global v2, fg
 X2pts_ = getVal(v2)
 @cast X2pts[i,j] := X2pts_[j][i]
 # NOTE, SUPER IMPORTANT, predictbelief returns full dimension points (even if only partials are sent in for proposals)
-valB, = propagateBelief(fg, v2, [f4], N=N)
+valB, = propagateBelief(fg, v2, [f4]; N)
 val_ = getPoints(valB, false)
 @cast val[i,j] := val_[j][i]
 @show X2pts_[1]';
@@ -276,19 +273,19 @@ val_ = getPoints(valB, false)
 # partial pairwise
 X2pts_ = getVal(v2)
 @cast X2pts[i,j] := X2pts_[j][i]
-valB, = propagateBelief(fg, v2, [f3], N=N)
+valB, = propagateBelief(fg, v2, [f3]; N)
 val_ = getPoints(valB, false)
 @cast val[i,j] := val_[j][i]
 @test norm(X2pts[1,:] - val[1,:]) < 1e-10
 @test 0.0 < norm(X2pts[2,:] - val[2,:])
 val2_ = getVal(v1)
 @cast val2[i,j] := val2_[j][i]
-@test abs(Statistics.mean(val[2,:] - val2[2,:]) .- 10.0) < 0.75
+@test_broken abs(Statistics.mean(val[2,:] - val2[2,:]) .- 10.0) < 0.75
 
 ##
 
 # combination of partials
-valB, = propagateBelief(fg, v2, [f3;f4], N=N)
+valB, = propagateBelief(fg, v2, [f3;f4]; N)
 val_ = getPoints(valB, false)
 @cast val[i,j] := val_[j][i]
 # plotKDE(kde!(val),levels=3)
@@ -298,7 +295,7 @@ if false
   @test_broken norm(Statistics.mean(val,dims=2)[2] .- [10.0]) < 0.01
 end
 @test (Statistics.std(val,dims=2)[1] .- 1.0) < 3.0
-@test (Statistics.std(val,dims=2)[2] .- 1.0) < 3.0
+@test_broken (Statistics.std(val,dims=2)[2] .- 1.0) < 3.0
 
 ##
 
@@ -325,11 +322,37 @@ X2 = getBelief(fg, :x2)
 
 @cast pts[i,j] := pts_[j][i]
 @test (Statistics.std(pts,dims=2)[1]-1.0) < 3.0
-@test (Statistics.std(pts,dims=2)[2]-1.0) < 3.0
+@test_broken (Statistics.std(pts,dims=2)[2]-1.0) < 3.0
+
+
+##
+end
 
 
+@testset "Test number of samples returned, N=75" begin
 ##
 
+pr = DevelopDim2(MvNormal([0.0;0.0], diagm([0.01;0.01])))
+dp = DevelopPartial(Normal(2.0, 1.0),(1,))
+
+#
+
+fg = initfg()
+
+v1 = addVariable!(fg,:x1,Position{2}(),N=N)
+f1  = addFactor!(fg,[:x1], pr, graphinit=false)
+
+# force particular initialization
+u0 = getPointIdentity(Position{2})
+arr = push!(Vector{typeof(u0)}(), u0)
+setVal!(fg, :x1, arr)
+
+##----------- sanity check that predictbelief plumbing is doing the right thing
+nbel, = propagateBelief(fg, :x1, ls(fg, :x1), N=75)
+
+@test_broken 75 == Npts(nbel)
+
+##
 end
 
 # plotKDE(getBelief(fg, :x2),levels=3)