Fix blob quarantine updateColumnQuarantine function. (#7759)

* Fix update() function for DataColumnQuarantine.
Add tests.
Add some helpers to ColumnMap.

* Update AllTests.

* Remove shortLog printer function.

Author: cheatfate

AllTests-mainnet.md

@@ -172,6 +172,18 @@ AllTests-mainnet
 ```
 ## ColumnQuarantine data structure test suite  [Preset: mainnet]
 ```diff
++ ColumnQuarantine: update(empty:grow) [node->node] test                                     OK
++ ColumnQuarantine: update(empty:grow) [node->supernode] test                                OK
++ ColumnQuarantine: update(empty:shrink) [node->node] test                                   OK
++ ColumnQuarantine: update(empty:shrink) [supernode->node] test                              OK
++ ColumnQuarantine: update(memory+disk:grow) [node->node] test                               OK
++ ColumnQuarantine: update(memory+disk:grow) [node->supernode] test                          OK
++ ColumnQuarantine: update(memory+disk:shrink) [node->node] test                             OK
++ ColumnQuarantine: update(memory+disk:shrink) [supernode->node] test                        OK
++ ColumnQuarantine: update(memory:grow) [node->node] test                                    OK
++ ColumnQuarantine: update(memory:grow) [node->supernode] test                               OK
++ ColumnQuarantine: update(memory:shrink) [node->node] test                                  OK
++ ColumnQuarantine: update(memory:shrink) [supernode->node] test                             OK
 + Empty in-memory scenario test [node]                                                       OK
 + Empty in-memory scenario test [supernode]                                                  OK
 + Mixed entries scenario test [node]                                                         OK

beacon_chain/consensus_object_pools/blob_quarantine.nim

@@ -82,6 +82,9 @@ type
   ColumnQuarantine* =
     SidecarQuarantine[fulu.DataColumnSidecar, OnDataColumnSidecarCallback]
 
+  ColumnQuarantineNode* =
+    DoublyLinkedNode[RootTableRecord[fulu.DataColumnSidecar]]
+
 func indexLog*[T: SomeSidecarRef](sidecars: openArray[ref T]): string =
   "[" & sidecars.mapIt($uint64(it[].index)).join(",") & "]"
 
@@ -886,12 +889,13 @@ proc init*(
     onDataColumnSidecarCallback: OnDataColumnSidecarCallback
 ): ColumnQuarantine =
   doAssert(len(custodyColumns) <= NUMBER_OF_COLUMNS)
+
+  let custodyMap = ColumnMap.init(custodyColumns)
   var indexMap = newSeqUninit[int](NUMBER_OF_COLUMNS)
   if len(custodyColumns) < NUMBER_OF_COLUMNS:
     for i in 0 ..< len(indexMap):
       indexMap[i] = -1
-  for index, item in custodyColumns.pairs():
-    doAssert(item < uint64(NUMBER_OF_COLUMNS))
+  for index, item in custodyMap.pairs():
     indexMap[int(item)] = index
 
   let size = maxSidecars(NUMBER_OF_COLUMNS)
@@ -911,27 +915,84 @@ proc init*(
     memSidecarsCount: 0,
     diskSidecarsCount: 0,
     indexMap: indexMap,
-    custodyColumns: @custodyColumns,
-    custodyMap: ColumnMap.init(custodyColumns),
+    custodyColumns: toSeq(custodyMap.items),
+    custodyMap: custodyMap,
     list: initDoublyLinkedList[RootTableRecord[fulu.DataColumnSidecar]](),
     db: database,
     onSidecarCallback: onDataColumnSidecarCallback
   )
 
-func updateColumnQuarantine*(
-    quarantine: ref ColumnQuarantine,
+proc update*(
+    quarantine: var ColumnQuarantine,
     cfg: RuntimeConfig,
-    custodyColumns: openArray[ColumnIndex]) =
+    custodyColumns: openArray[ColumnIndex]
+) =
   doAssert(len(custodyColumns) <= NUMBER_OF_COLUMNS)
+  let
+    custodyMap = ColumnMap.init(custodyColumns)
+    maxSidecarsPerBlockCount = len(custodyMap)
+
   var indexMap = newSeqUninit[int](NUMBER_OF_COLUMNS)
   if len(custodyColumns) < NUMBER_OF_COLUMNS:
     for i in 0 ..< len(indexMap):
       indexMap[i] = -1
-  for index, item in custodyColumns.pairs():
-    doAssert(item < uint64(NUMBER_OF_COLUMNS))
+  for index, item in custodyMap.pairs():
     indexMap[int(item)] = index
 
-  quarantine.maxSidecarsPerBlockCount = len(custodyColumns)
+  var
+    memSidecarsCount = 0
+    diskSidecarsCount = 0
+    nodesToRemove: seq[ColumnQuarantineNode]
+
+  for node in quarantine.list.nodes():
+    var
+      sidecars =
+        newSeq[SidecarHolder[fulu.DataColumnSidecar]](len(custodyMap))
+      count = 0
+      unloaded = 0
+
+    for cindex in quarantine.custodyMap.items():
+      let
+        index = quarantine.getIndex(cindex)
+        sidecar = node[].value.sidecars[index]
+
+      if not(isEmpty(sidecar)):
+        if cindex in custodyMap:
+          let dindex = indexMap[int(cindex)]
+          if dindex >= 0:
+            sidecars[dindex] = sidecar
+            inc(count)
+            if not(sidecar.isLoaded()):
+              inc(unloaded)
+
+    node.value.sidecars.reset()
+
+    if count > 0:
+      node[].value.sidecars = sidecars
+      node[].value.count = count
+      node[].value.unloaded = unloaded
+      # We do account sidecars which are useful in new configuration, but
+      # its possible that some sidecars will be left on disk which can't be
+      # used in new configuration, and we can't delete it easily. But this
+      # sidecars will be deleted as soon as sidecars with same `block_root`
+      # will be popped out from quarantine.
+      diskSidecarsCount.inc(unloaded)
+      memSidecarsCount.inc(count - unloaded)
+    else:
+      # If there no useful columns, we will mark this node for deletion.
+      nodesToRemove.add(node)
+
+  for node in nodesToRemove:
+    quarantine.removeNode(node, 0)
+
+  quarantine.diskSidecarsCount = diskSidecarsCount
+  quarantine.memSidecarsCount = memSidecarsCount
+  blob_quarantine_memory_slots_occupied.set(
+    int64(quarantine.memSidecarsCount))
+  blob_quarantine_database_slots_occupied.set(
+    int64(quarantine.diskSidecarsCount))
+
+  quarantine.maxSidecarsPerBlockCount = maxSidecarsPerBlockCount
   quarantine.indexMap = indexMap
-  quarantine.custodyColumns = @custodyColumns
-  quarantine.custodyMap = ColumnMap.init(custodyColumns)
+  quarantine.custodyColumns = toSeq(custodyMap.items)
+  quarantine.custodyMap = custodyMap

beacon_chain/nimbus_beacon_node.nim

@@ -2056,8 +2056,7 @@ proc onSlotEnd(node: BeaconNode, slot: Slot) {.async.} =
         node.validatorCustody.newer_column_set.toSeq()
       sort(custodyColumns)
       # update custody columns
-      node.dataColumnQuarantine.updateColumnQuarantine(
-        node.dag.cfg, custodyColumns)
+      node.dataColumnQuarantine[].update(node.dag.cfg, custodyColumns)
 
       # Update CGC and metadata with respect to the new detected validator custody
       let new_vcus = CgcCount node.validatorCustody.newer_column_set.lenu64

beacon_chain/spec/column_map.nim

@@ -65,6 +65,9 @@ func `xor`*(a, b: ColumnMap): ColumnMap =
 func `not`*(a: ColumnMap): ColumnMap =
   ColumnMap(data: [not(a.data[0]), not(a.data[1])])
 
+func `==`*(a, b: ColumnMap): bool =
+  (a.data[0] == b.data[0]) and (a.data[1] == b.data[1])
+
 func empty*(a: ColumnMap): bool =
   (a.data[0] == 0'u64) and (a.data[1] == 0'u64)
 
@@ -89,6 +92,12 @@ iterator items*(a: ColumnMap): ColumnIndex =
     yield ColumnIndex(64 + res - 1)
     data1 = data1 xor t
 
+iterator pairs*(a: ColumnMap): (int, ColumnIndex) =
+  var index = 0
+  for item in a.items():
+    yield (index, item)
+    inc(index)
+
 func len*(a: ColumnMap): int =
   # Returns number of columns in map.
   countOnes(a.data[0]) + countOnes(a.data[1])