|
| 1 | +/* |
| 2 | + * @notice |
| 3 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 4 | + * contributor license agreements. See the NOTICE file distributed with |
| 5 | + * this work for additional information regarding copyright ownership. |
| 6 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 7 | + * (the "License"); you may not use this file except in compliance with |
| 8 | + * the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, software |
| 13 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | + * See the License for the specific language governing permissions and |
| 16 | + * limitations under the License. |
| 17 | + * |
| 18 | + * Modifications copyright (C) 2025 Elasticsearch B.V. |
| 19 | + */ |
| 20 | +package org.elasticsearch.search.vectors; |
| 21 | + |
| 22 | +import org.apache.lucene.search.AcceptDocs; |
| 23 | +import org.apache.lucene.search.DocIdSetIterator; |
| 24 | +import org.apache.lucene.search.FilteredDocIdSetIterator; |
| 25 | +import org.apache.lucene.search.ScorerSupplier; |
| 26 | +import org.apache.lucene.util.BitSet; |
| 27 | +import org.apache.lucene.util.BitSetIterator; |
| 28 | +import org.apache.lucene.util.Bits; |
| 29 | +import org.apache.lucene.util.FixedBitSet; |
| 30 | + |
| 31 | +import java.io.IOException; |
| 32 | +import java.util.Objects; |
| 33 | +import java.util.Optional; |
| 34 | + |
| 35 | +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; |
| 36 | + |
| 37 | +/** |
| 38 | + * An extension of {@link AcceptDocs} that provides additional methods to get an approximate cost |
| 39 | + * and a BitSet representation of the accepted documents. |
| 40 | + */ |
| 41 | +public abstract sealed class ESAcceptDocs extends AcceptDocs { |
| 42 | + |
| 43 | + /** Returns an approximate cost of the accepted documents. |
| 44 | + * This is generally much cheaper than {@link #cost()}, as implementations may |
| 45 | + * not fully evaluate filters to provide this estimate and may ignore deletions |
| 46 | + * @return the approximate cost |
| 47 | + * @throws IOException if an I/O error occurs |
| 48 | + */ |
| 49 | + public abstract int approximateCost() throws IOException; |
| 50 | + |
| 51 | + /** |
| 52 | + * Returns an optional BitSet representing the accepted documents. |
| 53 | + * If a BitSet representation is not available, returns an empty Optional. An empty optional indicates that |
| 54 | + * there are some accepted documents, but they cannot be represented as a BitSet efficiently. |
| 55 | + * Null implies that all documents are accepted. |
| 56 | + * @return an Optional containing the BitSet of accepted documents, or empty if not available, or null if all documents are accepted |
| 57 | + * @throws IOException if an I/O error occurs |
| 58 | + */ |
| 59 | + public abstract Optional<BitSet> getBitSet() throws IOException; |
| 60 | + |
| 61 | + private static BitSet createBitSet(DocIdSetIterator iterator, Bits liveDocs, int maxDoc) throws IOException { |
| 62 | + if (liveDocs == null && iterator instanceof BitSetIterator bitSetIterator) { |
| 63 | + // If we already have a BitSet and no deletions, reuse the BitSet |
| 64 | + return bitSetIterator.getBitSet(); |
| 65 | + } else { |
| 66 | + int threshold = maxDoc >> 7; // same as BitSet#of |
| 67 | + if (iterator.cost() >= threshold) { |
| 68 | + FixedBitSet bitSet = new FixedBitSet(maxDoc); |
| 69 | + bitSet.or(iterator); |
| 70 | + if (liveDocs != null) { |
| 71 | + liveDocs.applyMask(bitSet, 0); |
| 72 | + } |
| 73 | + return bitSet; |
| 74 | + } else { |
| 75 | + return BitSet.of(liveDocs == null ? iterator : new FilteredDocIdSetIterator(iterator) { |
| 76 | + @Override |
| 77 | + protected boolean match(int doc) { |
| 78 | + return liveDocs.get(doc); |
| 79 | + } |
| 80 | + }, maxDoc); // create a sparse bitset |
| 81 | + } |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + /** An AcceptDocs that accepts all documents. */ |
| 86 | + public static final class ESAcceptDocsAll extends ESAcceptDocs { |
| 87 | + public static final ESAcceptDocsAll INSTANCE = new ESAcceptDocsAll(); |
| 88 | + |
| 89 | + private ESAcceptDocsAll() {} |
| 90 | + |
| 91 | + @Override |
| 92 | + public int approximateCost() throws IOException { |
| 93 | + return 0; |
| 94 | + } |
| 95 | + |
| 96 | + @Override |
| 97 | + public Optional<BitSet> getBitSet() throws IOException { |
| 98 | + return null; |
| 99 | + } |
| 100 | + |
| 101 | + @Override |
| 102 | + public Bits bits() throws IOException { |
| 103 | + return null; |
| 104 | + } |
| 105 | + |
| 106 | + @Override |
| 107 | + public DocIdSetIterator iterator() throws IOException { |
| 108 | + return null; |
| 109 | + } |
| 110 | + |
| 111 | + @Override |
| 112 | + public int cost() throws IOException { |
| 113 | + return 0; |
| 114 | + } |
| 115 | + } |
| 116 | + |
| 117 | + /** An AcceptDocs that wraps a Bits instance. Generally indicates that no filter was provided, but there are deleted docs */ |
| 118 | + public static final class BitsAcceptDocs extends ESAcceptDocs { |
| 119 | + private final Bits bits; |
| 120 | + private final BitSet bitSetRef; |
| 121 | + private final int maxDoc; |
| 122 | + private final int approximateCost; |
| 123 | + |
| 124 | + BitsAcceptDocs(Bits bits, int maxDoc) { |
| 125 | + if (bits != null && bits.length() != maxDoc) { |
| 126 | + throw new IllegalArgumentException("Bits length = " + bits.length() + " != maxDoc = " + maxDoc); |
| 127 | + } |
| 128 | + this.bits = Objects.requireNonNull(bits); |
| 129 | + if (bits instanceof BitSet bitSet) { |
| 130 | + this.maxDoc = Objects.requireNonNull(bitSet).cardinality(); |
| 131 | + this.approximateCost = Objects.requireNonNull(bitSet).approximateCardinality(); |
| 132 | + this.bitSetRef = bitSet; |
| 133 | + } else { |
| 134 | + this.maxDoc = maxDoc; |
| 135 | + this.approximateCost = maxDoc; |
| 136 | + this.bitSetRef = null; |
| 137 | + } |
| 138 | + } |
| 139 | + |
| 140 | + @Override |
| 141 | + public Bits bits() { |
| 142 | + return bits; |
| 143 | + } |
| 144 | + |
| 145 | + @Override |
| 146 | + public DocIdSetIterator iterator() { |
| 147 | + if (bitSetRef != null) { |
| 148 | + return new BitSetIterator(bitSetRef, maxDoc); |
| 149 | + } |
| 150 | + return new FilteredDocIdSetIterator(DocIdSetIterator.all(maxDoc)) { |
| 151 | + @Override |
| 152 | + protected boolean match(int doc) { |
| 153 | + return bits.get(doc); |
| 154 | + } |
| 155 | + }; |
| 156 | + } |
| 157 | + |
| 158 | + @Override |
| 159 | + public int cost() { |
| 160 | + // We have no better estimate. This should be ok in practice since background merges should |
| 161 | + // keep the number of deletes under control (< 20% by default). |
| 162 | + return maxDoc; |
| 163 | + } |
| 164 | + |
| 165 | + @Override |
| 166 | + public int approximateCost() { |
| 167 | + return approximateCost; |
| 168 | + } |
| 169 | + |
| 170 | + @Override |
| 171 | + public Optional<BitSet> getBitSet() { |
| 172 | + if (bits == null) { |
| 173 | + return null; |
| 174 | + } |
| 175 | + return Optional.ofNullable(bitSetRef); |
| 176 | + } |
| 177 | + } |
| 178 | + |
| 179 | + /** An AcceptDocs that wraps a ScorerSupplier. Indicates that a filter was provided. */ |
| 180 | + public static final class ScorerSupplierAcceptDocs extends ESAcceptDocs { |
| 181 | + private final ScorerSupplier scorerSupplier; |
| 182 | + private BitSet acceptBitSet; |
| 183 | + private final Bits liveDocs; |
| 184 | + private final int maxDoc; |
| 185 | + private int cardinality = -1; |
| 186 | + |
| 187 | + ScorerSupplierAcceptDocs(ScorerSupplier scorerSupplier, Bits liveDocs, int maxDoc) { |
| 188 | + this.scorerSupplier = scorerSupplier; |
| 189 | + this.liveDocs = liveDocs; |
| 190 | + this.maxDoc = maxDoc; |
| 191 | + } |
| 192 | + |
| 193 | + private void createBitSetIfNecessary() throws IOException { |
| 194 | + if (acceptBitSet == null) { |
| 195 | + acceptBitSet = createBitSet(scorerSupplier.get(NO_MORE_DOCS).iterator(), liveDocs, maxDoc); |
| 196 | + } |
| 197 | + } |
| 198 | + |
| 199 | + @Override |
| 200 | + public Bits bits() throws IOException { |
| 201 | + createBitSetIfNecessary(); |
| 202 | + return acceptBitSet; |
| 203 | + } |
| 204 | + |
| 205 | + @Override |
| 206 | + public DocIdSetIterator iterator() throws IOException { |
| 207 | + if (acceptBitSet != null) { |
| 208 | + return new BitSetIterator(acceptBitSet, cardinality); |
| 209 | + } |
| 210 | + return liveDocs == null |
| 211 | + ? scorerSupplier.get(NO_MORE_DOCS).iterator() |
| 212 | + : new FilteredDocIdSetIterator(scorerSupplier.get(NO_MORE_DOCS).iterator()) { |
| 213 | + @Override |
| 214 | + protected boolean match(int doc) { |
| 215 | + return liveDocs.get(doc); |
| 216 | + } |
| 217 | + }; |
| 218 | + } |
| 219 | + |
| 220 | + @Override |
| 221 | + public int cost() throws IOException { |
| 222 | + createBitSetIfNecessary(); |
| 223 | + if (cardinality == -1) { |
| 224 | + cardinality = acceptBitSet.cardinality(); |
| 225 | + } |
| 226 | + return cardinality; |
| 227 | + } |
| 228 | + |
| 229 | + @Override |
| 230 | + public int approximateCost() throws IOException { |
| 231 | + if (acceptBitSet != null) { |
| 232 | + return cardinality != -1 ? cardinality : acceptBitSet.approximateCardinality(); |
| 233 | + } |
| 234 | + return Math.toIntExact(scorerSupplier.cost()); |
| 235 | + } |
| 236 | + |
| 237 | + @Override |
| 238 | + public Optional<BitSet> getBitSet() throws IOException { |
| 239 | + createBitSetIfNecessary(); |
| 240 | + return Optional.of(acceptBitSet); |
| 241 | + } |
| 242 | + } |
| 243 | +} |
0 commit comments