|
33 | 33 |
|
34 | 34 | namespace swift { |
35 | 35 |
|
36 | | -/// This is a node in a concurrent linked list. |
37 | | -template <class ElemTy> struct ConcurrentListNode { |
38 | | - ConcurrentListNode(ElemTy Elem) : Payload(Elem), Next(nullptr) {} |
39 | | - ConcurrentListNode(const ConcurrentListNode &) = delete; |
40 | | - ConcurrentListNode &operator=(const ConcurrentListNode &) = delete; |
41 | | - |
42 | | - /// The element. |
43 | | - ElemTy Payload; |
44 | | - /// Points to the next link in the chain. |
45 | | - ConcurrentListNode<ElemTy> *Next; |
46 | | -}; |
47 | | - |
48 | | -/// This is a concurrent linked list. It supports insertion at the beginning |
49 | | -/// of the list and traversal using iterators. |
50 | | -/// This is a very simple implementation of a concurrent linked list |
51 | | -/// using atomic operations. The 'push_front' method allocates a new link |
52 | | -/// and attempts to compare and swap the old head pointer with pointer to |
53 | | -/// the new link. This operation may fail many times if there are other |
54 | | -/// contending threads, but eventually the head pointer is set to the new |
55 | | -/// link that already points to the old head value. Notice that the more |
56 | | -/// difficult feature of removing links is not supported. |
57 | | -/// See 'push_front' for more details. |
58 | | -template <class ElemTy> struct ConcurrentList { |
59 | | - ConcurrentList() : First(nullptr) {} |
60 | | - ~ConcurrentList() { |
61 | | - clear(); |
62 | | - } |
63 | | - |
64 | | - /// Remove all of the links in the chain. This method leaves |
65 | | - /// the list at a usable state and new links can be added. |
66 | | - /// Notice that this operation is non-sendable because |
67 | | - /// we have no way of ensuring that no one is currently |
68 | | - /// traversing the list. |
69 | | - void clear() { |
70 | | - // Iterate over the list and delete all the nodes. |
71 | | - auto Ptr = First.load(std::memory_order_acquire); |
72 | | - First.store(nullptr, std:: memory_order_release); |
73 | | - |
74 | | - while (Ptr) { |
75 | | - auto N = Ptr->Next; |
76 | | - delete Ptr; |
77 | | - Ptr = N; |
78 | | - } |
79 | | - } |
80 | | - |
81 | | - ConcurrentList(const ConcurrentList &) = delete; |
82 | | - ConcurrentList &operator=(const ConcurrentList &) = delete; |
83 | | - |
84 | | - /// A list iterator. |
85 | | - struct ConcurrentListIterator : |
86 | | - public std::iterator<std::forward_iterator_tag, ElemTy> { |
87 | | - |
88 | | - /// Points to the current link. |
89 | | - ConcurrentListNode<ElemTy> *Ptr; |
90 | | - /// C'tor. |
91 | | - ConcurrentListIterator(ConcurrentListNode<ElemTy> *P) : Ptr(P) {} |
92 | | - /// Move to the next element. |
93 | | - ConcurrentListIterator &operator++() { |
94 | | - Ptr = Ptr->Next; |
95 | | - return *this; |
96 | | - } |
97 | | - /// Access the element. |
98 | | - ElemTy &operator*() { return Ptr->Payload; } |
99 | | - /// Same? |
100 | | - bool operator==(const ConcurrentListIterator &o) const { |
101 | | - return o.Ptr == Ptr; |
102 | | - } |
103 | | - /// Not the same? |
104 | | - bool operator!=(const ConcurrentListIterator &o) const { |
105 | | - return o.Ptr != Ptr; |
106 | | - } |
107 | | - }; |
108 | | - |
109 | | - /// Iterator entry point. |
110 | | - typedef ConcurrentListIterator iterator; |
111 | | - /// Marks the beginning of the list. |
112 | | - iterator begin() const { |
113 | | - return ConcurrentListIterator(First.load(std::memory_order_acquire)); |
114 | | - } |
115 | | - /// Marks the end of the list. |
116 | | - iterator end() const { return ConcurrentListIterator(nullptr); } |
117 | | - |
118 | | - /// Add a new item to the list. |
119 | | - void push_front(ElemTy Elem) { |
120 | | - /// Allocate a new node. |
121 | | - ConcurrentListNode<ElemTy> *N = new ConcurrentListNode<ElemTy>(Elem); |
122 | | - // Point to the first element in the list. |
123 | | - N->Next = First.load(std::memory_order_acquire); |
124 | | - auto OldFirst = N->Next; |
125 | | - // Try to replace the current First with the new node. |
126 | | - while (!std::atomic_compare_exchange_weak_explicit(&First, &OldFirst, N, |
127 | | - std::memory_order_release, |
128 | | - std::memory_order_relaxed)) { |
129 | | - // If we fail, update the new node to point to the new head and try to |
130 | | - // insert before the new |
131 | | - // first element. |
132 | | - N->Next = OldFirst; |
133 | | - } |
134 | | - } |
135 | | - |
136 | | - /// Points to the first link in the list. |
137 | | - std::atomic<ConcurrentListNode<ElemTy> *> First; |
138 | | -}; |
139 | | - |
140 | | -/// A utility function for ordering two integers, which is useful |
141 | | -/// for implementing compareWithKey. |
142 | | -template <class T> |
143 | | -static inline int compareIntegers(T left, T right) { |
144 | | - return (left == right ? 0 : left < right ? -1 : 1); |
145 | | -} |
146 | | - |
147 | | -/// A utility function for ordering two pointers, which is useful |
148 | | -/// for implementing compareWithKey. |
149 | | -template <class T> |
150 | | -static inline int comparePointers(const T *left, const T *right) { |
151 | | - return (left == right ? 0 : std::less<const T *>()(left, right) ? -1 : 1); |
152 | | -} |
153 | | - |
154 | | -template <class EntryTy, bool ProvideDestructor, class Allocator> |
155 | | -class ConcurrentMapBase; |
156 | | - |
157 | | -/// The partial specialization of ConcurrentMapBase whose destructor is |
158 | | -/// trivial. The other implementation inherits from this, so this is a |
159 | | -/// base for all ConcurrentMaps. |
160 | | -template <class EntryTy, class Allocator> |
161 | | -class ConcurrentMapBase<EntryTy, false, Allocator> : protected Allocator { |
162 | | -protected: |
163 | | - struct Node { |
164 | | - std::atomic<Node*> Left; |
165 | | - std::atomic<Node*> Right; |
166 | | - EntryTy Payload; |
167 | | - |
168 | | - template <class... Args> |
169 | | - Node(Args &&... args) |
170 | | - : Left(nullptr), Right(nullptr), Payload(std::forward<Args>(args)...) {} |
171 | | - |
172 | | - Node(const Node &) = delete; |
173 | | - Node &operator=(const Node &) = delete; |
174 | | - |
175 | | - #ifndef NDEBUG |
176 | | - void dump() const { |
177 | | - auto L = Left.load(std::memory_order_acquire); |
178 | | - auto R = Right.load(std::memory_order_acquire); |
179 | | - printf("\"%p\" [ label = \" {<f0> %08lx | {<f1> | <f2>}}\" " |
180 | | - "style=\"rounded\" shape=\"record\"];\n", |
181 | | - this, (long) Payload.getKeyValueForDump()); |
182 | | - |
183 | | - if (L) { |
184 | | - L->dump(); |
185 | | - printf("\"%p\":f1 -> \"%p\":f0;\n", this, L); |
186 | | - } |
187 | | - if (R) { |
188 | | - R->dump(); |
189 | | - printf("\"%p\":f2 -> \"%p\":f0;\n", this, R); |
190 | | - } |
191 | | - } |
192 | | - #endif |
193 | | - }; |
194 | | - |
195 | | - std::atomic<Node*> Root; |
196 | | - |
197 | | - constexpr ConcurrentMapBase() : Root(nullptr) {} |
198 | | - |
199 | | - // Implicitly trivial destructor. |
200 | | - ~ConcurrentMapBase() = default; |
201 | | - |
202 | | - void destroyNode(Node *node) { |
203 | | - assert(node && "destroying null node"); |
204 | | - auto allocSize = sizeof(Node) + node->Payload.getExtraAllocationSize(); |
205 | | - |
206 | | - // Destroy the node's payload. |
207 | | - node->~Node(); |
208 | | - |
209 | | - // Deallocate the node. The static_cast here is required |
210 | | - // because LLVM's allocator API is insane. |
211 | | - this->Deallocate(static_cast<void*>(node), allocSize, alignof(Node)); |
212 | | - } |
213 | | -}; |
214 | | - |
215 | | -/// The partial specialization of ConcurrentMapBase which provides a |
216 | | -/// non-trivial destructor. |
217 | | -template <class EntryTy, class Allocator> |
218 | | -class ConcurrentMapBase<EntryTy, true, Allocator> |
219 | | - : protected ConcurrentMapBase<EntryTy, false, Allocator> { |
220 | | -protected: |
221 | | - using super = ConcurrentMapBase<EntryTy, false, Allocator>; |
222 | | - using Node = typename super::Node; |
223 | | - |
224 | | - constexpr ConcurrentMapBase() {} |
225 | | - |
226 | | - ~ConcurrentMapBase() { |
227 | | - destroyTree(this->Root); |
228 | | - } |
229 | | - |
230 | | -private: |
231 | | - void destroyTree(const std::atomic<Node*> &edge) { |
232 | | - // This can be a relaxed load because destruction is not allowed to race |
233 | | - // with other operations. |
234 | | - auto node = edge.load(std::memory_order_relaxed); |
235 | | - if (!node) return; |
236 | | - |
237 | | - // Destroy the node's children. |
238 | | - destroyTree(node->Left); |
239 | | - destroyTree(node->Right); |
240 | | - |
241 | | - // Destroy the node itself. |
242 | | - this->destroyNode(node); |
243 | | - } |
244 | | -}; |
245 | | - |
246 | | -/// A concurrent map that is implemented using a binary tree. It supports |
247 | | -/// concurrent insertions but does not support removals or rebalancing of |
248 | | -/// the tree. |
249 | | -/// |
250 | | -/// The entry type must provide the following operations: |
251 | | -/// |
252 | | -/// /// For debugging purposes only. Summarize this key as an integer value. |
253 | | -/// intptr_t getKeyIntValueForDump() const; |
254 | | -/// |
255 | | -/// /// A ternary comparison. KeyTy is the type of the key provided |
256 | | -/// /// to find or getOrInsert. |
257 | | -/// int compareWithKey(KeyTy key) const; |
258 | | -/// |
259 | | -/// /// Return the amount of extra trailing space required by an entry, |
260 | | -/// /// where KeyTy is the type of the first argument to getOrInsert and |
261 | | -/// /// ArgTys is the type of the remaining arguments. |
262 | | -/// static size_t getExtraAllocationSize(KeyTy key, ArgTys...) |
263 | | -/// |
264 | | -/// /// Return the amount of extra trailing space that was requested for |
265 | | -/// /// this entry. This method is only used to compute the size of the |
266 | | -/// /// object during node deallocation; it does not need to return a |
267 | | -/// /// correct value so long as the allocator's Deallocate implementation |
268 | | -/// /// ignores this argument. |
269 | | -/// size_t getExtraAllocationSize() const; |
270 | | -/// |
271 | | -/// If ProvideDestructor is false, the destructor will be trivial. This |
272 | | -/// can be appropriate when the object is declared at global scope. |
273 | | -template <class EntryTy, bool ProvideDestructor = true, |
274 | | - class Allocator = llvm::MallocAllocator> |
275 | | -class ConcurrentMap |
276 | | - : private ConcurrentMapBase<EntryTy, ProvideDestructor, Allocator> { |
277 | | - using super = ConcurrentMapBase<EntryTy, ProvideDestructor, Allocator>; |
278 | | - |
279 | | - using Node = typename super::Node; |
280 | | - |
281 | | - /// Inherited from base class: |
282 | | - /// std::atomic<Node*> Root; |
283 | | - using super::Root; |
284 | | - |
285 | | - /// This member stores the address of the last node that was found by the |
286 | | - /// search procedure. We cache the last search to accelerate code that |
287 | | - /// searches the same value in a loop. |
288 | | - std::atomic<Node*> LastSearch; |
289 | | - |
290 | | -public: |
291 | | - constexpr ConcurrentMap() : LastSearch(nullptr) {} |
292 | | - |
293 | | - ConcurrentMap(const ConcurrentMap &) = delete; |
294 | | - ConcurrentMap &operator=(const ConcurrentMap &) = delete; |
295 | | - |
296 | | - // ConcurrentMap<T, false> must have a trivial destructor. |
297 | | - ~ConcurrentMap() = default; |
298 | | - |
299 | | -public: |
300 | | - |
301 | | - Allocator &getAllocator() { |
302 | | - return *this; |
303 | | - } |
304 | | - |
305 | | -#ifndef NDEBUG |
306 | | - void dump() const { |
307 | | - auto R = Root.load(std::memory_order_acquire); |
308 | | - printf("digraph g {\n" |
309 | | - "graph [ rankdir = \"TB\"];\n" |
310 | | - "node [ fontsize = \"16\" ];\n" |
311 | | - "edge [ ];\n"); |
312 | | - if (R) { |
313 | | - R->dump(); |
314 | | - } |
315 | | - printf("\n}\n"); |
316 | | - } |
317 | | -#endif |
318 | | - |
319 | | - /// Search for a value by key \p Key. |
320 | | - /// \returns a pointer to the value or null if the value is not in the map. |
321 | | - template <class KeyTy> |
322 | | - EntryTy *find(const KeyTy &key) { |
323 | | - // Check if we are looking for the same key that we looked for in the last |
324 | | - // time we called this function. |
325 | | - if (Node *last = LastSearch.load(std::memory_order_acquire)) { |
326 | | - if (last->Payload.compareWithKey(key) == 0) |
327 | | - return &last->Payload; |
328 | | - } |
329 | | - |
330 | | - // Search the tree, starting from the root. |
331 | | - Node *node = Root.load(std::memory_order_acquire); |
332 | | - while (node) { |
333 | | - int comparisonResult = node->Payload.compareWithKey(key); |
334 | | - if (comparisonResult == 0) { |
335 | | - LastSearch.store(node, std::memory_order_release); |
336 | | - return &node->Payload; |
337 | | - } else if (comparisonResult < 0) { |
338 | | - node = node->Left.load(std::memory_order_acquire); |
339 | | - } else { |
340 | | - node = node->Right.load(std::memory_order_acquire); |
341 | | - } |
342 | | - } |
343 | | - |
344 | | - return nullptr; |
345 | | - } |
346 | | - |
347 | | - /// Get or create an entry in the map. |
348 | | - /// |
349 | | - /// \returns the entry in the map and whether a new node was added (true) |
350 | | - /// or already existed (false) |
351 | | - template <class KeyTy, class... ArgTys> |
352 | | - std::pair<EntryTy*, bool> getOrInsert(KeyTy key, ArgTys &&... args) { |
353 | | - // Check if we are looking for the same key that we looked for the |
354 | | - // last time we called this function. |
355 | | - if (Node *last = LastSearch.load(std::memory_order_acquire)) { |
356 | | - if (last && last->Payload.compareWithKey(key) == 0) |
357 | | - return { &last->Payload, false }; |
358 | | - } |
359 | | - |
360 | | - // The node we allocated. |
361 | | - Node *newNode = nullptr; |
362 | | - |
363 | | - // Start from the root. |
364 | | - auto edge = &Root; |
365 | | - |
366 | | - while (true) { |
367 | | - // Load the edge. |
368 | | - Node *node = edge->load(std::memory_order_acquire); |
369 | | - |
370 | | - // If there's a node there, it's either a match or we're going to |
371 | | - // one of its children. |
372 | | - if (node) { |
373 | | - searchFromNode: |
374 | | - |
375 | | - // Compare our key against the node's key. |
376 | | - int comparisonResult = node->Payload.compareWithKey(key); |
377 | | - |
378 | | - // If it's equal, we can use this node. |
379 | | - if (comparisonResult == 0) { |
380 | | - // Destroy the node we allocated before if we're carrying one around. |
381 | | - if (newNode) this->destroyNode(newNode); |
382 | | - |
383 | | - // Cache and report that we found an existing node. |
384 | | - LastSearch.store(node, std::memory_order_release); |
385 | | - return { &node->Payload, false }; |
386 | | - } |
387 | | - |
388 | | - // Otherwise, select the appropriate child edge and descend. |
389 | | - edge = (comparisonResult < 0 ? &node->Left : &node->Right); |
390 | | - continue; |
391 | | - } |
392 | | - |
393 | | - // Create a new node. |
394 | | - if (!newNode) { |
395 | | - size_t allocSize = |
396 | | - sizeof(Node) + EntryTy::getExtraAllocationSize(key, args...); |
397 | | - void *memory = this->Allocate(allocSize, alignof(Node)); |
398 | | - newNode = ::new (memory) Node(key, std::forward<ArgTys>(args)...); |
399 | | - } |
400 | | - |
401 | | - // Try to set the edge to the new node. |
402 | | - if (std::atomic_compare_exchange_strong_explicit(edge, &node, newNode, |
403 | | - std::memory_order_acq_rel, |
404 | | - std::memory_order_acquire)) { |
405 | | - // If that succeeded, cache and report that we created a new node. |
406 | | - LastSearch.store(newNode, std::memory_order_release); |
407 | | - return { &newNode->Payload, true }; |
408 | | - } |
409 | | - |
410 | | - // Otherwise, we lost the race because some other thread initialized |
411 | | - // the edge before us. node will be set to the current value; |
412 | | - // repeat the search from there. |
413 | | - assert(node && "spurious failure from compare_exchange_strong?"); |
414 | | - goto searchFromNode; |
415 | | - } |
416 | | - } |
417 | | -}; |
418 | | - |
419 | 36 | /// A simple linked list representing pointers that need to be freed. This is |
420 | 37 | /// not a concurrent data structure, just a bit of support used in the types |
421 | 38 | /// below. |
|
0 commit comments