Refactor shared library manager to be thread-safe and reorganize code

Author: marcharper

src/axelrod_fortran/player.py

@@ -1,139 +1,37 @@
-from collections import defaultdict
-from ctypes import cdll, c_int, c_float, byref, POINTER
-from ctypes.util import find_library
-import os
-import platform
+from ctypes import byref, c_float, c_int, POINTER
 import random
-import re
-import shutil
-import subprocess
-import tempfile
-import uuid
 import warnings
 
 import axelrod as axl
 from axelrod.interaction_utils import compute_final_score
 from axelrod.action import Action
+
 from .strategies import characteristics
+from .shared_library_manager import MultiprocessManager, load_library
 
 C, D = Action.C, Action.D
 actions = {0: C, 1: D}
 original_actions = {C: 0, D: 1}
 
 
-path_regex = r""".*?\s=>\s(.*?{}.*?)\\"""
-
 self_interaction_message = """
 You are playing a match with the same player against itself. However
 axelrod_fortran players share memory. You can initialise another instance of an
 Axelrod_fortran player with player.clone().
 """
 
 
-class LibraryManager(object):
-    """LibraryManager creates and loads copies of a shared library, which
-    enables multiple copies of the same strategy to be run without the end user
-    having to maintain many copies of the shared library.
-
-    This works by making a copy of the shared library file and loading it into
-    memory again. Loading the same file again will return a reference to the
-    same memory addresses.
-
-    Additionally, library manager tracks how many copies of the library have
-    been loaded, and how many copies there are of each Player, so as to load
-    only as many copies of the shared library as needed.
-    """
-
-    def __init__(self, shared_library_name, verbose=False):
-        self.shared_library_name = shared_library_name
-        self.verbose = verbose
-        self.library_copies = []
-        self.player_indices = defaultdict(set)
-        self.player_next = defaultdict(set)
-        # Generate a random prefix for tempfile generation
-        self.prefix = str(uuid.uuid4())
-        self.library_path = self.find_shared_library(shared_library_name)
-        self.filenames = []
-
-    def find_shared_library(self, shared_library_name):
-        # Hack for Linux since find_library doesn't return the full path.
-        if 'Linux' in platform.system():
-            output = subprocess.check_output(["ldconfig", "-p"])
-            for line in str(output).split(r"\n"):
-                rhs = line.split(" => ")[-1]
-                if shared_library_name in rhs:
-                    return rhs
-            raise ValueError("{} not found".format(shared_library_name))
-        else:
-            return find_library(
-                shared_library_name.replace("lib", "").replace(".so", ""))
-
-    def load_dll_copy(self):
-        """Load a new copy of the shared library."""
-        # Copy the library file to a new location so we can load the copy.
-        temp_directory = tempfile.gettempdir()
-        copy_number = len(self.library_copies)
-        new_filename = os.path.join(
-            temp_directory,
-            "{}-{}-{}".format(
-                self.prefix,
-                str(copy_number),
-                self.shared_library_name)
-        )
-        if self.verbose:
-            print("Loading {}".format(new_filename))
-        shutil.copy2(self.library_path, new_filename)
-        self.filenames.append(new_filename)
-        shared_library = cdll.LoadLibrary(new_filename)
-        self.library_copies.append(shared_library)
-
-    def next_player_index(self, name):
-        """Determine the index of the next free shared library copy to
-        allocate for the player. If none is available then load another copy."""
-        # Is there a free index?
-        if len(self.player_next[name]) > 0:
-            return self.player_next[name].pop()
-        # Do we need to load a new copy?
-        player_count = len(self.player_indices[name])
-        if player_count == len(self.library_copies):
-            self.load_dll_copy()
-            return player_count
-        # Find the first unused index
-        for i in range(len(self.library_copies)):
-            if i not in self.player_indices[name]:
-                return i
-        raise ValueError("We shouldn't be here.")
-
-    def load_library_for_player(self, name):
-        """For a given player return a copy of the shared library for use
-        in a Player class, along with an index for later releasing."""
-        index = self.next_player_index(name)
-        self.player_indices[name].add(index)
-        if self.verbose:
-            print("allocating {}".format(index))
-        return index, self.library_copies[index]
-
-    def release(self, name, index):
-        """Release the copy of the library so that it can be re-allocated."""
-        self.player_indices[name].remove(index)
-        if self.verbose:
-            print("releasing {}".format(index))
-        self.player_next[name].add(index)
-
-    def __del__(self):
-        """Cleanup temp files on object deletion."""
-        for filename in self.filenames:
-            if os.path.exists(filename):
-                os.remove(filename)
+# Initialize a module-wide manager for loading copies of the shared library.
+manager = MultiprocessManager()
+manager.start()
+shared_library_manager = manager.SharedLibraryManager("libstrategies.so")
 
 
 class Player(axl.Player):
 
     classifier = {"stochastic": True}
-    library_manager = None
 
-    def __init__(self, original_name,
-                 shared_library_name='libstrategies.so'):
+    def __init__(self, original_name):
         """
         Parameters
         ----------
@@ -144,17 +42,16 @@ def __init__(self, original_name,
             A instance of an axelrod Game
         """
         super().__init__()
-        if not Player.library_manager:
-            Player.library_manager = LibraryManager(shared_library_name)
-        self.index, self.shared_library = \
-            self.library_manager.load_library_for_player(original_name)
+        self.index, self.shared_library_filename = \
+            shared_library_manager.get_filename_for_player(original_name)
+        self.shared_library = load_library(self.shared_library_filename)
         self.original_name = original_name
         self.original_function = self.original_name
-
         is_stochastic = characteristics[self.original_name]['stochastic']
         if is_stochastic is not None:
             self.classifier['stochastic'] = is_stochastic
 
+
     def __enter__(self):
         return self
 
@@ -216,15 +113,21 @@ def strategy(self, opponent):
             my_last_move)
         return actions[original_action]
 
+    def _release_shared_library(self):
+        try:
+            shared_library_manager.release(self.original_name, self.index)
+        except FileNotFoundError:
+            pass
+
     def reset(self):
-        # Release the library before rest, which regenerates the player.
-        self.library_manager.release(self.original_name, self.index)
+        # Release the shared library since the object is rebuilt on reset.
+        self._release_shared_library()
         super().reset()
         self.original_function = self.original_name
 
     def __del__(self):
         # Release the library before deletion.
-        self.library_manager.release(self.original_name, self.index)
+        self._release_shared_library()
 
     def __repr__(self):
         return self.original_name

src/axelrod_fortran/shared_library_manager.py

@@ -0,0 +1,121 @@
+from collections import defaultdict
+from ctypes import cdll
+from ctypes.util import find_library
+from multiprocessing.managers import BaseManager
+import os
+import platform
+import shutil
+import subprocess
+import tempfile
+import uuid
+
+
+def load_library(filename):
+    """Loads a shared library."""
+    lib = None
+    if os.path.exists(filename):
+        lib = cdll.LoadLibrary(filename)
+    return lib
+
+
+class SharedLibraryManager(object):
+    """LibraryManager creates (and deletes) copies of a shared library, which
+    enables multiple copies of the same strategy to be run without the end user
+    having to maintain many copies of the shared library.
+
+    This works by making a copy of the shared library file and loading it into
+    memory again. Loading the same file again will return a reference to the
+    same memory addresses. To be thread-safe, this class just passes filenames
+    back to the Player class (which actually loads a reference to the library),
+    ensuring that multiple copies of a given player type do not use the same
+    copy of the shared library.
+    """
+
+    def __init__(self, shared_library_name, verbose=False):
+        self.shared_library_name = shared_library_name
+        self.verbose = verbose
+        self.filenames = []
+        self.player_indices = defaultdict(set)
+        self.player_next = defaultdict(set)
+        # Generate a random prefix for tempfile generation
+        self.prefix = str(uuid.uuid4())
+        self.library_path = self.find_shared_library(shared_library_name)
+
+    def find_shared_library(self, shared_library_name):
+        # Hack for Linux since find_library doesn't return the full path.
+        if 'Linux' in platform.system():
+            output = subprocess.check_output(["ldconfig", "-p"])
+            for line in str(output).split(r"\n"):
+                rhs = line.split(" => ")[-1]
+                if shared_library_name in rhs:
+                    return rhs
+            raise ValueError("{} not found".format(shared_library_name))
+        else:
+            return find_library(
+                shared_library_name.replace("lib", "").replace(".so", ""))
+
+    def load_dll_copy(self):
+        """Load a new copy of the shared library."""
+        # Copy the library file to a new location so we can load the copy.
+        temp_directory = tempfile.gettempdir()
+        copy_number = len(self.filenames)
+        new_filename = os.path.join(
+            temp_directory,
+            "{}-{}-{}".format(
+                self.prefix,
+                str(copy_number),
+                self.shared_library_name)
+        )
+        if self.verbose:
+            print("Loading {}".format(new_filename))
+        shutil.copy2(self.library_path, new_filename)
+        self.filenames.append(new_filename)
+
+    def next_player_index(self, name):
+        """Determine the index of the next free shared library copy to
+        allocate for the player. If none is available then load another copy."""
+        # Is there a free index?
+        if len(self.player_next[name]) > 0:
+            return self.player_next[name].pop()
+        # Do we need to load a new copy?
+        player_count = len(self.player_indices[name])
+        if player_count == len(self.filenames):
+            self.load_dll_copy()
+            return player_count
+        # Find the first unused index
+        for i in range(len(self.filenames)):
+            if i not in self.player_indices[name]:
+                return i
+        raise ValueError("We shouldn't be here.")
+
+    def get_filename_for_player(self, name):
+        """For a given player return a copy of the shared library for use
+        in a Player class, along with an index for later releasing."""
+        index = self.next_player_index(name)
+        self.player_indices[name].add(index)
+        if self.verbose:
+            print("allocating {}".format(index))
+        return index, self.filenames[index]
+
+    def release(self, name, index):
+        """Release the copy of the library so that it can be re-allocated."""
+        self.player_indices[name].remove(index)
+        if self.verbose:
+            print("releasing {}".format(index))
+        self.player_next[name].add(index)
+
+    def __del__(self):
+        """Cleanup temp files on object deletion."""
+        for filename in self.filenames:
+            if os.path.exists(filename):
+                if self.verbose:
+                    print("deleting", filename)
+                os.remove(filename)
+
+
+# Setup up thread safe library manager.
+class MultiprocessManager(BaseManager):
+    pass
+
+
+MultiprocessManager.register('SharedLibraryManager', SharedLibraryManager)

tests/test_player.py

@@ -1,11 +1,13 @@
+from ctypes import c_int, c_float, POINTER, CDLL
+import itertools
+
+import pytest
+
 from axelrod_fortran import Player, characteristics, all_strategies
 from axelrod import (Alternator, Cooperator, Defector, Match, MoranProcess,
                      Game, basic_strategies, seed)
 from axelrod.action import Action
-from ctypes import c_int, c_float, POINTER, CDLL
 
-import itertools
-import pytest
 
 C, D = Action.C, Action.D