enhanced mutation docstrings, especially mutate_fix_var

Author: joernhees

gp_learner.py

@@ -364,12 +364,7 @@ def _mutate_merge_var_helper(vars_):
 
 
 def mutate_merge_var_mix(child):
-    """Merges two variables into one.
-
-    Considers both node variables and edge variables together.
-    It is possible to merge an edge and a node too.
-    Randomly chooses a variable to replace and a variable to merge into.
-    """
+    """Merges two variables into one, potentially merging node and edge vars."""
     vars_ = child.vars_in_graph
     rand_vars, merge_able_vars = _mutate_merge_var_helper(vars_)
 
@@ -383,11 +378,10 @@ def mutate_merge_var_mix(child):
 
 
 def mutate_merge_var_sep(child):
-    """Merges two variables into one.
+    """Merges two variables into one, won't merge node and edge vars.
 
     Considers the node variables and edge variables separately.
-    Either merges 2 node variables or 2 edge variable, depending on a random
-    choice.Randomly chooses a variable to replace and a variable to merge into.
+    Depending on availability either merges 2 node variables or 2 edge variable.
     """
     node_vars = {n for n in child.nodes if isinstance(n, Variable)}
     rand_node_vars, merge_able_node_vars = _mutate_merge_var_helper(node_vars)
@@ -426,12 +420,13 @@ def mutate_del_triple(child):
 
 
 def mutate_expand_node(child, pb_en_out_link):
-    """Expands a random node of the pattern by adding a new triple to it.
+    """Expands a random node by adding a new var-only triple to it.
 
-    The variables to be attached to this node, to form a triple, are chosen
-    randomly.Depending on the probability, makes it an outgoing edge or an
-    incoming edge.
-    :return: The modified child, with the added triple.
+    Randomly selects a node. Then (depending on the probability pb_en_out_link)
+    adds an outgoing or incoming triple with two new vars to it.
+
+    :arg pb_en_out_link: Probability to create an outgoing triple.
+    :return: A child with the added outgoing/incoming triple.
     """
     # TODO: can maybe be improved by sparqling
     nodes = list(child.nodes)
@@ -446,10 +441,12 @@ def mutate_expand_node(child, pb_en_out_link):
 
 
 def mutate_add_edge(child):
-    """Chooses any 2 nodes from the pattern, and adds an edge between them.
+    """Adds an edge between 2 randomly selected nodes.
+
+    Randomly selects two nodes, then adds a new triple (n1, e, n2), where e is
+    a new variable.
 
-    The edge is labeled with a new randomly chosen variable.
-    :return: Modified child, with the new edge
+    :return: A child with the added edge.
     """
     # TODO: can maybe be improved by sparqling
     nodes = list(child.nodes)
@@ -462,11 +459,12 @@ def mutate_add_edge(child):
 
 
 def mutate_increase_dist(child):
-    """increases distance between source and target by one hop.
+    """Increases the distance between ?source and ?target by one hop.
 
-    Adds a triple, to either the source var or the target var.
-    Interchange the new node with source/target variable to increase distance.
-    :return: The modified child, with the new triple.
+    Randomly adds a var only triple to the ?source or ?target var. Then swaps
+    the new node with ?source/?target to increase the distance by one hop.
+
+    :return: A child with increased distance between ?source and ?target.
     """
     if not child.complete():
         return child
@@ -485,6 +483,13 @@ def mutate_increase_dist(child):
 
 
 def mutate_fix_var_filter(item_counts):
+    """Filters results for fix var mutation.
+
+    Excludes:
+    - too long literals
+    - URIs with encoding errors (real world!)
+    - BNode results (they will not be fixed but stay SPARQL vars)
+    """
     assert isinstance(item_counts, Counter)
     for i in list(item_counts.keys()):
         if isinstance(i, Literal):
@@ -527,9 +532,42 @@ def mutate_fix_var(
         sample_max_n=config.MUTPB_FV_SAMPLE_MAXN,
         limit=config.MUTPB_FV_QUERY_LIMIT,
 ):
-    """Chooses a random variable from the pattern(node or edge).
-
-    Substitutes it with all possible fixed variables.
+    """Finds possible fixations for a randomly selected variable of the pattern.
+
+    This is the a very important mutation of the gp learner, as it is the main
+    source of actually gaining information from the SPARQL endpoint.
+
+    The outline of the mutation is as follows:
+    - If not passed in, randomly selects a variable (rand_var) of the pattern
+      (node or edge var, excluding ?source and ?target).
+    - Randomly selects a subset of up to gtp_sample_max_n GTPs with
+      probabilities according to their remaining gains. The number of GTPs
+      picked is randomized (see below).
+    - Issues SPARQL queries to find possible fixations for the selected variable
+      under the previously selected GTPs subset. Counts the fixation's
+      occurrences wrt. the GTPs and sorts the result descending by these counts.
+    - Limits the result rows to deal with potential long-tails.
+    - Filters the resulting rows with mutate_fix_var_filter.
+    - From the limited, filtered result rows randomly selects up to sample_max_n
+      candidate fixations with probabilities according to their counts.
+    - For each candidate fixation returns a child in which rand_var is replaced
+      with the candidate fixation.
+
+    The reasons for fixing rand_var based on a randomly sized subset of GTPs
+    are efficiency and shadowing problems with common long-tails. Due to the
+    later imposed limit (which is vital in real world use-cases),
+    a few remaining GTPs that share more than `limit` potential fixations (so
+    have a common long-tail) could otherwise hide solutions for other
+    remaining GTPs. This can be the case if these common fixations have low
+    fitness. By randomizing the subset size, we will eventually (and more
+    likely) select other combinations of remaining GTPs.
+
+    :param gtp_sample_max_n: Maximum GTPs subset size to base fixations on.
+    :param rand_var: If given uses this variable instead of a random one.
+    :param sample_max_n: Maximum number of children.
+    :param limit: SPARQL limit for the top-k result rows.
+    :return: A list of children in which the selected variable is substituted
+        with fixation candidates wrt. GTPs.
     """
     assert isinstance(child, GraphPattern)
     assert isinstance(gtp_scores, GTPScores)