refactoring of deep-narrow-paths query, sparql (uses template now), and args passing, also tons of minor things

Author: joernhees

gp_learner.py

@@ -48,7 +48,7 @@
 from gp_query import predict_query
 from gp_query import query_time_hard_exceeded
 from gp_query import query_time_soft_exceeded
-from gp_query import variable_substitution_deep_narrow_mut_query
+from gp_query import dnp_query
 from gp_query import variable_substitution_query
 from graph_pattern import canonicalize
 from graph_pattern import gen_random_var
@@ -655,15 +655,12 @@ def mutate_fix_var(
 
 
 def _mutate_deep_narrow_path_helper(
-            sparql,
-            timeout,
-            gtp_scores,
-            child,
-            edge_var,
-            node_var,
-            gtp_sample_n=config.MUTPB_FV_RGTP_SAMPLE_N,
-            limit_res=config.MUTPB_DN_QUERY_LIMIT,
-            sample_n=config.MUTPB_FV_SAMPLE_MAXN,
+        sparql, timeout, gtp_scores, child, edge_var, node_var,
+        gtp_sample_n=config.MUTPB_FV_RGTP_SAMPLE_N,
+        max_node_count=config.MUTPB_DN_MAX_NODE_COUNT,
+        min_edge_count=config.MUTPB_DN_MIN_EDGE_COUNT,
+        limit=config.MUTPB_DN_QUERY_LIMIT,
+        sample_n=config.MUTPB_FV_SAMPLE_MAXN,
 ):
     assert isinstance(child, GraphPattern)
     assert isinstance(gtp_scores, GTPScores)
@@ -675,10 +672,15 @@ def _mutate_deep_narrow_path_helper(
     gtp_sample_n = random.randint(1, gtp_sample_n)
 
     ground_truth_pairs = gtp_scores.remaining_gain_sample_gtps(
-     n=gtp_sample_n)
-    t, substitution_counts = variable_substitution_deep_narrow_mut_query(
-     sparql, timeout, child, edge_var, node_var, ground_truth_pairs,
-     limit_res)
+        max_n=gtp_sample_n)
+    t, substitution_counts = dnp_query(
+        sparql, timeout, child, ground_truth_pairs,
+        edge_var=edge_var,
+        node_var=node_var,
+        max_node_count=max_node_count,
+        min_edge_count=min_edge_count,
+        limit=limit,
+    )
     edge_count, node_sum_count = substitution_counts
     if not node_sum_count:
         # the current pattern is unfit, as we can't find anything fulfilling it
@@ -698,7 +700,7 @@ def _mutate_deep_narrow_path_helper(
     for edge, node_sum in node_sum_count.items():
         ec = edge_count[edge]
         prio[edge] = ec / (node_sum / ec)  # ec / AVG degree
-    # randomly pick n of the substitutions with a prob ~ to their counts
+    # randomly pick n of the substitutions with a prob ~ to their prios
     edges, prios = zip(*prio.most_common())
 
     substs = sample_from_list(edges, prios, sample_n)
@@ -715,9 +717,9 @@ def _mutate_deep_narrow_path_helper(
     fixed = True
     orig_child = child
     children = [
-     GraphPattern(child, mapping={edge_var: subst})
-     for subst in substs
-     ]
+        GraphPattern(child, mapping={edge_var: subst})
+        for subst in substs
+    ]
     children = [
         c if fit_to_live(c) else orig_child
         for c in children

gp_query.py

@@ -458,43 +458,55 @@ def _var_subst_res_update(res, update, **_):
     res += update
 
 
-def variable_substitution_deep_narrow_mut_query(
-        sparql, timeout, graph_pattern, edge_var, node_var,
-        source_target_pairs, limit_res, batch_size=config.BATCH_SIZE):
+def dnp_query(
+        sparql, timeout, graph_pattern, source_target_pairs,
+        edge_var, node_var, max_node_count, min_edge_count, limit,
+        batch_size=config.BATCH_SIZE
+):
     _vars, _values, _ret_val_mapping = _get_vars_values_mapping(
         graph_pattern, source_target_pairs)
-    _edge_var_node_var_and_vars = (edge_var, node_var, _vars)
     return _multi_query(
         sparql, timeout, graph_pattern, source_target_pairs, batch_size,
-        _edge_var_node_var_and_vars, _values, _ret_val_mapping,
-        _var_subst_dnp_res_init, _var_subst_dnp_chunk_q,
-        _var_subst_dnp_chunk_result_ext,
-        _res_update=_var_subst_dnp_update,
-        limit=limit_res,
+        _vars, _values, _ret_val_mapping,
+        _dnp_res_init, _dnp_chunk_q,
+        _dnp_chunk_result_ext,
+        _res_update=_dnp_res_update,
+        edge_var=edge_var,
+        node_var=node_var,
+        max_node_count=max_node_count,
+        min_edge_count=min_edge_count,
+        limit=limit,
         # non standard, passed via **kwds, see handling below
     )
 
 
 # noinspection PyUnusedLocal
-def _var_subst_dnp_res_init(_, **kwds):
+def _dnp_res_init(_, **kwds):
     return Counter(), Counter()
 
 
-def _var_subst_dnp_chunk_q(gp, _edge_var_node_var_and_vars,
-                           values_chunk, limit):
-    edge_var, node_var, _vars = _edge_var_node_var_and_vars
-    return gp.to_find_edge_var_for_narrow_path_query(
+def _dnp_chunk_q(
+        gp, _vars, values_chunk,
+        edge_var, node_var, max_node_count, min_edge_count, limit,
+        **_
+):
+    return gp.to_deep_narrow_path_query(
         edge_var=edge_var,
         node_var=node_var,
         vars_=_vars,
         values={_vars: values_chunk},
-        limit_res=limit)
+        max_node_count=max_node_count,
+        min_edge_count=min_edge_count,
+        limit=limit,
+    )
 
 
 # noinspection PyUnusedLocal
-def _var_subst_dnp_chunk_result_ext(
-        q_res, _edge_var_node_var_and_vars, _, **kwds):
-    edge_var, node_var, _vars = _edge_var_node_var_and_vars
+def _dnp_chunk_result_ext(
+        q_res, _vars, _,
+        edge_var,
+        **kwds
+):
     chunk_edge_count, chunk_node_sum = Counter(), Counter()
     res_rows_path = ['results', 'bindings']
     bindings = sparql_json_result_bindings_to_rdflib(
@@ -510,14 +522,12 @@ def _var_subst_dnp_chunk_result_ext(
     return chunk_edge_count, chunk_node_sum,
 
 
-def _var_subst_dnp_update(res, up, **_):
+def _dnp_res_update(res, up, **_):
     edge_count, node_sum_count = res
-    try:
+    if up:
         chunk_edge_count, chunk_node_sum = up
         edge_count.update(chunk_edge_count)
         node_sum_count.update(chunk_node_sum)
-    except ValueError:
-        pass
 
 
 def generate_stps_from_gp(sparql, gp):

graph_pattern.py

@@ -16,6 +16,7 @@
 import logging
 import random
 import string
+import textwrap
 
 import deap
 import deap.base
@@ -31,7 +32,6 @@
 import six
 
 from utils import URIShortener
-import config
 
 logger = logging.getLogger(__name__)
 
@@ -717,84 +717,101 @@ def to_count_var_over_values_query(self, var, vars_, values, limit):
         res += 'LIMIT %d\n' % limit
         return self._sparql_prefix(res)
 
-    def to_find_edge_var_for_narrow_path_query(
-            self, edge_var, node_var, vars_, values, limit_res,
-            filter_node_count=config.MUTPB_DN_FILTER_NODE_COUNT,
-            filter_edge_count=config.MUTPB_DN_FILTER_EDGE_COUNT,
+    def to_deep_narrow_path_query(
+            self, edge_var, node_var, vars_, values,
+            limit, max_node_count, min_edge_count,
     ):
-        """Counts possible substitutions for edge_var to get a narrow path
+        """Counts possible substitutions for edge_var to get a narrow path.
 
         Meant to perform a query like this:
-        SELECT *
-        {
+         PREFIX dbr: <http://dbpedia.org/resource/>
+         SELECT * WHERE {
           {
-            SELECT
-              ?edge_var
-              (COUNT(*) AS ?edge_var_count)
-              (MAX(?node_var_count) AS ?max_node_count)
-              (COUNT(*)/AVG(?node_var_count) as ?prio_var)
-            {
-              SELECT DISTINCT
-                ?source ?target ?edge_var (COUNT(?node_var) AS ?node_var_count)
-              {
-                VALUES (?source ?target) {
-                  (dbr:Adolescence dbr:Youth)
-                  (dbr:Adult dbr:Child)
-                  (dbr:Angel dbr:Heaven)
-                  (dbr:Arithmetic dbr:Mathematics)
-                }
-                ?node_var ?edge_var ?source .
-                ?source dbo:wikiPageWikiLink ?target .
-              }
+           SELECT ?edge_var
+                  (SUM(?node_var_count) AS ?node_var_sum)
+                  (COUNT(?source && ?target) AS ?edge_var_count)
+                  (MAX(?node_var_count) AS ?max_node_count)
+           WHERE {
+            SELECT DISTINCT ?source ?target ?edge_var
+                            (COUNT(?node_var) AS ?node_var_count)
+            WHERE {
+             VALUES (?source ?target) {
+              (dbr:Barrel dbr:Wine)
+              (dbr:Barrister dbr:Law)
+              (dbr:Beak dbr:Bird)
+              (dbr:Blanket dbr:Bed)
+             }
+             ?node_var ?edge_var ?source .
+             ?source <http://dbpedia.org/ontology/wikiPageWikiLink> ?target .
             }
-            GROUP BY ?edge_var
-            ORDER BY DESC(?edge_var_count)
+           }
+           GROUP BY ?edge_var
           }
-          FILTER(?max_node_count < 10 && ?edge_var_count > 1)
-        }
-        ORDER BY DESC(?prio_var)
-        LIMIT 32
+          FILTER(?max_node_count <= 10
+                 && ?edge_var_count >= 2)
+         }
+         ORDER BY DESC(?edge_var_count) ASC(?node_var_sum)
+         LIMIT 32
+
+        The idea here is to expand a random node (?source in the example above)
+        with new variable triple and then try to fix its edge in a way that the
+        degree (?node_var_count) isn't too high (<= max_node_count). We're also
+        interested in the avg degree being low. In light of query chunking the
+        sum is returned here (instead of AVG).
+
+        Apart from minimizing the degrees, we would also like to maximize the
+        number of stps an ?edge_var fixation is valid for (?edge_var_count).
+
+        See gp_learner.mutate_deep_narrow_path() for more.
 
         :param edge_var: Edge variable to find substitution for.
         :param node_var: Node variable to count.
         :param vars_: List of vars to fix values for (e.g. ?source, ?target).
         :param values: List of value lists for vars_.
-        :param filter_node_count: Filter on node count of edge variable.
-        :param filter_edge_count: Filter for edge count of triples.
-        :param limit_res : limit result size
+        :param max_node_count: Filter on node count of edge variable.
+        :param min_edge_count: Filter for edge count of triples.
+        :param limit : limit result size.
         :return: Query String.
         """
 
-        res = 'SELECT * WHERE {\n'
-        res += ' {\n'\
-               '  SELECT %s (SUM (?node_var_count) AS %s) (COUNT(%s) AS %s) ' \
-               '(MAX(?node_var_count) AS ?max_node_count) WHERE {\n' % (
-                     edge_var.n3(),
-                     NODE_VAR_SUM.n3(),
-                     ' && '.join([v.n3() for v in vars_]),
-                     EDGE_VAR_COUNT.n3(), )
-        res += '    SELECT DISTINCT %s %s (COUNT(%s) AS ?node_var_count) ' \
-               'WHERE {\n   ' % (' '.join([v.n3() for v in vars_]),
-                                 edge_var.n3(), node_var.n3(), )
-        res += self._sparql_values_part(values)
-
-        # triples part
-        tres = []
-        for s, p, o in self:
-            tres.append('%s %s %s .' % (s.n3(), p.n3(), o.n3()))
-        indent = ' ' * 3
-        triples = indent + ('\n' + indent).join(tres) + '\n'
-        res += triples
-        res += '    }\n'\
-               '   }\n'
-        res += '   GROUP BY %s\n' % edge_var.n3()
-        res += '  }\n'
-        res += '  FILTER(?max_node_count < %d && %s > %d)\n' \
-               % (filter_node_count, EDGE_VAR_COUNT.n3(),
-                  filter_edge_count)
-        res += '}\n'
-        res += 'ORDER BY ASC(%s)\n' % NODE_VAR_SUM.n3()
-        res += 'LIMIT %d' % limit_res
+        res = '''\
+            SELECT * WHERE {
+             {
+              SELECT %(edge_var)s
+                     (SUM(?node_var_count) AS %(node_var_sum)s)
+                     (COUNT(%(vars_and)s) AS %(edge_var_count)s)
+                     (MAX(?node_var_count) AS ?max_node_count)
+              WHERE {
+               SELECT DISTINCT %(vars)s %(edge_var)s
+                               (COUNT(%(node_var)s) AS ?node_var_count)
+               WHERE {\n%(values_part)s                %(triples)s
+               }
+              }
+              GROUP BY %(edge_var)s
+             }
+             FILTER(?max_node_count <= %(max_node_count)d
+                    && %(edge_var_count)s >= %(min_edge_count)d)
+            }
+            ORDER BY DESC(%(edge_var_count)s) ASC(%(node_var_sum)s)
+            LIMIT %(limit)d
+        ''' % {
+            # TODO: adapt self._sparql_values_part for template use (indent)
+            'edge_var': edge_var.n3(),
+            'node_var_sum': NODE_VAR_SUM.n3(),
+            'vars_and': ' && '.join([v.n3() for v in vars_]),
+            'edge_var_count': EDGE_VAR_COUNT.n3(),
+            'vars': ' '.join([v.n3() for v in vars_]),
+            'node_var': node_var.n3(),
+            'values_part': self._sparql_values_part(
+                values, indent='                '),
+            'triples': '\n                '.join(
+                '%s %s %s .' % (s.n3(), p.n3(), o.n3()) for s, p, o in self
+            ),
+            'limit': limit,
+            'max_node_count': max_node_count,
+            'min_edge_count': min_edge_count,
+        }
+        res = textwrap.dedent(res)
         return self._sparql_prefix(res)
 
     def to_dict(self):