Assorted performance improvements (#1061)

* Remove custom fullmatch function (it was pretty slow)
Change regular expressions to match full strings and just use .match.

* Add _str_to_dict_path fastpath for common case

* Use dict rather than graph_objs when build up dendrogram

* Store trace index in the trace object. This lets us avoid the _index_is calls that are expensive for large trace arrays.

* Fix fig error when setting nested property on Frame hierarchy

* Additional optimizations to use trace._trace_ind rather than _index_is

Author: jonmmease

plotly/basedatatypes.py

@@ -31,20 +31,11 @@
 Undefined = object()
 
 
-# back-port of fullmatch from Py3.4+
-def fullmatch(regex, string, flags=0):
-    """Emulate python-3.4 re.fullmatch()."""
-    if 'pattern' in dir(regex):
-        regex_string = regex.pattern
-    else:
-        regex_string = regex
-    return re.match("(?:" + regex_string + r")\Z", string, flags=flags)
-
-
 class BaseFigure(object):
     """
     Base class for all figure types (both widget and non-widget)
     """
+    _bracket_re = re.compile('^(.*)\[(\d+)\]$')
 
     # Constructor
     # -----------
@@ -143,7 +134,7 @@ class is a subclass of both BaseFigure and widgets.DOMWidget.
         self._data_defaults = [{} for _ in data]
 
         # ### Reparent trace objects ###
-        for trace in data:
+        for trace_ind, trace in enumerate(data):
             # By setting the trace's parent to be this figure, we tell the
             # trace object to use the figure's _data and _data_defaults
             # dicts to get/set it's properties, rather than using the trace
@@ -153,6 +144,9 @@ class is a subclass of both BaseFigure and widgets.DOMWidget.
             # We clear the orphan props since the trace no longer needs then
             trace._orphan_props.clear()
 
+            # Set trace index
+            trace._trace_ind = trace_ind
+
         # Layout
         # ------
         # ### Construct layout validator ###
@@ -463,6 +457,7 @@ def data(self, new_data):
                 old_trace = self.data[i]
                 old_trace._orphan_props.update(deepcopy(old_trace._props))
                 old_trace._parent = None
+                old_trace._trace_ind = None
 
         # ### Compute trace props / defaults after removal ###
         traces_props_post_removal = [t for t in self._data]
@@ -527,6 +522,10 @@ def data(self, new_data):
         # Update trace objects tuple
         self._data_objs = list(new_data)
 
+        # Update trace indexes
+        for trace_ind, trace in enumerate(self._data_objs):
+            trace._trace_ind = trace_ind
+
     # Restyle
     # -------
     def plotly_restyle(self, restyle_data, trace_indexes=None, **kwargs):
@@ -690,7 +689,7 @@ def _restyle_child(self, child, key_path_str, val):
 
         # Compute trace index
         # -------------------
-        trace_index = BaseFigure._index_is(self.data, child)
+        trace_index = child._trace_ind
 
         # Not in batch mode
         # -----------------
@@ -743,7 +742,12 @@ def _str_to_dict_path(key_path_str):
         -------
         tuple[str | int]
         """
-        if isinstance(key_path_str, tuple):
+        if isinstance(key_path_str, string_types) and \
+                '.' not in key_path_str and \
+                '[' not in key_path_str:
+            # Fast path for common case that avoids regular expressions
+            return (key_path_str,)
+        elif isinstance(key_path_str, tuple):
             # Nothing to do
             return key_path_str
         else:
@@ -752,11 +756,9 @@ def _str_to_dict_path(key_path_str):
 
             # Split out bracket indexes.
             # e.g. ['foo', 'bar[1]'] -> ['foo', 'bar', '1']
-            bracket_re = re.compile('(.*)\[(\d+)\]')
             key_path2 = []
             for key in key_path:
-                match = fullmatch(bracket_re, key)
-                #match = bracket_re.fullmatch(key)
+                match = BaseFigure._bracket_re.match(key)
                 if match:
                     key_path2.extend(match.groups())
                 else:
@@ -1065,6 +1067,10 @@ def add_traces(self, data, rows=None, cols=None):
         # Validate traces
         data = self._data_validator.validate_coerce(data)
 
+        # Set trace indexes
+        for ind, new_trace in enumerate(data):
+            new_trace._trace_ind = ind + len(self.data)
+
         # Validate rows / cols
         n = len(data)
         BaseFigure._validate_rows_cols('rows', n, rows)
@@ -1212,14 +1218,9 @@ def _get_child_props(self, child):
         """
         # Try to find index of child as a trace
         # -------------------------------------
-        try:
-            trace_index = BaseFigure._index_is(self.data, child)
-        except ValueError as _:
-            trace_index = None
-
-        # Child is a trace
-        # ----------------
-        if trace_index is not None:
+        if isinstance(child, BaseTraceType):
+            # ### Child is a trace ###
+            trace_index = child._trace_ind
             return self._data[trace_index]
 
         # Child is the layout
@@ -1247,16 +1248,10 @@ def _get_child_prop_defaults(self, child):
         -------
         dict
         """
-        # Try to find index of child as a trace
-        # -------------------------------------
-        try:
-            trace_index = BaseFigure._index_is(self.data, child)
-        except ValueError as _:
-            trace_index = None
-
         # Child is a trace
         # ----------------
-        if trace_index is not None:
+        if isinstance(child, BaseTraceType):
+            trace_index = child._trace_ind
             return self._data_defaults[trace_index]
 
         # Child is the layout
@@ -3365,7 +3360,7 @@ class BaseLayoutType(BaseLayoutHierarchyType):
                              'polar']
 
     _subplotid_prop_re = re.compile(
-        '(' + '|'.join(_subplotid_prop_names) + ')(\d+)')
+        '^(' + '|'.join(_subplotid_prop_names) + ')(\d+)$')
 
     @property
     def _subplotid_validators(self):
@@ -3429,16 +3424,14 @@ def _process_kwargs(self, **kwargs):
         unknown_kwargs = {
             k: v
             for k, v in kwargs.items()
-            if not fullmatch(self._subplotid_prop_re, k)
-            # if not self._subplotid_prop_re.fullmatch(k)
+            if not self._subplotid_prop_re.match(k)
         }
         super(BaseLayoutHierarchyType, self)._process_kwargs(**unknown_kwargs)
 
         subplot_kwargs = {
             k: v
             for k, v in kwargs.items()
-            if fullmatch(self._subplotid_prop_re, k)
-            #if self._subplotid_prop_re.fullmatch(k)
+            if self._subplotid_prop_re.match(k)
         }
 
         for prop, value in subplot_kwargs.items():
@@ -3458,8 +3451,7 @@ def _set_subplotid_prop(self, prop, value):
         # Get regular expression match
         # ----------------------------
         # Note: we already tested that match exists in the constructor
-        # match = self._subplotid_prop_re.fullmatch(prop)
-        match = fullmatch(self._subplotid_prop_re, prop)
+        match = self._subplotid_prop_re.match(prop)
         subplot_prop = match.group(1)
         suffix_digit = int(match.group(2))
 
@@ -3520,7 +3512,7 @@ def _strip_subplot_suffix_of_1(self, prop):
         # Handle subplot suffix digit of 1
         # --------------------------------
         # Remove digit of 1 from subplot id (e.g.. xaxis1 -> xaxis)
-        match = fullmatch(self._subplotid_prop_re, prop)
+        match = self._subplotid_prop_re.match(prop)
 
         if match:
             subplot_prop = match.group(1)
@@ -3580,7 +3572,7 @@ def __setitem__(self, prop, value):
 
         # Check for subplot assignment
         # ----------------------------
-        match = fullmatch(self._subplotid_prop_re, prop)
+        match = self._subplotid_prop_re.match(prop)
         if match is None:
             # Set as ordinary property
             super(BaseLayoutHierarchyType, self).__setitem__(prop, value)
@@ -3594,8 +3586,7 @@ def __setattr__(self, prop, value):
         """
         # Check for subplot assignment
         # ----------------------------
-        # match = self._subplotid_prop_re.fullmatch(prop)
-        match = fullmatch(self._subplotid_prop_re, prop)
+        match = self._subplotid_prop_re.match(prop)
         if match is None:
             # Set as ordinary property
             super(BaseLayoutHierarchyType, self).__setattr__(prop, value)
@@ -3649,6 +3640,7 @@ class BaseTraceHierarchyType(BasePlotlyType):
 
     def __init__(self, plotly_name, **kwargs):
         super(BaseTraceHierarchyType, self).__init__(plotly_name, **kwargs)
+
     def _send_prop_set(self, prop_path_str, val):
         if self.parent:
             # ### Inform parent of restyle operation ###
@@ -3680,6 +3672,9 @@ def __init__(self, plotly_name, **kwargs):
         # ### Callbacks to be called on selection ###
         self._select_callbacks = []
 
+        # ### Trace index in figure ###
+        self._trace_ind = None
+
     # uid
     # ---
     # All trace types must have a top-level UID
@@ -3951,6 +3946,11 @@ def _send_prop_set(self, prop_path_str, val):
         # propagated to parents
         pass
 
+    def _restyle_child(self, child, key_path_str, val):
+        # Note: Frames are not supported by FigureWidget, and updates are not
+        # propagated to parents
+        pass
+
     def on_change(self, callback, *args):
         raise NotImplementedError(
             'Change callbacks are not supported on Frames')

plotly/basewidget.py

@@ -9,7 +9,7 @@
 
 import ipywidgets as widgets
 from traitlets import List, Unicode, Dict, observe, Integer
-from .basedatatypes import BaseFigure, BasePlotlyType, fullmatch
+from .basedatatypes import BaseFigure, BasePlotlyType
 from .callbacks import (BoxSelector, LassoSelector,
                         InputDeviceState, Points)
 from .serializers import custom_serializers
@@ -550,7 +550,7 @@ def _handler_js2py_layoutDelta(self, change):
             # may include axes that weren't explicitly defined by the user.
             for proppath in delta_transform:
                 prop = proppath[0]
-                match = fullmatch(self.layout._subplotid_prop_re, prop)
+                match = self.layout._subplotid_prop_re.match(prop)
                 if match and prop not in self.layout:
                     # We need to create a subplotid object
                     self.layout[prop] = {}

plotly/figure_factory/_dendrogram.py

@@ -288,11 +288,12 @@ def get_dendrogram_traces(self, X, colorscale, distfun, linkagefun, hovertext):
             hovertext_label = None
             if hovertext:
                 hovertext_label = hovertext[i]
-            trace = graph_objs.Scatter(
+            trace = dict(
+                type='scatter',
                 x=np.multiply(self.sign[self.xaxis], xs),
                 y=np.multiply(self.sign[self.yaxis], ys),
                 mode='lines',
-                marker=graph_objs.scatter.Marker(color=colors[color_key]),
+                marker=dict(color=colors[color_key]),
                 text=hovertext_label,
                 hoverinfo='text'
             )