Implementing %degreeDistribution magic command

(cherry picked from commit 37e84b4bb9c21c9b19ab259df6a0a93009e20add)

Author: Erdem Sariyuce

ChangeLog.md

@@ -3,7 +3,7 @@
 Starting with v1.31.6, this file will contain a record of major features and updates made in each release of graph-notebook.
 
 ## Upcoming
-- Added %degreeDistribution magic command ([PR](https://github.com/aws/graph-notebook/pull/749))
+- Added %degreeDistribution magic command ([PR](https://github.com/aws/graph-notebook/pull/749)) TODO: add to the specific release below when it's released
 - Locked numba dependency to 0.60.0 to avoid numpy conflict ([Link to PR](https://github.com/aws/graph-notebook/pull/735))
 - Fixed library target for nbclassic nbextension for graph_notebook_widget ([Link to PR](https://github.com/aws/graph-notebook/pull/739))
 

pyproject.toml

@@ -46,6 +46,7 @@ dependencies = [
     'networkx==2.4',
     'numpy>=1.23.5,<1.24.0',
     'pandas>=2.1.0,<=2.2.2',
+    'matplotlib>=3.9.4',
 
     # Graph databases and query languages
     'gremlinpython>=3.5.1,<=3.7.2',

requirements.txt

@@ -18,6 +18,7 @@ itables>=2.0.0,<=2.1.0
 networkx==2.4
 numpy>=1.23.5,<1.24.0
 pandas>=2.1.0,<=2.2.2
+matplotlib>=3.9.4
 
 # Graph databases and query languages
 gremlinpython>=3.5.1,<=3.7.2

src/graph_notebook/magics/graph_magic.py

@@ -19,6 +19,9 @@
 import numpy as np
 import matplotlib.pyplot as plt
 
+import numpy as np
+import matplotlib.pyplot as plt
+
 from ipyfilechooser import FileChooser
 from enum import Enum
 from copy import copy
@@ -57,6 +60,7 @@
     SPARQL_EXPLAIN_MODES, OPENCYPHER_EXPLAIN_MODES, GREMLIN_EXPLAIN_MODES, \
     OPENCYPHER_PLAN_CACHE_MODES, OPENCYPHER_DEFAULT_TIMEOUT, OPENCYPHER_STATUS_STATE_MODES, \
     normalize_service_name, NEPTUNE_DB_SERVICE_NAME, NEPTUNE_ANALYTICS_SERVICE_NAME, GRAPH_PG_INFO_METRICS, TRAVERSAL_DIRECTIONS, \
+    normalize_service_name, NEPTUNE_DB_SERVICE_NAME, NEPTUNE_ANALYTICS_SERVICE_NAME, GRAPH_PG_INFO_METRICS, TRAVERSAL_DIRECTIONS, \
     GREMLIN_PROTOCOL_FORMATS, DEFAULT_HTTP_PROTOCOL, DEFAULT_WS_PROTOCOL, GRAPHSONV4_UNTYPED, \
     GREMLIN_SERIALIZERS_WS, get_gremlin_serializer_mime, normalize_protocol_name, generate_snapshot_name)
 from graph_notebook.network import SPARQLNetwork
@@ -3928,15 +3932,29 @@ def handle_opencypher_status(self, line, local_ns):
 
 
 
-    # %degreeDistribution. Takes traversalDirection, vertexLabels, edgeLabels parameters, and visualizes
-    # the degree distribution.
+    # %degreeDistribution magic command.
+    # It obtains the degree distribution of a graph in the form of a visual histogram in notebook. Histogram simply
+    # shows the number of vertices with a given degree, where degree is shown on the x-axis and the count on y-axis.
+    # It takes traversalDirection [both (default), inbound, outbound], vertexLabels [default is empty list],
+    # edgeLabels parameters [default is empty list], and then gives the histogram for the specified degree
+    # (both/in/out) distribution of the vertices in the graph filtered by the specified vertex labels and edge
+    # labels. Parameters can be defined as command line argument and/or through the dropdown widgets.
+    # Example usages:
+    # > %degreeDistribution
+    # > %degreeDistribution --traversalDirection inbound
+    # > %degreeDistribution --traversalDirection inbound --vertexLabels airport country
+
     # TODO: Error handling
 
     @line_magic
     @needs_local_scope
     @display_exceptions
     @neptune_graph_only
     def degreeDistribution(self, line, local_ns: dict = None):
+        if not self.client.is_analytics_domain():
+            print("This command is only supported for Neptune Analytics domains.")
+            return
+    
         parser = argparse.ArgumentParser()
 
         # Get the vertexLabels and edgeLabels from graph summary, to be shown in the widgets for selection.
@@ -3950,18 +3968,23 @@ def degreeDistribution(self, line, local_ns: dict = None):
             print(f"Error retrieving graph summary: {e}")
             return
 
-        # traversalDirection parameter
+        # traversalDirection: Type of the degree computed:
+        # - inbound: Counts only the incoming edges for each vertex
+        # - outbound: Counts only the outgoing edges for each vertex
+        # - both [default]: Counts both the incoming and outgoing edges for each vertex.
         parser.add_argument('--traversalDirection', nargs='?', type=str.lower, default='both',
                             help=f'Type of the degree for which the distribution is shown. Valid inputs: {TRAVERSAL_DIRECTIONS}. '
                                  f'Default: both.',
                             choices=TRAVERSAL_DIRECTIONS)
 
-        # vertexLabels parameter
+        # vertexLabels: List of the vertex labels, space separated, for which the degrees are computed:
+        # - default value is empty list, which means the degrees are computed for any vertex label.
         parser.add_argument('--vertexLabels', nargs='*', default=[],
                             help="The vertex labels for which the induced graph is considered and the degree distribution is shown. "
                                  "If not supplied, we will default to using all the vertex labels.")
 
-        # edgeLabels parameter
+        # edgeLabels: List of the edge labels, space separated, for which the degrees are computed:
+        # - default value is empty list, which means the degrees are computed for any edge label.
         parser.add_argument('--edgeLabels', nargs='*', default=[],
                             help="The edge labels for which the degree distribution is shown. If not supplied, "
                                  "we will default to using all the edge labels.")
@@ -3973,7 +3996,8 @@ def degreeDistribution(self, line, local_ns: dict = None):
 
         args = parser.parse_args(line.split())
 
-        # Put the selection specified on the command line, if any; o.w. default is 'both'
+        # If the traversalDirection parameter selection is specified on the command line, it is shown as the default
+        # in the dropdown menu. Othweise, the default in the dropdown is 'both'
         td_val = args.traversalDirection
         td_val = td_val.lower() if td_val else 'both' 
 
@@ -3985,7 +4009,9 @@ def degreeDistribution(self, line, local_ns: dict = None):
             value = td_val
         )
 
-        # Put the vertex label(s) specified on the command line, if any; o.w. default is all the vertex labels (denoted by [])
+        # Existing vertex labels in the graph are shown in the dropdown menu. If any vertex label is specified on
+        # the command line, they are shown to be selected in the dropdown menu. Otherwise, no label is selected
+        # in the dropdown menu, which means any label and all the labels are considered in the computation.
         available_vertex_labels = sorted(available_vertex_labels)
         selected_vlabels = args.vertexLabels if args.vertexLabels else []
         vertex_labels_select = widgets.SelectMultiple(
@@ -3996,7 +4022,9 @@ def degreeDistribution(self, line, local_ns: dict = None):
             value = selected_vlabels
         )
 
-        # Put the edge label(s) specified on the command line, if any; o.w. default is all the edge labels (denoted by [])
+        # Existing edge labels in the graph are shown in the dropdown menu. If any edge label is specified on
+        # the command line, they are shown to be selected in the dropdown menu. Otherwise, no label is selected
+        # in the dropdown menu, which means any label and all the labels are considered in the computation.
         available_edge_labels = sorted(available_edge_labels)
         selected_elabels = args.edgeLabels if args.edgeLabels else []
         edge_labels_select = widgets.SelectMultiple(
@@ -4024,7 +4052,7 @@ def on_button_clicked(b):
 
             # Call the function with the selected parameters
             with output:
-                res = self.callDD(td, vlabels, elabels, local_ns)
+                res = self.execute_degree_distribution_query(td, vlabels, elabels, local_ns)
 
                 # Retrieve the distribution
                 pairs = np.array(res['results'][0]['output']['distribution'])
@@ -4041,7 +4069,7 @@ def on_button_clicked(b):
 
         submit_button.on_click(on_button_clicked)
 
-    def callDD (self, td, vlabels, elabels, local_ns):
+    def execute_degree_distribution_query (self, td, vlabels, elabels, local_ns):
         query_parts = [f'traversalDirection: "{td}"']
 
         if vlabels:
@@ -4055,8 +4083,7 @@ def callDD (self, td, vlabels, elabels, local_ns):
         # Construct the query
         line = "CALL neptune.algo.degreeDistribution({" + ", ".join(query_parts) + "}) YIELD output RETURN output"
 
-        # oc_rebuild_args = (f"{f'--store-to js --silent'}")
-        oc_rebuild_args = (f"{f'--store-to js'}")
+        oc_rebuild_args = (f"{f'--store-to js --silent'}")
 
         self.handle_opencypher_query(oc_rebuild_args, line, local_ns)
 
@@ -4068,14 +4095,17 @@ def plot_interactive_degree_distribution(self, unique_degrees, counts, max_deg,
         min_deg = 0
 
         def update_plot(scale_type, bin_type, bin_width, y_max, x_range, show_mindeg, show_maxdeg):
-            marker_size = 50
-            alpha = 0.6
+            # Start timing
+            start_time = time.time()
+            
+            alpha = 1
             plt.clf()
 
             # Get zero degree count
             zero_idx = np.where(unique_degrees == 0)[0]
             zero_degree_count = counts[zero_idx[0]] if len(zero_idx) > 0 else 0
 
+            isolateds_exist = zero_degree_count > 0
             # Get non-zero degrees and counts
             mask = unique_degrees > 0
             filtered_degrees = unique_degrees[mask]
@@ -4085,8 +4115,8 @@ def update_plot(scale_type, bin_type, bin_width, y_max, x_range, show_mindeg, sh
             if len(filtered_degrees) == 0:
                 min_deg = 0
             else:
-                min_deg = np.min(filtered_degrees)            
-            
+                min_deg = np.min(filtered_degrees)                
+
             n_bins = 1
             # Create histogram only if there are non-zero degree nodes
             if len(filtered_degrees) > 0:
@@ -4111,20 +4141,32 @@ def update_plot(scale_type, bin_type, bin_width, y_max, x_range, show_mindeg, sh
                         label='Raw', color='#000000')
 
             # Plot zero degree node count separately
-            if zero_degree_count > 0:
-                plt.bar(0, zero_degree_count, color='red', 
-                    label='Isolated', alpha=alpha, width=0.2)
+            if isolateds_exist:
+                # Use a special x position for zero degree nodes in log scale
+                zero_x_pos = 0.1 if scale_type in ['Log-Log', 'Log(x)-Linear(y)'] else 0
+                plt.bar(zero_x_pos, zero_degree_count, color='red', 
+                    label='Isolated', alpha=alpha, width=0.1 if scale_type in ['Log-Log', 'Log(x)-Linear(y)'] else 2)
 
             plt.xlim(x_range[0], x_range[1])
 
+            if isolateds_exist:
+                plt.xlim(x_range[0], x_range[1])
+
             # Set scales based on selection
             if scale_type == 'Log-Log':
                 plt.xscale('log')
                 plt.yscale('log')
-                plt.xlim(x_range[0]+1, x_range[1])
+                if isolateds_exist:
+                    plt.xlim(0.05, x_range[1])
+                else:
+                    plt.xlim(x_range[0]+0.05, x_range[1])
+
             elif scale_type == 'Log(x)-Linear(y)':
                 plt.xscale('log')
-                plt.xlim(x_range[0]+1, x_range[1])
+                if isolateds_exist:
+                    plt.xlim(0.05, x_range[1])
+                else:
+                    plt.xlim(x_range[0]+0.05, x_range[1])
             elif scale_type == 'Linear(x)-Log(y)':
                 plt.yscale('log')
 
@@ -4143,13 +4185,21 @@ def update_plot(scale_type, bin_type, bin_width, y_max, x_range, show_mindeg, sh
             plt.legend()   
 
             plt.title(f'Degree Distribution')
-            
+
+            # End timing and display
+            end_time = time.time()
+            runtime = end_time - start_time
+                        
             # Update statistics
             with stats_output:
                 stats_output.clear_output(wait=True)
                 total_nodes = sum(counts)
                 total_edges = sum(d * c for d, c in zip(unique_degrees, counts)) // 2
                 avg_degree = sum(d * c for d, c in zip(unique_degrees, counts)) / total_nodes
+                
+                print(f"Render time: {runtime:.3f} seconds")
+                print(f"--------------------")
+
                 print(f"Number of nodes: {total_nodes}")
                 print(f"Number of edges: {total_edges}")
                 print(f"Number of isolated nodes: {zero_degree_count}")
@@ -4178,18 +4228,56 @@ def update_plot(scale_type, bin_type, bin_width, y_max, x_range, show_mindeg, sh
             description='Binning:'
         )
 
+        # Define a function to update bin_width_widget based on bin_type
+        def update_bin_width_widget(change):
+            if change['new'] == 'Logarithmic':
+                # For logarithmic binning, use a FloatSlider with smaller values
+                bin_width_widget.min = 1.00
+                bin_width_widget.max = 10.00
+                bin_width_widget.step = 0.01
+                bin_width_widget.value = 1.00
+                bin_width_widget.readout_format = '.2f'
+                bin_width_widget.disabled = False
+            elif change['new'] == 'Raw':
+                # For raw binning, disable the widget
+                bin_width_widget.value = 1
+                bin_width_widget.disabled = True
+            else:
+                # For linear binning, use integer values
+                bin_width_widget.min = 1
+                bin_width_widget.max = (max_deg+2)/10
+                bin_width_widget.step = 1
+                bin_width_widget.value = 1
+                bin_width_widget.readout_format = 'd'
+                bin_width_widget.disabled = False
+
+        def update_y_max_widget(change):
+            if bin_widget.value == 'Raw':
+                # For raw data, use the original max count
+                y_max_widget.max = max_count * 1.1
+                y_max_widget.value = max_count * 1.1
+            elif bin_widget.value == 'Linear':
+                y_max_widget.max = max_count * bin_width_widget.value * 0.5
+                y_max_widget.value = max_count * bin_width_widget.value * 0.5
+            else: # 'Logarithmic'
+                y_max_widget.max = max_count * (10 ** bin_width_widget.value) * 0.5
+                y_max_widget.value = max_count * (10 ** bin_width_widget.value) * 0.5
+                    
         # Bin width widget, integer options in [1, 1+(max_deg/2)] interval 
-        # TODO: if logarithmic binning, a much smaller range makes more sense 
-        bin_width_widget = widgets.IntSlider(
+        bin_width_widget = widgets.FloatSlider(
             value=1,
             min=1,
-            max=(max_deg+2)/2,
+            max=(max_deg+2)/10,
             step=1,
             description='Bin width:',
             tooltip=('For linear binning: actual width\n'
                     'For log binning: multiplicative factor')
         )
 
+        # Observe changes to bin_width_widget and bin_widget
+        bin_width_widget.observe(update_y_max_widget, names='value')
+        bin_widget.observe(update_y_max_widget, names='value')
+
         # Upper limit for y-axis range, enables zooming (lower limit is always zero)
         y_max_widget = widgets.IntSlider(
             value=max_count * 1.1,
@@ -4203,7 +4291,7 @@ def update_plot(scale_type, bin_type, bin_width, y_max, x_range, show_mindeg, sh
         x_range_widget = widgets.FloatRangeSlider(            
             min=0,
             max=max_deg * 1.1 + 5,
-            value=[min, max],
+            value=[0, max_deg * 1.1 + 5],
             step=1,
             description='x-axis range:',
             disabled=False,