Merge pull request #886 from netenglabs/device-namespace-fixes

Device namespace fixes

Author: ddutt

suzieq/config/schema/namespace.avsc

@@ -61,6 +61,12 @@
             "display": 7,
             "description": "Is MLAG used in this namespace"
         },
+        {
+            "name": "lastUpdate",
+            "type": "timestamp",
+            "display": 8,
+            "description": "The last time the poller updated this namespace"
+        },
         {
             "name": "namespace",
             "type": "string",

suzieq/engines/pandas/device.py

@@ -52,12 +52,15 @@ def get(self, **kwargs):
         if view == 'latest' and 'status' in df.columns:
             df['status'] = np.where(df.active, df['status'], 'dead')
 
-        poller_df = pd.DataFrame()
+        poller_df: pd.DataFrame = pd.DataFrame()
+
+        # Retrieve the devices that have possibly not made it into the device
+        # list because they never could connect due to any reason
         if not ignore_neverpoll:
             poller_df = self._get_table_sqobj('sqPoller').get(
                 namespace=kwargs.get('namespace', []),
                 hostname=kwargs.get('hostname', []),
-                service='device',
+                service='device', status='fail',
                 columns='namespace hostname status timestamp'.split())
 
         if not poller_df.empty:
@@ -77,7 +80,7 @@ def get(self, **kwargs):
                 .reset_index(drop=True)
 
             df = df.merge(poller_df, on=['namespace', 'hostname'],
-                          how='outer', suffixes=['', '_y'])  \
+                          how='outer', suffixes=('', '_y')) \
                 .fillna({'bootupTimestamp': 0,
                          'active': True})
 
@@ -121,6 +124,7 @@ def get(self, **kwargs):
         # The poller merge kills the filtering we did earlier, so redo:
         if status:
             df = df.loc[df.status.isin(status)]
+
         if os_version:
             opdict = {'>': operator.gt, '<': operator.lt, '>=': operator.ge,
                       '<=': operator.le, '=': operator.eq, '!': operator.ne}

suzieq/engines/pandas/namespace.py

@@ -92,7 +92,6 @@ def get(self, **kwargs):
         if 'sqvers' in fields:
             newdf['sqvers'] = self.schema.version
         newdf['active'] = True
-        newdf = self._handle_user_query_str(newdf, user_query)
 
         # Look for the rest of info only in selected namepaces
         newdf['lastUpdate'] = nsgrp['timestamp'].max() \

tests/integration/sqcmds/common-samples/describe.yml

@@ -405,12 +405,14 @@ tests:
     {"name": "hasOspf", "type": "boolean", "key": "", "display": 4, "description":
     "Is OSPF used in this namespace"}, {"name": "hasVxlan", "type": "boolean", "key":
     "", "display": 6, "description": "Is VXLAN used in this namespace"}, {"name":
-    "lastUpdate", "type": "timestamp", "key": "", "display": 8, "description": "Time
-    when any service was last polled"}, {"name": "namespace", "type": "string", "key":
-    0, "display": 0, "description": "Namespace associated with this record"}, {"name":
-    "serviceCnt", "type": "long", "key": "", "display": 2, "description": "Number
-    of services polled in this namespace"}, {"name": "sqvers", "type": "string", "key":
-    "", "display": "", "description": "Schema version, not selectable"}]'
+    "lastUpdate", "type": "timestamp", "key": "", "display": 8, "description": "The
+    last time the poller updated this namespace"}, {"name": "lastUpdate", "type":
+    "timestamp", "key": "", "display": 8, "description": "Time when any service was
+    last polled"}, {"name": "namespace", "type": "string", "key": 0, "display": 0,
+    "description": "Namespace associated with this record"}, {"name": "serviceCnt",
+    "type": "long", "key": "", "display": 2, "description": "Number of services polled
+    in this namespace"}, {"name": "sqvers", "type": "string", "key": "", "display":
+    "", "description": "Schema version, not selectable"}]'
 - command: path describe --format=json
   data-directory: tests/data/parquet
   marks: path describe