Hparams: Use DataProvider.list_hyperparameters() to generate HparamInfos (#6391)

Use the result from a DataProvider.list_hyperparameters() call to
generate HparamInfos for operations in the Hparams plugin. Generate the
HparamInfo name, display_name, data_type, and domain from each
Hyperparameter.

We use the same two-step strategy for generating these as is done with
tensor-based hparams. The tensor-based hparams has one operation,
`hparams_metadata()` for retrieving the tensor data, and then passes the
result to `experiment_metadata()` to generate the HparamInfos and
MetricInfos. Similarly, we add an `hparams_from_data_provider()`
operation for retrieving the hyperparameter data and we pass the result
to `experiment_metadata()`.

Note: We don't yet generate `MetricInfos`. We don't yet have enough
information from the data provider.

Note: We don't yet merge the new HparamInfos with the tensor-based
HparamInfos.

Author: bmd3k

tensorboard/plugins/hparams/backend_context.py

@@ -20,11 +20,18 @@
 import os
 
 
+from tensorboard.data import provider
 from tensorboard.plugins.hparams import api_pb2
 from tensorboard.plugins.hparams import metadata
 from google.protobuf import json_format
 from tensorboard.plugins.scalar import metadata as scalar_metadata
 
+_DISCRETE_DOMAIN_TYPE_TO_DATA_TYPE = {
+    provider.HyperparameterDomainType.DISCRETE_BOOL: api_pb2.DATA_TYPE_BOOL,
+    provider.HyperparameterDomainType.DISCRETE_FLOAT: api_pb2.DATA_TYPE_FLOAT64,
+    provider.HyperparameterDomainType.DISCRETE_STRING: api_pb2.DATA_TYPE_STRING,
+}
+
 
 class Context:
     """Wraps the base_plugin.TBContext to stores additional data shared across
@@ -51,33 +58,57 @@ def __init__(self, tb_context, max_domain_discrete_len=10):
         self._max_domain_discrete_len = max_domain_discrete_len
 
     def experiment_from_metadata(
-        self, ctx, experiment_id, hparams_run_to_tag_to_content
+        self,
+        ctx,
+        experiment_id,
+        hparams_run_to_tag_to_content,
+        data_provider_hparams,
     ):
-        """Returns the experiment protobuffer defining the experiment.
-
-        Accepts a dict containing the plugin contents for all summary tags
-        associated with the hparams plugin, as an optimization for callers
-        who already have this information available, so that this function
-        can minimize its calls to the underlying `DataProvider`.
+        """Returns the experiment proto defining the experiment.
 
         This method first attempts to find a metadata.EXPERIMENT_TAG tag and
-        retrieve the associated protobuffer. If no such tag is found, the method
-        will attempt to build a minimal experiment protobuffer by scanning for
-        all metadata.SESSION_START_INFO_TAG tags (to compute the hparam_infos
-        field of the experiment) and for all scalar tags (to compute the
-        metric_infos field of the experiment).
+        retrieve the associated proto.
+
+        If no such tag is found, the method will attempt to build a minimal
+        experiment proto by scanning for all metadata.SESSION_START_INFO_TAG
+        tags (to compute the hparam_infos field of the experiment) and for all
+        scalar tags (to compute the metric_infos field of the experiment).
+
+        If no metadata.EXPERIMENT_TAG nor metadata.SESSION_START_INFO_TAG tags
+        are found, then will build an experiment proto using the results from
+        DataProvider.list_hyperparameters().
+
+        Args:
+          experiment_id: String, from `plugin_util.experiment_id`.
+          hparams_run_to_tag_to_content: The output from an hparams_metadata()
+            call. A dict `d` such that `d[run][tag]` is a `bytes` value with the
+            summary metadata content for the keyed time series.
+          data_provider_hparams: The ouput from an hparams_from_data_provider()
+            call, corresponding to DataProvider.list_hyperparameters().
+            A Collection[provider.Hyperparameter].
 
         Returns:
-          The experiment protobuffer. If no tags are found from which an experiment
-          protobuffer can be built (possibly, because the event data has not been
-          completely loaded yet), returns an entirely empty experiment.
+          The experiment proto. If no data is found for an experiment proto to
+          be built, returns an entirely empty experiment.
         """
         experiment = self._find_experiment_tag(hparams_run_to_tag_to_content)
         if experiment:
             return experiment
-        return self._compute_experiment_from_runs(
+
+        experiment_from_runs = self._compute_experiment_from_runs(
             ctx, experiment_id, hparams_run_to_tag_to_content
         )
+        if experiment_from_runs:
+            return experiment_from_runs
+
+        experiment_from_data_provider_hparams = (
+            self._experiment_from_data_provider_hparams(data_provider_hparams)
+        )
+        return (
+            experiment_from_data_provider_hparams
+            if experiment_from_data_provider_hparams
+            else api_pb2.Experiment()
+        )
 
     @property
     def tb_context(self):
@@ -159,6 +190,12 @@ def read_last_scalars(self, ctx, experiment_id, run_tag_filter):
             for (run, tag_to_data) in data_provider_output.items()
         }
 
+    def hparams_from_data_provider(self, ctx, experiment_id):
+        """Calls DataProvider.list_hyperparameters() and returns the result."""
+        return self._tb_context.data_provider.list_hyperparameters(
+            ctx, experiment_ids=[experiment_id]
+        )
+
     def _find_experiment_tag(self, hparams_run_to_tag_to_content):
         """Finds the experiment associcated with the metadata.EXPERIMENT_TAG
         tag.
@@ -179,7 +216,7 @@ def _compute_experiment_from_runs(
     ):
         """Computes a minimal Experiment protocol buffer by scanning the runs.
 
-        Returns an empty Experiment if there are no hparam infos logged.
+        Returns None if there are no hparam infos logged.
         """
         hparam_infos = self._compute_hparam_infos(hparams_run_to_tag_to_content)
         if hparam_infos:
@@ -188,6 +225,9 @@ def _compute_experiment_from_runs(
             )
         else:
             metric_infos = []
+        if not hparam_infos and not metric_infos:
+            return None
+
         return api_pb2.Experiment(
             hparam_infos=hparam_infos, metric_infos=metric_infos
         )
@@ -273,6 +313,56 @@ def _compute_hparam_info_from_values(self, name, values):
 
         return result
 
+    def _experiment_from_data_provider_hparams(
+        self,
+        data_provider_hparams,
+    ):
+        """Returns an experiment protobuffer based on data provider hparams.
+
+        Args:
+          data_provider_hparams: The ouput from an hparams_from_data_provider()
+            call, corresponding to DataProvider.list_hyperparameters().
+            A Collection[provider.Hyperparameter].
+
+        Returns:
+          The experiment proto. If there are no hyperparameters in the input,
+          returns None.
+        """
+        if not data_provider_hparams:
+            return None
+
+        hparam_infos = [
+            self._convert_data_provider_hparam(dp_hparam)
+            for dp_hparam in data_provider_hparams
+        ]
+        return api_pb2.Experiment(hparam_infos=hparam_infos)
+
+    def _convert_data_provider_hparam(self, dp_hparam):
+        """Builds an HParamInfo message from data provider Hyperparameter.
+
+        Args:
+          dp_hparam: The provider.Hyperparameter returned by the call to
+            provider.DataProvider.list_hyperparameters().
+
+        Returns:
+          An HParamInfo to include in the Experiment.
+        """
+        hparam_info = api_pb2.HParamInfo(
+            name=dp_hparam.hyperparameter_name,
+            display_name=dp_hparam.hyperparameter_display_name,
+        )
+        if dp_hparam.domain_type == provider.HyperparameterDomainType.INTERVAL:
+            hparam_info.type = api_pb2.DATA_TYPE_FLOAT64
+            (dp_hparam_min, dp_hparam_max) = dp_hparam.domain
+            hparam_info.domain_interval.min_value = dp_hparam_min
+            hparam_info.domain_interval.max_value = dp_hparam_max
+        elif dp_hparam.domain_type in _DISCRETE_DOMAIN_TYPE_TO_DATA_TYPE.keys():
+            hparam_info.type = _DISCRETE_DOMAIN_TYPE_TO_DATA_TYPE.get(
+                dp_hparam.domain_type
+            )
+            hparam_info.domain_discrete.extend(dp_hparam.domain)
+        return hparam_info
+
     def _compute_metric_infos(
         self, ctx, experiment_id, hparams_run_to_tag_to_content
     ):