Hparams: Populate differs field for TF summary hparams. (#6582)

Currently `differs` fields are not populated for hparams written by TF summary API.

Also changed the boolean hparam domain discrete values to be the actual values used in the sessions rather than both True and False.

#hparams

Author: yatbear

tensorboard/plugins/hparams/backend_context.py

@@ -104,8 +104,7 @@ def experiment_from_metadata(
             ctx, experiment_id, include_metrics, hparams_run_to_tag_to_content
         )
         if experiment_from_runs:
-            # TODO(yatbear): Apply `hparams_limit` to `experiment_from_runs` after `differs`
-            # fields are populated in `_compute_hparam_info_from_values()`.
+            # TODO(yatbear): Apply `hparams_limit` to `experiment_from_runs`.
             return experiment_from_runs
 
         experiment_from_data_provider_hparams = (
@@ -331,24 +330,27 @@ def _compute_hparam_info_from_values(self, name, values):
         if result.type == api_pb2.DATA_TYPE_UNSET:
             return None
 
-        # TODO(yatbear): Populate `differs` fields for hparams once go/tbpr/6574 is merged.
         if result.type == api_pb2.DATA_TYPE_STRING:
             distinct_string_values = set(
                 _protobuf_value_to_string(v)
                 for v in values
                 if _can_be_converted_to_string(v)
             )
             result.domain_discrete.extend(distinct_string_values)
+            result.differs = len(distinct_string_values) > 1
 
         if result.type == api_pb2.DATA_TYPE_BOOL:
-            result.domain_discrete.extend([True, False])
+            distinct_bool_values = set(v.bool_value for v in values)
+            result.domain_discrete.extend(distinct_bool_values)
+            result.differs = len(distinct_bool_values) > 1
 
         if result.type == api_pb2.DATA_TYPE_FLOAT64:
             # Always uses interval domain type for numeric hparam values.
             distinct_float_values = sorted([v.number_value for v in values])
             if distinct_float_values:
                 result.domain_interval.min_value = distinct_float_values[0]
                 result.domain_interval.max_value = distinct_float_values[-1]
+                result.differs = len(set(distinct_float_values)) > 1
 
         return result
 

tensorboard/plugins/hparams/backend_context_test.py

@@ -256,6 +256,7 @@ def test_experiment_with_session_tags(self):
                 min_value: 100.0
                 max_value: 300.0
               }
+              differs: true
             },
             hparam_infos: {
               name: 'lr'
@@ -264,6 +265,7 @@ def test_experiment_with_session_tags(self):
                 min_value: 0.01
                 max_value: 0.05
               }
+              differs: true
             },
             hparam_infos: {
               name: 'model_type'
@@ -272,6 +274,106 @@ def test_experiment_with_session_tags(self):
                 values: [{string_value: 'CNN'},
                          {string_value: 'LATTICE'}]
               }
+              differs: true
+            }
+            metric_infos: {
+              name: {group: '', tag: 'accuracy'}
+            }
+            metric_infos: {
+              name: {group: '', tag: 'loss'}
+            }
+            metric_infos: {
+              name: {group: 'eval', tag: 'loss'}
+            }
+            metric_infos: {
+              name: {group: 'train', tag: 'loss'}
+            }
+        """
+        actual_exp = self._experiment_from_metadata()
+        _canonicalize_experiment(actual_exp)
+        self.assertProtoEquals(expected_exp, actual_exp)
+
+    def test_experiment_with_session_tags_differs_field(self):
+        self.session_1_start_info_ = """
+            hparams: [
+              {key: 'bool_hparam_differs_true' value: {bool_value: false}},
+              {key: 'bool_hparam_differs_true' value: {bool_value: true}},
+              {key: 'float_hparam_differs_false' value: {number_value: 1024}},
+              {key: 'float_hparam_differs_true' value: {number_value: 0.01}},
+              {key: 'string_hparams_differs_false' value: {string_value: 'momentum'}},
+              {key: 'string_hparams_differs_true' value: {string_value: 'CNN'}}
+            ]
+        """
+        self.session_2_start_info_ = """
+            hparams:[
+              {key: 'bool_hparam_differs_true' value: {bool_value: false}},
+              {key: 'float_hparam_differs_false' value: {number_value: 1024}},
+              {key: 'float_hparam_differs_true' value: {number_value: 0.02}},
+              {key: 'string_hparams_differs_false' value: {string_value: 'momentum'}},
+              {key: 'string_hparams_differs_true' value: {string_value: 'LATTICE'}}
+            ]
+        """
+        self.session_3_start_info_ = """
+            hparams:[
+              {key: 'bool_hparam_differs_false' value: {bool_value: false}},
+              {key: 'bool_hparam_differs_true' value: {bool_value: false}},
+              {key: 'float_hparam_differs_false' value: {number_value: 1024}},
+              {key: 'float_hparam_differs_true' value: {number_value: 0.05}},
+              {key: 'string_hparams_differs_false' value: {string_value: 'momentum'}},
+              {key: 'string_hparams_differs_true' value: {string_value: 'CNN'}}
+            ]
+        """
+        expected_exp = """
+            hparam_infos: {
+              name: 'bool_hparam_differs_false'
+              type: DATA_TYPE_BOOL
+              domain_discrete: {
+                values: [{bool_value: false}]
+              }
+              differs: false
+            }
+            hparam_infos: {
+              name: 'bool_hparam_differs_true'
+              type: DATA_TYPE_BOOL
+              domain_discrete: {
+                values: [{bool_value: false}, {bool_value: true}]
+              }
+              differs: true
+            }
+            hparam_infos: {
+              name: 'float_hparam_differs_false'
+              type: DATA_TYPE_FLOAT64
+              domain_interval {
+                min_value: 1024
+                max_value: 1024
+              }
+              differs: false
+            }
+            hparam_infos: {
+              name: 'float_hparam_differs_true'
+              type: DATA_TYPE_FLOAT64
+              domain_interval {
+                min_value: 0.01
+                max_value: 0.05
+              }
+              differs: true
+            }
+            hparam_infos: {
+              name: 'string_hparams_differs_false'
+              type: DATA_TYPE_STRING
+              domain_discrete: {
+                values: [{string_value: 'momentum'}]
+              }
+              differs: false
+            }
+            hparam_infos: {
+              name: 'string_hparams_differs_true'
+              type: DATA_TYPE_STRING
+              domain_discrete: {
+                values: [{string_value: 'CNN'},
+                         {string_value: 'LATTICE'}]
+              }
+              differs: true
             }
             metric_infos: {
               name: {group: '', tag: 'accuracy'}
@@ -317,6 +419,7 @@ def test_experiment_with_session_tags_different_hparam_types(self):
                 values: [{string_value: '100.0'},
                          {string_value: 'true'}]
               }
+              differs: true
             }
             hparam_infos: {
               name: 'lr'
@@ -325,6 +428,7 @@ def test_experiment_with_session_tags_different_hparam_types(self):
                 values: [{string_value: '0.01'},
                          {string_value: '0.02'}]
               }
+              differs: true
             }
             hparam_infos: {
               name: 'model_type'
@@ -333,6 +437,7 @@ def test_experiment_with_session_tags_different_hparam_types(self):
                 values: [{string_value: 'CNN'},
                          {string_value: 'LATTICE'}]
               }
+              differs: true
             }
             metric_infos: {
               name: {group: '', tag: 'accuracy'}
@@ -354,12 +459,12 @@ def test_experiment_with_session_tags_different_hparam_types(self):
     def test_experiment_with_session_tags_bool_types(self):
         self.session_1_start_info_ = """
             hparams:[
-              {key: 'batch_size' value: {bool_value: true}}
+              {key: 'use_batch_norm' value: {bool_value: true}}
             ]
         """
         self.session_2_start_info_ = """
             hparams:[
-              {key: 'batch_size' value: {bool_value: true}}
+              {key: 'use_batch_norm' value: {bool_value: true}}
             ]
         """
         self.session_3_start_info_ = """
@@ -368,11 +473,12 @@ def test_experiment_with_session_tags_bool_types(self):
         """
         expected_exp = """
             hparam_infos: {
-              name: 'batch_size'
+              name: 'use_batch_norm'
               type: DATA_TYPE_BOOL
               domain_discrete: {
-                values: [{bool_value: true}, {bool_value: false}]
+                values: [{bool_value: true}]
               }
+              differs: false
             }
             metric_infos: {
               name: {group: '', tag: 'accuracy'}