refactor(lookml): Fixed log levels for source reporters (#15318)

Author: askumar27

metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py

@@ -492,7 +492,8 @@ def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManife
         manifest_file = folder / "manifest.lkml"
 
         if not manifest_file.exists():
-            self.reporter.info(
+            self.reporter.report_warning(
+                title="Manifest File Missing",
                 message="manifest.lkml file missing from project",
                 context=str(manifest_file),
             )

metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py

@@ -917,7 +917,7 @@ def _report_field_splitting_stats(
 
         if success_rate == 100:
             # All queries succeeded
-            self.reporter.report_warning(
+            self.reporter.info(
                 title="Field Splitting Statistics - Complete Success",
                 message="Field splitting completed successfully",
                 context=f"View-name: {self.view_context.name()}, Success rate: {success_rate:.1f}%, Total fields: {total_fields}, Chunks: {total_chunks}, Upstream tables: {upstream_tables}, Column lineages: {column_lineages}{fallback_info}",

metadata-ingestion/tests/integration/lookml/test_lookml.py

@@ -98,7 +98,8 @@ def test_lookml_ingest(pytestconfig, tmp_path, mock_time):
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     mce_helpers.check_golden_file(
         pytestconfig,
@@ -133,7 +134,8 @@ def test_lookml_refinement_ingest(pytestconfig, tmp_path, mock_time):
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     golden_path = test_resources_dir / "refinements_ingestion_golden.json"
     mce_helpers.check_golden_file(
@@ -164,7 +166,8 @@ def test_lookml_refinement_include_order(pytestconfig, tmp_path, mock_time):
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     golden_path = test_resources_dir / "refinement_include_order_golden.json"
     mce_helpers.check_golden_file(
@@ -363,7 +366,8 @@ def test_lookml_ingest_offline(pytestconfig, tmp_path, mock_time):
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     mce_helpers.check_golden_file(
         pytestconfig,
@@ -409,7 +413,8 @@ def test_lookml_ingest_offline_with_model_deny(pytestconfig, tmp_path, mock_time
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     mce_helpers.check_golden_file(
         pytestconfig,
@@ -465,7 +470,8 @@ def test_lookml_ingest_offline_platform_instance(pytestconfig, tmp_path, mock_ti
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     mce_helpers.check_golden_file(
         pytestconfig,
@@ -557,7 +563,8 @@ def ingestion_test(
         pipeline.run()
         pipeline.pretty_print_summary()
         pipeline.raise_from_status(raise_warnings=False)
-        assert pipeline.source.get_report().warnings.total_elements == 1
+        # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+        assert pipeline.source.get_report().warnings.total_elements == 2
 
         mce_helpers.check_golden_file(
             pytestconfig,
@@ -612,7 +619,8 @@ def test_lookml_git_info(pytestconfig, tmp_path, mock_time):
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     mce_helpers.check_golden_file(
         pytestconfig,
@@ -673,7 +681,7 @@ def test_reachable_views(pytestconfig, tmp_path, mock_time):
     )
     pipeline.run()
     pipeline.pretty_print_summary()
-    pipeline.raise_from_status(raise_warnings=True)
+    pipeline.raise_from_status(raise_warnings=False)
 
     mce_helpers.check_golden_file(
         pytestconfig,
@@ -736,7 +744,8 @@ def test_hive_platform_drops_ids(pytestconfig, tmp_path, mock_time):
     pipeline.run()
     pipeline.pretty_print_summary()
     pipeline.raise_from_status(raise_warnings=False)
-    assert pipeline.source.get_report().warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert pipeline.source.get_report().warnings.total_elements == 2
 
     events = read_metadata_file(tmp_path / mce_out)
     for mce in events:
@@ -882,7 +891,7 @@ def test_same_name_views_different_file_path(pytestconfig, tmp_path, mock_time):
     )
     pipeline.run()
     pipeline.pretty_print_summary()
-    pipeline.raise_from_status(raise_warnings=True)
+    pipeline.raise_from_status(raise_warnings=False)
 
     mce_helpers.check_golden_file(
         pytestconfig,
@@ -918,7 +927,7 @@ def test_duplicate_field_ingest(pytestconfig, tmp_path, mock_time):
     pipeline = Pipeline.create(new_recipe)
     pipeline.run()
     pipeline.pretty_print_summary()
-    pipeline.raise_from_status(raise_warnings=True)
+    pipeline.raise_from_status(raise_warnings=False)
 
     golden_path = test_resources_dir / "duplicate_field_ingestion_golden.json"
     mce_helpers.check_golden_file(
@@ -953,7 +962,7 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t
     pipeline = Pipeline.create(new_recipe)
     pipeline.run()
     pipeline.pretty_print_summary()
-    pipeline.raise_from_status(raise_warnings=True)
+    pipeline.raise_from_status(raise_warnings=False)
 
     golden_path = test_resources_dir / "vv_lineage_liquid_template_golden.json"
     mce_helpers.check_golden_file(
@@ -978,6 +987,7 @@ def test_view_to_view_lineage_and_lookml_constant(pytestconfig, tmp_path, mock_t
     pipeline = Pipeline.create(new_recipe)
     pipeline.run()
     pipeline.pretty_print_summary()
+    # Expect 1 warning: "LookML constant not found" (no manifest file or skipped views in this test scenario)
     assert pipeline.source.get_report().warnings.total_elements == 1
 
     golden_path = test_resources_dir / "vv_lineage_lookml_constant_golden.json"
@@ -1223,7 +1233,7 @@ def test_field_tag_ingest(pytestconfig, tmp_path, mock_time):
     pipeline = Pipeline.create(new_recipe)
     pipeline.run()
     pipeline.pretty_print_summary()
-    pipeline.raise_from_status(raise_warnings=True)
+    pipeline.raise_from_status(raise_warnings=False)
 
     golden_path = test_resources_dir / "field_tag_ingestion_golden.json"
     mce_helpers.check_golden_file(
@@ -1250,7 +1260,7 @@ def test_drop_hive(pytestconfig, tmp_path, mock_time):
     pipeline = Pipeline.create(new_recipe)
     pipeline.run()
     pipeline.pretty_print_summary()
-    pipeline.raise_from_status(raise_warnings=True)
+    pipeline.raise_from_status(raise_warnings=False)
 
     golden_path = test_resources_dir / "drop_hive_dot_golden.json"
     mce_helpers.check_golden_file(
@@ -1287,7 +1297,7 @@ def test_gms_schema_resolution(pytestconfig, tmp_path, mock_time):
         pipeline = Pipeline.create(new_recipe)
         pipeline.run()
         pipeline.pretty_print_summary()
-        pipeline.raise_from_status(raise_warnings=True)
+        pipeline.raise_from_status(raise_warnings=False)
 
     golden_path = test_resources_dir / "gms_schema_resolution_golden.json"
     mce_helpers.check_golden_file(
@@ -1336,7 +1346,8 @@ def test_unreachable_views(pytestconfig):
     assert (
         len(converted_workunits) == 22
     )  # this num was updated when we converted entities to metadata work units part of SDKv2 migration
-    assert source.reporter.warnings.total_elements == 1
+    # Expect 2 warnings: "Manifest File Missing" and "Skipped View File"
+    assert source.reporter.warnings.total_elements == 2
     assert (
         "The Looker view file was skipped because it may not be referenced by any models."
         in [failure.message for failure in source.get_report().warnings]
@@ -1512,7 +1523,7 @@ def mock_run_inline_query(
         pipeline = Pipeline.create(recipe)
         pipeline.run()
         pipeline.pretty_print_summary()
-        pipeline.raise_from_status(raise_warnings=True)
+        pipeline.raise_from_status(raise_warnings=False)
 
     mce_helpers.check_golden_file(
         pytestconfig,

metadata-ingestion/tests/integration/lookml/test_lookml_field_splitting_integration.py

@@ -166,9 +166,12 @@ def run_inline_query_side_effect(result_format=None, body=None, **kwargs):
         report = pipeline.source.get_report()
         assert report is not None
 
-        # Check that warnings contain field splitting statistics
+        # Check that field splitting statistics are reported (in warnings for partial/failure, or infos for complete success)
         warnings = [w for w in report.warnings if "Field Splitting" in str(w)]
-        assert len(warnings) > 0, "Expected field splitting statistics to be reported"
+        infos = [w for w in report.infos if "Field Splitting" in str(w)]
+        assert len(warnings) > 0 or len(infos) > 0, (
+            "Expected field splitting statistics to be reported"
+        )
 
 
 @freeze_time(FROZEN_TIME)
@@ -205,13 +208,17 @@ def test_integration_individual_field_fallback_on_chunk_failure(
     query_call_count = [0]
 
     def run_inline_query_side_effect(result_format=None, body=None, **kwargs):
-        """Simulate chunk failure, then individual field success."""
+        """Simulate chunk failure, then some individual field failures and successes."""
         if result_format == "sql":
             query_call_count[0] += 1
             # First chunk fails (returns invalid SQL that causes parsing error)
             if query_call_count[0] == 1:
                 return "INVALID SQL SYNTAX ERROR"
-            # Subsequent individual field queries succeed
+            # Some individual field queries fail (to test problematic fields reporting)
+            # Fail every 5th field to create some problematic fields
+            if (query_call_count[0] - 1) % 5 == 0:
+                return "INVALID SQL SYNTAX ERROR"
+            # Other individual field queries succeed
             return f"SELECT {view_name}.field_{query_call_count[0] - 2} FROM test_table"
         return ""
 

metadata-ingestion/tests/unit/lookml/test_lookml_api_based_view_upstream.py

@@ -1356,4 +1356,5 @@ def test_field_splitting_statistics_reporting(
             upstream_instance_with_splitting._get_spr()
 
             # Verify reporter was called for statistics
-            assert upstream_instance_with_splitting.reporter.report_warning.called
+            # For complete success (100% success rate), info() is called instead of report_warning()
+            assert upstream_instance_with_splitting.reporter.info.called