From 0845fe9be0a0d02ee00038e82d6188f8f8fd5726 Mon Sep 17 00:00:00 2001 From: Leon Lin Date: Fri, 29 Aug 2025 02:54:23 +0000 Subject: [PATCH 1/6] Fix get catalog await --- .../datafusion/tests/integration_datafusion_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/integrations/datafusion/tests/integration_datafusion_test.rs b/crates/integrations/datafusion/tests/integration_datafusion_test.rs index c079dce865..cb2809d0be 100644 --- a/crates/integrations/datafusion/tests/integration_datafusion_test.rs +++ b/crates/integrations/datafusion/tests/integration_datafusion_test.rs @@ -440,7 +440,7 @@ async fn test_metadata_table() -> Result<()> { #[tokio::test] async fn test_insert_into() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog(); + let iceberg_catalog = get_iceberg_catalog().await; let namespace = NamespaceIdent::new("test_insert_into".to_string()); set_test_namespace(&iceberg_catalog, &namespace).await?; @@ -553,7 +553,7 @@ fn get_nested_struct_type() -> StructType { #[tokio::test] async fn test_insert_into_nested() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog(); + let iceberg_catalog = get_iceberg_catalog().await; let namespace = NamespaceIdent::new("test_insert_nested".to_string()); set_test_namespace(&iceberg_catalog, &namespace).await?; let table_name = "nested_table"; From 0a059a68649664e6fd2267eb4d32df07121dbcd8 Mon Sep 17 00:00:00 2001 From: Leon Lin Date: Fri, 29 Aug 2025 03:25:41 +0000 Subject: [PATCH 2/6] Attempt to fix the binding failure with setting underlying catalog safe --- crates/integrations/datafusion/src/table/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs index a8c49837c3..f16bed3e91 100644 --- a/crates/integrations/datafusion/src/table/mod.rs +++ b/crates/integrations/datafusion/src/table/mod.rs @@ -52,7 +52,7 @@ pub struct IcebergTableProvider { /// A reference-counted arrow `Schema`. schema: ArrowSchemaRef, /// The catalog that the table belongs to. - catalog: Option>, + catalog: Option>, } impl IcebergTableProvider { From 12aa489adbbf635f9228630c292ed5a12379b1ea Mon Sep 17 00:00:00 2001 From: Leon Lin Date: Fri, 29 Aug 2025 03:44:38 +0000 Subject: [PATCH 3/6] Not sure about the issue, trying out fixes --- bindings/python/tests/test_datafusion_table_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/tests/test_datafusion_table_provider.py b/bindings/python/tests/test_datafusion_table_provider.py index 915b787697..50b7f2ddac 100644 --- a/bindings/python/tests/test_datafusion_table_provider.py +++ b/bindings/python/tests/test_datafusion_table_provider.py @@ -27,7 +27,7 @@ import datafusion assert ( - datafusion.__version__ >= "45" + datafusion.__version__ >= "47" ) # iceberg table provider only works for datafusion >= 45 From c228eb6ffafd498a64ae42426d4bb6bfac3c8beb Mon Sep 17 00:00:00 2001 From: Leon Lin Date: Fri, 29 Aug 2025 09:37:45 +0000 Subject: [PATCH 4/6] Revert attempt to fix python binding issue --- bindings/python/tests/test_datafusion_table_provider.py | 2 +- crates/integrations/datafusion/src/table/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/tests/test_datafusion_table_provider.py b/bindings/python/tests/test_datafusion_table_provider.py index 50b7f2ddac..915b787697 100644 --- a/bindings/python/tests/test_datafusion_table_provider.py +++ b/bindings/python/tests/test_datafusion_table_provider.py @@ -27,7 +27,7 @@ import datafusion assert ( - datafusion.__version__ >= "47" + datafusion.__version__ >= "45" ) # iceberg table provider only works for datafusion >= 45 diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs index f16bed3e91..a8c49837c3 100644 --- a/crates/integrations/datafusion/src/table/mod.rs +++ b/crates/integrations/datafusion/src/table/mod.rs @@ -52,7 +52,7 @@ pub struct IcebergTableProvider { /// A reference-counted arrow `Schema`. schema: ArrowSchemaRef, /// The catalog that the table belongs to. - catalog: Option>, + catalog: Option>, } impl IcebergTableProvider { From 1e4eb00b3a910586348dc2b4ecd4cbb7e31a63f7 Mon Sep 17 00:00:00 2001 From: Leon Lin Date: Sat, 30 Aug 2025 23:32:57 +0000 Subject: [PATCH 5/6] pin arrow and datafusion version --- bindings/python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 7b2d3d591f..346ee168da 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -44,7 +44,7 @@ python-source = "python" ignore = ["F403", "F405"] [tool.hatch.envs.dev] -dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "pyarrow>=17.0.0", "datafusion>=45", "pyiceberg[sql-sqlite]>=0.9.1"] +dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "pyarrow==16.*", "datafusion==47.*", "pyiceberg[sql-sqlite]>=0.9.1"] [tool.hatch.envs.dev.scripts] build = "maturin build --out dist --sdist" From 709d17b9ddcf1ec6d0fa03d33ac2c0df04d7b407 Mon Sep 17 00:00:00 2001 From: Leon Lin Date: Sun, 31 Aug 2025 00:49:05 +0000 Subject: [PATCH 6/6] Update comment and retrigger flaky CI --- bindings/python/tests/test_datafusion_table_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/tests/test_datafusion_table_provider.py b/bindings/python/tests/test_datafusion_table_provider.py index 915b787697..5c6a15eb78 100644 --- a/bindings/python/tests/test_datafusion_table_provider.py +++ b/bindings/python/tests/test_datafusion_table_provider.py @@ -27,8 +27,8 @@ import datafusion assert ( - datafusion.__version__ >= "45" -) # iceberg table provider only works for datafusion >= 45 + datafusion.__version__ >= "47" +) # iceberg table provider only works for datafusion >= 47 @pytest.fixture(scope="session")