ci/maint: unblock fsspec and s3 tests (#339)

* update how we connect to s3 in ci

* update region

* use other bucket w creds

* check on databricks older dependency

* streamline dependencies

* debug why skip isnt working

* move up debug statement

* check if underlying board protocol is dbc

* check hasattr first

* maybe fsspec changed?

* Update pyproject.toml

* Update pyproject.toml

Author: isabelizimm

.github/workflows/ci.yml

@@ -60,7 +60,6 @@ jobs:
           project_id: siuba-tests
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           export_default_credentials: true
-
       - name: Run tests
         shell: bash
         run: |

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.1'
-
 services:
 
   rsconnect:

pins/tests/helpers.py

@@ -26,7 +26,7 @@
 
 BOARD_CONFIG = {
     "file": {"path": ["PINS_TEST_FILE__PATH", None]},
-    "s3": {"path": ["PINS_TEST_S3__PATH", "ci-pins"]},
+    "s3": {"path": ["PINS_TEST_S3__PATH", "pins-test-hadley"]},
     "gcs": {"path": ["PINS_TEST_GCS__PATH", "pins-python"]},
     "abfs": {"path": ["PINS_TEST_AZURE__PATH", "ci-pins"]},
     "rsc": {"path": ["PINS_TEST_RSC__PATH", RSC_SERVER_URL]},
@@ -65,7 +65,10 @@ def wrapper(*args, **kwargs):
                     board = arg_value
                     break
 
-        if board and board.fs.protocol == "dbc":
+        if board and (
+            board.fs.protocol == "dbc"
+            or (hasattr(board.fs, "fs") and board.fs.fs.protocol == "dbc")
+        ):
             pytest.skip("All Databricks tests must be read only")
 
         return func(*args, **kwargs)

pyproject.toml

@@ -21,7 +21,7 @@ requires-python = ">=3.9"
 dynamic = ["version"]
 dependencies = [
     "appdirs<2",                # Using appdirs rather than platformdirs is deliberate, see https://github.com/rstudio/pins-python/pull/239
-    "fsspec>=2022.2",
+    "fsspec>=2022.2,<2025.9",
     "humanize>=1",
     "importlib-metadata>=4.4",
     "importlib-resources>=1.3",
@@ -37,13 +37,13 @@ dependencies = [
 
 [project.optional-dependencies]
 aws = ["s3fs"]
-azure = ["adlfs"]
+azure = ["adlfs>=2024.4.1"]
 check = [
     "pre-commit",
     "pyright==1.1.372", # Pinned; manually sync with .github/workflows/code-checks.yml
     "ruff==0.5.4", # Pinned; manually sync with pre-commit-config.yaml
     "types-appdirs",
-    "databricks-sdk"
+    "pins[databricks]"
 ]
 databricks = ["databricks-sdk"]
 doc = [
@@ -55,18 +55,18 @@ doc = [
 ]
 gcs = ["gcsfs"]
 test = [
-    "adlfs>=2024.4.1",
+    "pins[azure]",
+    "pins[gcs]",
+    "pins[databricks]",
+    "pins[aws]",
     "fastparquet",
-    "gcsfs",
     "pip-tools",
     "pyarrow",
     "pytest==7.1.3",
     "pytest-cases",
     "pytest-dotenv",
     "pytest-parallel",
-    "s3fs",
     "rdata",
-    "databricks-sdk",
 ]
 
 [build-system]