Merge branch 'read-csv-from-directory' of https://github.com/fangchenli/pandas into read-csv-from-directory

Author: fangchenli

.github/workflows/broken-linkcheck.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   cleanup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: github.repository_owner == 'pandas-dev'
     permissions:
       actions: write

.github/workflows/cache-cleanup-daily.yml

@@ -6,7 +6,8 @@ on:
 
 jobs:
   cleanup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
+    if: github.repository_owner == 'pandas-dev'
     steps:
       - name: Clean Cache
         run: |

.github/workflows/cache-cleanup.yml

@@ -28,8 +28,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v5
-      - uses: github/codeql-action/init@v3
+      - uses: github/codeql-action/init@v4
         with:
           languages: ${{ matrix.language }}
-      - uses: github/codeql-action/autobuild@v3
-      - uses: github/codeql-action/analyze@v3
+      - uses: github/codeql-action/autobuild@v4
+      - uses: github/codeql-action/analyze@v4

.github/workflows/codeql.yml

@@ -9,15 +9,6 @@ permissions:
   pull-requests: write
 
 jobs:
-  issue_assign:
-    runs-on: ubuntu-24.04
-    if: (!github.event.issue.pull_request) && github.event.comment.body == 'take'
-    concurrency:
-      group: ${{ github.actor }}-issue-assign
-    steps:
-      - run: |
-          echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
-          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
   preview_docs:
     runs-on: ubuntu-24.04
     if: github.event.issue.pull_request && github.event.comment.body == '/preview'
@@ -28,64 +19,3 @@ jobs:
         with:
           previewer-server: "https://pandas.pydata.org/preview"
           artifact-job: "Doc Build and Upload"
-  asv_run:
-    runs-on: ubuntu-24.04
-    # TODO: Support more benchmarking options later, against different branches, against self, etc
-    if: github.event.issue.pull_request && startsWith(github.event.comment.body, '@github-actions benchmark')
-    defaults:
-      run:
-        shell: bash -el {0}
-    env:
-      ENV_FILE: environment.yml
-      COMMENT: ${{github.event.comment.body}}
-
-    concurrency:
-      # Set concurrency to prevent abuse(full runs are ~5.5 hours !!!)
-      # each user can only run one concurrent benchmark bot at a time
-      # We don't cancel in progress jobs, but if you want to benchmark multiple PRs, you're gonna have
-      # to wait
-      group: ${{ github.actor }}-asv
-      cancel-in-progress: false
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v5
-        with:
-          fetch-depth: 0
-
-        # Although asv sets up its own env, deps are still needed
-        # during discovery process
-      - name: Set up Conda
-        uses: ./.github/actions/setup-conda
-
-      - name: Run benchmarks
-        id: bench
-        continue-on-error: true # asv will exit code 1 for regressions
-        run: |
-          # extracting the regex, see https://stackoverflow.com/a/36798723
-          REGEX=$(echo "$COMMENT" | sed -n "s/^.*-b\s*\(\S*\).*$/\1/p")
-          cd asv_bench
-          asv check -E existing
-          git remote add upstream https://github.com/pandas-dev/pandas.git
-          git fetch upstream
-          asv machine --yes
-          asv continuous -f 1.1 -b $REGEX upstream/main HEAD
-          echo 'BENCH_OUTPUT<<EOF' >> $GITHUB_ENV
-          asv compare -f 1.1 upstream/main HEAD >> $GITHUB_ENV
-          echo 'EOF' >> $GITHUB_ENV
-          echo "REGEX=$REGEX" >> $GITHUB_ENV
-
-      - uses: actions/github-script@v8
-        env:
-          BENCH_OUTPUT: ${{env.BENCH_OUTPUT}}
-          REGEX: ${{env.REGEX}}
-        with:
-          script: |
-            const ENV_VARS = process.env
-            const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: '\nBenchmarks completed. View runner logs here.' + run_url + '\nRegex used: '+ 'regex ' + ENV_VARS["REGEX"] + '\n' + ENV_VARS["BENCH_OUTPUT"]
-            })

.github/workflows/comment-commands.yml

@@ -93,7 +93,7 @@ jobs:
       run: mv doc/build/html web/build/docs
 
     - name: Save website as an artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: website
         path: web/build

.github/workflows/docbuild-and-upload.yml

@@ -181,8 +181,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        # Note: Don't use macOS latest since macos 14 appears to be arm64 only
-        os: [macos-13, macos-14, windows-latest]
+        os: [macos-15-intel, macos-15, windows-2025]
         env_file: [actions-311.yaml, actions-312.yaml, actions-313.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
@@ -243,7 +242,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy -Csetup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython==3.0.10 python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.116.0
+          python -m pip install --no-cache-dir versioneer[toml] cython==3.0.10 python-dateutil pytest>=8.3.4 pytest-xdist>=3.6.1 hypothesis>=6.116.0
           python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
           python -m pip list --no-cache-dir
           PANDAS_CI=1 python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
@@ -280,7 +279,7 @@ jobs:
           /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.116.0
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=8.3.4 pytest-xdist>=3.6.1 hypothesis>=6.116.0
           python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
           python -m pip list --no-cache-dir
 
@@ -321,8 +320,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Separate out macOS 13 and 14, since macOS 14 is arm64 only
-        os: [ubuntu-24.04, macOS-13, macOS-14, windows-latest]
+        os: [ubuntu-24.04, macos-15-intel, macos-15, windows-2025]
 
     timeout-minutes: 90
 
@@ -352,14 +350,12 @@ jobs:
           python --version
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
-          python -m pip install versioneer[toml] python-dateutil tzdata cython hypothesis>=6.116.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
+          python -m pip install versioneer[toml] python-dateutil tzdata cython hypothesis>=6.116.0 pytest>=8.3.4 pytest-xdist>=3.6.1 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps -Csetup-args="--werror"
           python -m pip list
 
       - name: Run Tests
         uses: ./.github/actions/run-tests
-        # TEMP allow this to fail until we fixed all test failures (related to chained assignment warnings)
-        continue-on-error: true
 
   # NOTE: this job must be kept in sync with the Pyodide build job in wheels.yml
   emscripten:
@@ -401,7 +397,7 @@ jobs:
           pyodide build
 
       - name: Set up Node.js
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v6
         with:
           node-version: '20'
 

.github/workflows/unit-tests.yml

@@ -13,6 +13,8 @@
 name: Wheel builder
 
 on:
+  release:
+    types: [published]
   schedule:
   # 3:27 UTC every day
   - cron: "27 3 * * *"
@@ -62,7 +64,7 @@ jobs:
           python -m pip install build
           python -m build --sdist
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         with:
           name: sdist
           path: ./dist/*
@@ -96,12 +98,11 @@ jobs:
         - [ubuntu-24.04, musllinux_x86_64]
         - [ubuntu-24.04-arm, manylinux_aarch64]
         - [ubuntu-24.04-arm, musllinux_aarch64]
-        - [macos-13, macosx_x86_64]
-        # Note: M1 images on Github Actions start from macOS 14
-        - [macos-14, macosx_arm64]
-        - [windows-2022, win_amd64]
+        - [macos-15-intel, macosx_x86_64]
+        - [macos-15, macosx_arm64]
+        - [windows-2025, win_amd64]
         - [windows-11-arm, win_arm64]
-        python: [["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
+        python: [["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"], ["cp314", "3.14"], ["cp314t", "3.14"]]
         include:
         # Build Pyodide wheels and upload them to Anaconda.org
         # NOTE: this job is similar to the one in unit-tests.yml except for the fact
@@ -137,7 +138,7 @@ jobs:
       # removes unnecessary files from the release
       - name: Download sdist (not macOS)
         #if: ${{ matrix.buildplat[1] != 'macosx_*' }}
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: sdist
           path: ./dist
@@ -160,7 +161,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.1.4
+        uses: pypa/cibuildwheel@v3.2.1
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
@@ -195,7 +196,7 @@ jobs:
         shell: bash -el {0}
         run: for whl in $(ls wheelhouse); do wheel unpack wheelhouse/$whl -d /tmp; done
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         with:
           name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl
@@ -216,3 +217,43 @@ jobs:
           source ci/upload_wheels.sh
           set_upload_vars
           upload_wheels
+
+  publish:
+    if: >
+      github.repository == 'pandas-dev/pandas' &&
+      github.event_name == 'release' &&
+      startsWith(github.ref, 'refs/tags/v')
+
+    needs:
+      - build_sdist
+      - build_wheels
+
+    runs-on: ubuntu-24.04
+
+    environment:
+      name: pypi
+    permissions:
+      id-token: write         # OIDC for Trusted Publishing
+      contents: read
+
+    steps:
+      - name: Download all artefacts
+        uses: actions/download-artifact@v6
+        with:
+          path: dist          # everything lands in ./dist/**
+
+      # TODO: This step can be probably be achieved by actions/download-artifact@v6
+      # by specifying merge-multiple: true, and a glob pattern
+      - name: Collect files
+        run: |
+          mkdir -p upload
+          # skip any wheel that contains 'pyodide'
+          find dist -name '*pyodide*.whl' -prune -o \
+                    -name '*.whl'   -exec mv {} upload/ \;
+          find dist -name '*.tar.gz' -exec mv {} upload/ \;
+
+      - name: Publish to **PyPI** (Trusted Publishing)
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: upload
+          skip-existing: true

.github/workflows/wheels.yml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.11
+    rev: v0.14.3
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -46,7 +46,7 @@ repos:
     -   id: codespell
         types_or: [python, rst, markdown, cython, c]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.16.7
+    rev: v0.18.1
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -67,11 +67,11 @@ repos:
     -   id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
 -   repo: https://github.com/PyCQA/isort
-    rev: 6.0.1
+    rev: 7.0.0
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
+    rev: v3.21.0
     hooks:
     -   id: pyupgrade
         args: [--py311-plus]
@@ -87,19 +87,19 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v1.0.0
+    rev: v1.0.1
     hooks:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v21.1.0
+    rev: v21.1.2
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.9.0
+    rev: v1.9.1
     hooks:
     - id: meson-fmt
       args: ['--inplace']