Merge branch '17-read-ahead-buffering' into 'devel'

Resolve "Read-Ahead Buffering"

See merge request parallel-programming/bs-daos-tensorflow-io!17

Author: omar-hmarzouk

.github/workflows/benchmarks.yml

@@ -36,7 +36,7 @@ jobs:
           echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
           echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
           python -m pip install tensorflow-io-gcs-filesystem-nightly
-          python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
+          python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 google-cloud-bigtable==1.6.0 azure-storage-blob==12.7.1
           python -m pip freeze
           python -c 'import tensorflow as tf; print(tf.version.VERSION)'
           python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
@@ -75,7 +75,7 @@ jobs:
           echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
           echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
           python -m pip install tensorflow-io-gcs-filesystem-nightly
-          python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
+          python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 google-cloud-bigtable==1.6.0 azure-storage-blob==12.7.1
           python -m pip freeze
           python -c 'import tensorflow as tf; print(tf.version.VERSION)'
           python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'

.github/workflows/build.bazel.sh

@@ -24,11 +24,14 @@ $PYTHON --version
 
 export TENSORFLOW_INSTALL="$($PYTHON setup.py --install-require)"
 
-export BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]')
-export BAZEL_VERSION=$(cat .bazelversion)
 export PYTHON_BIN_PATH=`which $PYTHON`
-curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
-bash -e bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
+
+if [[ $(uname) == "Linux" ]]; then
+  curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64
+  mv bazelisk-linux-amd64 /usr/local/bin/bazel
+  chmod +x /usr/local/bin/bazel
+fi
+
 bazel version
 
 $PYTHON -m pip --version

.github/workflows/build.yml

@@ -15,17 +15,13 @@ env:
 jobs:
   lint:
     name: Lint
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
       - name: Run Lint Script for Bazel/Pyupgrade/Black/Clang
         run: |
           set -x -e
           git log --pretty -1
-          BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]')
-          BAZEL_VERSION=$(cat .bazelversion)
-          curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
-          sudo bash -e bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
           sudo python3 -m pip install -U numpy
           bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade black clang
       - name: Run Lint Script for Docs
@@ -47,7 +43,7 @@ jobs:
 
   macos:
     name: macOS
-    runs-on: macos-latest
+    runs-on: macOS-11
     steps:
       - uses: actions/checkout@v2
       - name: GCP
@@ -71,7 +67,7 @@ jobs:
 
   linux:
     name: Linux
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     container:
       image: ubuntu:20.04
       env:
@@ -100,7 +96,7 @@ jobs:
 
   macos-bazel:
     name: Bazel macOS
-    runs-on: macos-latest
+    runs-on: macOS-11
     steps:
       - uses: actions/checkout@v2
       - name: GCP
@@ -126,7 +122,7 @@ jobs:
   macos-wheel:
     name: Wheel ${{ matrix.python }} macOS
     needs: macos-bazel
-    runs-on: macos-latest
+    runs-on: macOS-11
     strategy:
       matrix:
         python: ['3.7', '3.8', '3.9', '3.10']
@@ -167,7 +163,7 @@ jobs:
   macos-test:
     name: Test ${{ matrix.python }} macOS
     needs: macos-wheel
-    runs-on: macos-latest
+    runs-on: macOS-11
     strategy:
       matrix:
         python: ['3.8', '3.9']
@@ -182,6 +178,7 @@ jobs:
           python-version: ${{ matrix.python }}
       - run: |
           set -x -e
+          python -m pip install pip==21.3.1
           python -m pip install -U wheel setuptools
           python --version
       - name: Setup ${{ matrix.python }} macOS
@@ -206,7 +203,7 @@ jobs:
 
   linux-bazel:
     name: Bazel Linux
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     container:
       image: gcr.io/tensorflow-testing/nosla-ubuntu16.04-manylinux2010
       env:
@@ -239,7 +236,7 @@ jobs:
   linux-wheel:
     name: Wheel ${{ matrix.python }} Linux
     needs: linux-bazel
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: ['3.7', '3.8', '3.9', '3.10']
@@ -308,7 +305,7 @@ jobs:
 
   windows-bazel:
     name: Bazel Windows
-    runs-on: windows-latest
+    runs-on: windows-2019
     steps:
       - uses: actions/checkout@v2
       - uses: egor-tensin/vs-shell@v2
@@ -323,26 +320,21 @@ jobs:
       - name: Bazel on Windows
         env:
           BAZEL_VC: "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/"
-        shell: cmd
+        shell: bash
         run: |
-          @echo on
-          set "BAZEL_OPTIMIZATION="
-          if "%EVENT_NAME%" == "push" (
-            if "%REPO_NAME%" == "tensorflow/io" (
-              set "BAZEL_OPTIMIZATION=%BAZEL_OPTIMIZATION% --remote_upload_local_results=true --google_credentials=service_account_creds.json"
-            )
-          )
-          set /P BAZEL_VERSION=< .bazelversion
-          curl -sSL -o bazel.exe https://github.com/bazelbuild/bazel/releases/download/%BAZEL_VERSION%/bazel-%BAZEL_VERSION%-windows-x86_64.exe
+          set -x -e
+          export BAZEL_OPTIMIZATION="--config=cache"
+          if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then
+            export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json"
+          fi
           bazel version
           python3 --version
           python3 -m pip install wheel setuptools
           python3 -m pip --version
           python3 setup.py --install-require | xargs python3 -m pip install
           python3 tools/build/configure.py
           cat .bazelrc
-          bazel build -s %BAZEL_OPTIMIZATION% //tensorflow_io:python/ops/libtensorflow_io.so //tensorflow_io:python/ops/libtensorflow_io_plugins.so  //tensorflow_io_gcs_filesystem/...
-          if %errorlevel% neq 0 exit /b %errorlevel%
+          bazel build -s ${BAZEL_OPTIMIZATION} //tensorflow_io:python/ops/libtensorflow_io.so //tensorflow_io:python/ops/libtensorflow_io_plugins.so  //tensorflow_io_gcs_filesystem/...
           mkdir -p build
           cp -r bazel-bin/tensorflow_io build
           cp -r bazel-bin/tensorflow_io_gcs_filesystem build
@@ -517,6 +509,46 @@ jobs:
           name: tensorflow-io-release
           path: wheelhouse
 
+  release-candidate:
+    name: Release Candidate
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: [lint, release]
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v1
+        with:
+          name: tensorflow-io-release
+          path: wheelhouse
+      - run: |
+          set -x -e
+          sha256sum wheelhouse/*.whl | sort -u > wheelhouse.sha256
+          git rev-parse --verify HEAD > wheelhouse.commit
+          grep 'version = ' tensorflow_io/python/ops/version_ops.py  | sed -E 's@^.*version = "@@g' | sed -E 's@".*$@@g' > wheelhouse.version
+          cat wheelhouse.sha256
+          cat wheelhouse.commit
+          cat wheelhouse.version
+          echo "::set-output name=version::$(cat wheelhouse.version)"
+        id: info
+      - run: |-
+          cat <<EOF > Dockerfile
+          FROM gcr.io/distroless/base-debian11
+          COPY wheelhouse /wheelhouse
+          COPY wheelhouse.sha256 /wheelhouse.sha256
+          COPY wheelhouse.commit /wheelhouse.commit
+          COPY wheelhouse.version /wheelhouse.version
+          EOF
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/login-action@v1
+        with:
+          username: tfsigio
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: tfsigio/candidate:${{ steps.info.outputs.version }}
+
   docker-release:
     name: Docker Release
     if: github.event_name == 'push' && github.ref == 'refs/heads/master'
@@ -542,7 +574,7 @@ jobs:
   build-number:
     name: Build Number
     if: github.event_name == 'push'
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     steps:
       - run: |
           set -e -x
@@ -557,7 +589,7 @@ jobs:
     name: Nightly ${{ matrix.python }} macOS
     if: github.event_name == 'push'
     needs: [build-number, macos-wheel]
-    runs-on: macos-latest
+    runs-on: macOS-11
     strategy:
       matrix:
         python: ['3.7', '3.8', '3.9', '3.10']

.github/workflows/release.note.yml

@@ -0,0 +1,28 @@
+name: Release Note
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  release-note:
+    name: README.md
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - run: git tag
+      - run: python3 tools/release/note_create.py
+      - run: git diff
+      - run: python3 tools/release/note_update.py
+      - run: git diff
+      - uses: peter-evans/create-pull-request@v3
+        with:
+          commit-message: Update RELEASE.md [bot]
+          branch: bot-RELEASE.md
+          delete-branch: true
+          title: 'Update RELEASE.md [bot]'
+          body: |
+            README.md: auto-updated by .github/workflows/release.note.yml

.github/workflows/release.yml

@@ -0,0 +1,77 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version (e.g., v0.24.0)"
+        required: true
+      commit:
+        description: "Commit (e.g., 92b44e1)"
+        required: true
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - run: |
+          set -x -e
+          COMMIT=$(git rev-parse --quiet --verify ${{ github.event.inputs.commit }})
+          if [[ "$(git tag -l ${{ github.event.inputs.version }})" == "${{ github.event.inputs.version }}" ]]; then
+            echo "${{ github.event.inputs.version }} already released"
+            exit 1
+          fi
+          VERSION=${{ github.event.inputs.version }}
+
+          docker pull tfsigio/candidate:${VERSION:1}
+          docker create -it --name storage tfsigio/candidate:${VERSION:1} bash
+          docker cp storage:/wheelhouse .
+          docker cp storage:/wheelhouse.sha256 .
+          docker cp storage:/wheelhouse.commit .
+          docker cp storage:/wheelhouse.version .
+
+          sha256sum wheelhouse/*.whl | sort -u | diff wheelhouse.sha256 -
+          mv wheelhouse dist
+
+          if [[ "${COMMIT}" != "$(cat wheelhouse.commit)" ]]; then
+            echo "${COMMIT} != $(cat wheelhouse.commit)"
+            exit 1
+          fi
+          if [[ "${VERSION}" != "v$(cat wheelhouse.version)" ]]; then
+            echo "${VERSION} != v$(cat wheelhouse.version)"
+            exit 1
+          fi
+
+          python3 tools/release/note_take.py ${VERSION}
+
+          echo "::set-output name=tag::${VERSION}"
+          echo "::set-output name=name::TensorFlow I/O ${VERSION:1}"
+          echo "::set-output name=commit::${COMMIT}"
+        id: info
+      - run: |
+          set -x -e
+          echo ${{ steps.info.outputs.tag }}
+          echo ${{ steps.info.outputs.name }}
+          echo ${{ steps.info.outputs.commit }}
+          cat CURRENT.md
+      - uses: softprops/action-gh-release@v1
+        with:
+          body_path: CURRENT.md
+          name: ${{ steps.info.outputs.name }}
+          tag_name: ${{ steps.info.outputs.tag }}
+          target_commitish: ${{ steps.info.outputs.commit }}
+          draft: true
+      - uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.TEST_PYPI_TOKEN }}
+          repository_url: https://test.pypi.org/legacy/
+      #- uses: pypa/gh-action-pypi-publish@master
+      #  with:
+      #    user: __token__
+      #    password: ${{ secrets.PYPI_TOKEN }}
+          

.github/workflows/reviewdog.yml

@@ -0,0 +1,44 @@
+name: Reviewdog
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  bazel:
+    name: Bazel Buildifier
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          sudo python3 -m pip install -U numpy
+          bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- bazel
+      - uses: reviewdog/action-suggester@v1
+  black:
+    name: Python Black
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          sudo python3 -m pip install -U numpy
+          bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- black
+      - uses: reviewdog/action-suggester@v1
+  clang:
+    name: Clang Format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          sudo python3 -m pip install -U numpy
+          bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- clang
+      - uses: reviewdog/action-suggester@v1
+  pyupgrade:
+    name: Python Pyupgrade
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          sudo python3 -m pip install -U numpy
+          bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:lint -- pyupgrade
+      - uses: reviewdog/action-suggester@v1

README.md

@@ -126,6 +126,7 @@ of releases [here](https://github.com/tensorflow/io/releases).
 
 | TensorFlow I/O Version | TensorFlow Compatibility | Release Date |
 | --- | --- | --- |
+| 0.24.0 | 2.8.x | Feb 04, 2022 |
 | 0.23.1 | 2.7.x | Dec 15, 2021 |
 | 0.23.0 | 2.7.x | Dec 14, 2021 |
 | 0.22.0 | 2.7.x | Nov 10, 2021 |