Skip to content

Commit 49932f3

Browse files
author
Your Name
committed
Merge remote-tracking branch 'upstream/master'
2 parents 0036acc + 75545f5 commit 49932f3

File tree

307 files changed

+26942
-8495
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

307 files changed

+26942
-8495
lines changed

‎.bazelversion‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.1.0

‎.github/workflows/api.yml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: API Compatibility
2+
3+
on:
4+
schedule:
5+
- cron: "0 12 * * *"
6+
7+
env:
8+
BAZEL_OPTIMIZATION: --copt=-msse4.2 --copt=-mavx --compilation_mode=opt
9+
10+
jobs:
11+
12+
macos:
13+
name: macOS ${{ matrix.python }} + ${{ matrix.version }}
14+
runs-on: macos-latest
15+
strategy:
16+
matrix:
17+
python: ['3.8']
18+
version: ['tensorflow==2.4.0rc4:tensorflow-io-nightly', 'tf-nightly:tensorflow-io-nightly']
19+
steps:
20+
- uses: actions/checkout@v2
21+
- uses: docker-practice/actions-setup-docker@v1
22+
- uses: actions/setup-python@v1
23+
with:
24+
python-version: ${{ matrix.python }}
25+
- run: |
26+
set -x -e
27+
python -m pip install -U wheel setuptools
28+
python --version
29+
- name: Setup macOS
30+
run: |
31+
set -x -e
32+
docker version
33+
bash -x -e tests/test_azure/start_azure.sh
34+
bash -x -e tests/test_aws/aws_test.sh
35+
- name: Test macOS
36+
run: |
37+
set -x -e
38+
python --version
39+
df -h
40+
rm -rf tensorflow_io
41+
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
42+
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
43+
python -m pip install pytest-benchmark boto3
44+
python -m pip freeze
45+
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
46+
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
47+
python -m pytest -s -v tests/test_http_eager.py
48+
python -m pytest -s -v tests/test_s3_eager.py
49+
python -m pytest -s -v tests/test_azure.py
50+
51+
linux:
52+
name: Linux ${{ matrix.python }} + ${{ matrix.version }}
53+
runs-on: ubuntu-20.04
54+
strategy:
55+
matrix:
56+
python: ['3.8']
57+
version: ['tensorflow==2.4.0rc4:tensorflow-io-nightly', 'tf-nightly:tensorflow-io-nightly']
58+
steps:
59+
- uses: actions/checkout@v2
60+
- uses: actions/setup-python@v1
61+
with:
62+
python-version: ${{ matrix.python }}
63+
- name: Setup Linux
64+
run: |
65+
set -x -e
66+
bash -x -e .github/workflows/build.space.sh
67+
bash -x -e tests/test_aws/aws_test.sh
68+
bash -x -e tests/test_azure/start_azure.sh
69+
bash -x -e tests/test_gcloud/test_gcs.sh gcs-emulator
70+
- name: Test Linux
71+
run: |
72+
set -x -e
73+
python --version
74+
df -h
75+
rm -rf tensorflow_io
76+
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
77+
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
78+
python -m pip install pytest-benchmark boto3
79+
python -m pip freeze
80+
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
81+
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
82+
python -m pytest -s -v tests/test_http_eager.py
83+
python -m pytest -s -v tests/test_s3_eager.py
84+
python -m pytest -s -v tests/test_azure.py
85+
86+
windows:
87+
name: Windows ${{ matrix.python }} + ${{ matrix.version }}
88+
runs-on: windows-latest
89+
strategy:
90+
matrix:
91+
python: ['3.8']
92+
version: ['tensorflow==2.4.0rc4:tensorflow-io-nightly', 'tf-nightly:tensorflow-io-nightly']
93+
steps:
94+
- uses: actions/checkout@v2
95+
- uses: actions/setup-python@v1
96+
with:
97+
python-version: ${{ matrix.python }}
98+
- uses: actions/setup-node@v1
99+
with:
100+
node-version: '8.x'
101+
- name: Setup Windows
102+
shell: cmd
103+
run: |
104+
@echo on
105+
bash -x -e tests/test_azure/start_azure.sh
106+
- name: Test Windows
107+
shell: cmd
108+
run: |
109+
@echo on
110+
python --version
111+
rm -rf tensorflow_io
112+
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
113+
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
114+
python -m pip install pytest-benchmark
115+
python -m pip freeze
116+
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
117+
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
118+
python -m pytest -s -v tests/test_http_eager.py -k remote

‎.github/workflows/build.instruction.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,23 @@
22

33
sudo = False
44
if sys.argv[1].startswith("--sudo="):
5-
sudo = (sys.argv[1][7:].lower() == "true")
5+
sudo = sys.argv[1][7:].lower() == "true"
66

77
source = sys.argv[len(sys.argv) - 2]
88
section = sys.argv[len(sys.argv) - 1]
9-
with open (source, "r") as f:
9+
with open(source, "r") as f:
1010
lines = [line.rstrip() for line in list(f)]
1111

1212
# Remove lines before section title
13-
lines = lines[lines.index(section):]
13+
lines = lines[lines.index(section) :]
1414

1515
# Remove lines outside (including) "```sh" and "```"
16-
lines = lines[lines.index("```sh")+1:lines.index("```")]
16+
lines = lines[lines.index("```sh") + 1 : lines.index("```")]
1717

1818
# Remove sudo
1919
if not sudo:
20-
lines = [(line[len("sudo "):] if line.startswith("sudo ") else line) for line in lines]
20+
lines = [
21+
(line[len("sudo ") :] if line.startswith("sudo ") else line) for line in lines
22+
]
2123

2224
print("\n".join(lines))

‎.github/workflows/build.space.sh‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Free disk space on Linux
2-
sudo swapoff /swapfile
3-
sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc
4-
sudo apt-get remove php* ruby-* subversion mongodb-org -yq >/dev/null 2>&1
5-
sudo apt-get autoremove -y >/dev/null 2>&1
6-
sudo apt-get autoclean -y >/dev/null 2>&1
7-
sudo rm -rf /usr/local/lib/android >/dev/null 2>&1
8-
docker rmi $(docker image ls -aq) >/dev/null 2>&1
2+
#sudo swapoff /swapfile
3+
#sudo rm -rf /swapfile
4+
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
5+
sudo apt-get remove php* ruby-* subversion mongodb-org -yq >/dev/null 2>&1
6+
sudo apt-get autoremove -y >/dev/null 2>&1
7+
sudo apt-get autoclean -y >/dev/null 2>&1
8+
sudo rm -rf /usr/local/lib/android >/dev/null 2>&1
9+
docker rmi $(docker image ls -aq) >/dev/null 2>&1

‎.github/workflows/build.wheel.sh‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,11 @@ run_test() {
66
entry=$1
77
CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))')
88
(cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl)
9-
$entry -m pip install -q pytest pytest-benchmark boto3 python-dateutil==2.8.0 google-cloud-pubsub==0.39.1 pyarrow==0.16.0 pandas==0.24.2 scikit-learn==0.20.4 google-cloud-bigquery-storage==0.7.0 google-cloud-bigquery==1.22.0 fastavro avro-python3
10-
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_eager.py" -o -iname "test_gcs_config_ops.py" \) \)))
9+
$entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==0.16.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0
10+
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_eager.py" \) \)))
1111
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*_eager.py" ! \( -iname "test_bigquery_eager.py" \) \)))
1212
# GRPC and test_bigquery_eager tests have to be executed separately because of https://github.com/grpc/grpc/issues/20034
1313
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_bigquery_eager.py" \)))
14-
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_gcs_config_ops.py" \)))
1514
}
1615

1716
PYTHON_VERSION=python
@@ -20,11 +19,6 @@ if [[ "$#" -gt 0 ]]; then
2019
shift
2120
fi
2221

23-
if [[ "$PYTHON_VERSION" == "python3.8" ]]; then
24-
echo "TODO: Python 3.8 test is not supported yet as dependency might not be available"
25-
exit 0
26-
fi
27-
2822
if [[ $(uname) == "Linux" ]]; then
2923
apt-get -y -qq update
3024
if [[ "${PYTHON_VERSION}" == "python3.7" ]]; then
@@ -35,5 +29,20 @@ if [[ $(uname) == "Linux" ]]; then
3529
apt-get -y -qq install $PYTHON_VERSION ffmpeg dnsutils libmp3lame0
3630
curl -sSOL https://bootstrap.pypa.io/get-pip.py
3731
$PYTHON_VERSION get-pip.py -q
32+
33+
# Install Java
34+
apt-get -y -qq install openjdk-8-jdk
35+
update-alternatives --config java
36+
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
37+
38+
# Install Hadoop
39+
curl -OL https://archive.apache.org/dist/hadoop/common/hadoop-2.7.0/hadoop-2.7.0.tar.gz
40+
tar -xzf hadoop-2.7.0.tar.gz -C /usr/local
41+
export HADOOP_HOME=/usr/local/hadoop-2.7.0
42+
43+
# Update environmental variable
44+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${JAVA_HOME}/jre/lib/amd64/server:${HADOOP_HOME}/lib/native
45+
export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
46+
export
3847
fi
3948
run_test $PYTHON_VERSION

0 commit comments

Comments
 (0)