Fix c4_test.

PiperOrigin-RevId: 796108166

Author: fineguy

.github/workflows/pytest-template.yml

@@ -52,10 +52,7 @@ jobs:
 
     # Run tests
     # Ignores:
-    # * Nsynth is run in isolation due to dependency conflict (crepe).
-    # * Lsun tests is disabled because the tensorflow_io used in open-source
-    #   is linked to static libraries compiled again specific TF version, which
-    #   makes test fails with linking error (libtensorflow_io_golang.so).
+    # * c4, lsun, nsynth are run separately to avoid dependency conflicts.
     # * imagenet2012_corrupted requires imagemagick binary.
     # * import_without_tf_test.py, because the test relies on TensorFlow not being imported.
     # * github_api is run separately to not overuse API quota.
@@ -65,8 +62,9 @@ jobs:
     - name: Run core tests
       run: |
         pytest --durations=100 -vv -n auto --shard-id=$((${{ matrix.shard-id }} - 1)) --num-shards=${{ matrix.num-shards }} \
-          --ignore="tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py" \
+          --ignore-glob="tensorflow_datasets/text/c4*_test.py" \
           --ignore="tensorflow_datasets/image/lsun_test.py" \
+          --ignore="tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py" \
           --ignore="tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder_test.py" \
           --ignore="tensorflow_datasets/scripts/documentation/build_api_docs_test.py" \
           --ignore="tensorflow_datasets/import_without_tf_test.py" \
@@ -86,3 +84,27 @@ jobs:
         retry_on: timeout
         command: |
           pytest -vv -o faulthandler_timeout=10 tensorflow_datasets/translate/wmt19_test.py
+
+    # Run `c4` dataset tests separately.
+    - name: Run c4 dataset tests
+      if: ${{ matrix.shard-id == 1 }}
+      run: |
+        pip install .[c4]
+        pytest -vv -n auto \
+          tensorflow_datasets/text/c4*_test.py
+
+    # Run `lsun` dataset tests separately.
+    - name: Run lsun dataset tests
+      if: ${{ matrix.shard-id == 2 }}
+      run: |
+        pip install .[lsun]
+        pytest -vv -n auto \
+          tensorflow_datasets/image/lsun_test.py
+
+    # Run `nsynth` dataset tests separately.
+    - name: Run nsynth dataset tests
+      if: ${{ matrix.shard-id == 3 }}
+      run: |
+        pip install .[nsynth]
+        pytest -vv -n auto \
+          tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py

setup.py

@@ -81,7 +81,6 @@
     'pytest-shard',
     'pytest-xdist',
     # Lazy-deps required by core
-    # TODO(b/418761065): Update to 2.65.0 once the bug is fixed.
     'apache-beam<2.65.0',
     'conllu',
     'mlcroissant>=1.0.9',
@@ -150,6 +149,9 @@
         # nltk==3.8.2 is broken: https://github.com/nltk/nltk/issues/3293
         'nltk==3.8.1',
         'tldextract',
+        # tensorflow==2.20.0 is not compatible with gcld3 because of protobuf
+        # version conflict.
+        'tensorflow<2.20.0',
     ],
     'c4_wsrs': ['apache-beam<2.65.0'],
     'cats_vs_dogs': ['matplotlib'],
@@ -167,11 +169,17 @@
         'scipy',
     ],
     'librispeech': ['pydub'],  # and ffmpeg installed
-    'lsun': ['tensorflow-io[tensorflow]'],
-    # sklearn version required to avoid conflict with librosa from
-    # https://github.com/scikit-learn/scikit-learn/issues/14485
-    # See https://github.com/librosa/librosa/issues/1160
-    'nsynth': ['crepe>=0.0.11', 'librosa', 'scikit-learn==0.20.3'],
+    'lsun': [
+        # tensorflow-io is compiled against specific versions of TF.
+        'tensorflow-io[tensorflow]',
+    ],
+    'nsynth': [
+        'crepe',
+        'librosa',
+        # tensorflow==2.20.0 is not compatible with librosa because of protobuf
+        # version conflict.
+        'tensorflow<2.20.0',
+    ],
     'ogbg_molpcba': ['pandas', 'networkx'],
     'pet_finder': ['pandas'],
     'qm9': ['pandas'],
@@ -196,7 +204,7 @@
 
 # Those datasets have dependencies which conflict with the rest of TFDS, so
 # running them in an isolated environments.
-ISOLATED_DATASETS = ('nsynth', 'lsun')
+ISOLATED_DATASETS = ('c4', 'lsun', 'nsynth')
 
 # Extra dataset deps are required for the tests
 all_dataset_dependencies = list(
@@ -238,18 +246,21 @@
     license='Apache 2.0',
     packages=setuptools.find_packages(),
     package_data={
-        'tensorflow_datasets': DATASET_FILES + [
-            # Bundle `datasets/` folder in PyPI releases
-            'datasets/*/*',
-            'core/utils/colormap.csv',
-            'scripts/documentation/templates/*',
-            'url_checksums/*',
-            'checksums.tsv',
-            'community-datasets.toml',
-            'dataset_collections/*/*.md',
-            'dataset_collections/*/*.bib',
-            'core/valid_tags.txt',
-        ],
+        'tensorflow_datasets': (
+            DATASET_FILES
+            + [
+                # Bundle `datasets/` folder in PyPI releases
+                'datasets/*/*',
+                'core/utils/colormap.csv',
+                'scripts/documentation/templates/*',
+                'url_checksums/*',
+                'checksums.tsv',
+                'community-datasets.toml',
+                'dataset_collections/*/*.md',
+                'dataset_collections/*/*.bib',
+                'core/valid_tags.txt',
+            ]
+        ),
     },
     exclude_package_data={
         'tensorflow_datasets': [

tensorflow_datasets/core/lazy_imports_lib_test.py

@@ -25,17 +25,18 @@ class LazyImportsTest(testing.TestCase, parameterized.TestCase):
   # The following deps are not in the test list because the datasets that
   # require them need to have their tests run in isolation:
   # * crepe (NSynth)
+  # * gcld3 (c4)
+  # * langdetect (c4)
   # * librosa (NSynth)
+  # * nltk (c4)
+  # * tldextract (c4)
   @parameterized.parameters(
       "bs4",
       "cv2",
-      "gcld3",
       "gcsfs_store",
-      "langdetect",
       "lxml",
       "matplotlib",
       "mwparserfromhell",
-      "nltk",
       "os",
       "pandas",
       "pretty_midi",
@@ -44,7 +45,6 @@ class LazyImportsTest(testing.TestCase, parameterized.TestCase):
       "scipy",
       "skimage",
       "tifffile",
-      "tldextract",
       "zarr",
   )
   def test_import(self, module_name):