diff --git a/WORKSPACE b/WORKSPACE index b5b90f39b..7d77c5235 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -56,10 +56,10 @@ http_archive( http_archive( name = "org_tensorflow", - strip_prefix = "tensorflow-40998f44c0c500ce0f6e3b1658dfbc54f838a82a", - sha256 = "5a5bc4599964c71277dcac0d687435291e5810d2ac2f6283cc96736febf73aaf", + strip_prefix = "tensorflow-72fbba3d20f4616d7312b5e2b7f79daf6e82f2fa", + urls = [ - "https://github.com/tensorflow/tensorflow/archive/40998f44c0c500ce0f6e3b1658dfbc54f838a82a.zip" + "https://github.com/tensorflow/tensorflow/archive/72fbba3d20f4616d7312b5e2b7f79daf6e82f2fa.zip" ], ) @@ -95,7 +95,7 @@ load("@org_tensorflow//third_party/py:python_init_rules.bzl", "python_init_rules python_init_rules() -load("//tensorflow_text:tftext.bzl", "py_deps_profile") +load("//tensorflow_text:py_deps_profile.bzl", "py_deps_profile") py_deps_profile( name = "release_or_nightly", @@ -158,7 +158,22 @@ load( python_wheel_version_suffix_repository(name = "tf_wheel_version_suffix") load( - "@local_xla//third_party/gpus/cuda/hermetic:cuda_json_init_repository.bzl", + "@rules_ml_toolchain//cc/deps:cc_toolchain_deps.bzl", + "cc_toolchain_deps", +) + +cc_toolchain_deps() + +register_toolchains("@rules_ml_toolchain//cc:linux_x86_64_linux_x86_64") + +register_toolchains("@rules_ml_toolchain//cc:linux_x86_64_linux_x86_64_cuda") + +register_toolchains("@rules_ml_toolchain//cc:linux_aarch64_linux_aarch64") + +register_toolchains("@rules_ml_toolchain//cc:linux_aarch64_linux_aarch64_cuda") + +load( + "@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_json_init_repository.bzl", "cuda_json_init_repository", ) @@ -170,7 +185,7 @@ load( "CUDNN_REDISTRIBUTIONS", ) load( - "@local_xla//third_party/gpus/cuda/hermetic:cuda_redist_init_repositories.bzl", + "@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_redist_init_repositories.bzl", "cuda_redist_init_repositories", "cudnn_redist_init_repository", ) @@ -184,21 +199,21 @@ cudnn_redist_init_repository( ) load( - "@local_xla//third_party/gpus/cuda/hermetic:cuda_configure.bzl", + "@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_configure.bzl", "cuda_configure", ) cuda_configure(name = "local_config_cuda") load( - "@local_xla//third_party/nccl/hermetic:nccl_redist_init_repository.bzl", + "@rules_ml_toolchain//third_party/nccl/hermetic:nccl_redist_init_repository.bzl", "nccl_redist_init_repository", ) nccl_redist_init_repository() load( - "@local_xla//third_party/nccl/hermetic:nccl_configure.bzl", + "@rules_ml_toolchain//third_party/nccl/hermetic:nccl_configure.bzl", "nccl_configure", ) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index a7f4e96a6..98d776ad6 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -38,11 +38,7 @@ if [[ $(pip show tensorflow) == *tensorflow* ]] || echo 'Using installed tensorflow.' else echo 'Installing tensorflow.' - if [[ "$IS_NIGHTLY" == "nightly" ]]; then - pip install tf-nightly - else - pip install tensorflow==2.18.0 - fi + pip install tf-nightly fi # Copy the current bazelversion of TF. @@ -54,6 +50,17 @@ curl https://raw.githubusercontent.com/tensorflow/tensorflow/master/.bazelrc -o sed -i -e 's/build --noincompatible_remove_legacy_whole_archive//' .bazelrc write_to_bazelrc "build:manylinux2014 --config=release_cpu_linux" +DIR="org_tensorflow" +# Check if the directory exists +if [ ! -d "$DIR" ]; then + git clone https://github.com/tensorflow/tensorflow.git org_tensorflow + cd org_tensorflow + git checkout 735467e89ccfd7ace190363412bb5698164628b5 + cd .. +else + echo "Directory $DIR already exists." +fi +write_to_bazelrc "common:linux --override_repository=org_tensorflow=./org_tensorflow" if (which python3) | grep -q "python3"; then installed_python="python3" @@ -76,8 +83,16 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre" fi +# JUST FOR TESTING +wget https://storage.googleapis.com/ml-sysroot-testing/tensorflow_metadata-1.18.0.dev0-py3-none-any.whl +mv tensorflow_metadata-1.18.0.dev0-py3-none-any.whl /github/tensorflow_metadata-1.18.0.dev0-py3-none-any.whl +echo "tensorflow-metadata @ file:///github/tensorflow_metadata-1.18.0.dev0-py3-none-any.whl" >> oss_scripts/pip_package/requirements.in + bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS +# JUST FOR TESTING +sed -i 's#file://tensorflow_metadata-1.18.0.dev0-py3-none-any.whl#file:///github/tensorflow_metadata-1.18.0.dev0-py3-none-any.whl#g' oss_scripts/pip_package/requirements_lock_3_9.txt + TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) SHARED_LIBRARY_NAME="libtensorflow_framework.so.2" if is_macos; then diff --git a/oss_scripts/pip_package/BUILD b/oss_scripts/pip_package/BUILD index ff9c51d42..359636abd 100644 --- a/oss_scripts/pip_package/BUILD +++ b/oss_scripts/pip_package/BUILD @@ -1,6 +1,7 @@ # Tools for building the TF.Text pip package. -load("@python//:defs.bzl", "compile_pip_requirements") + load("@python_version_repo//:py_version.bzl", "REQUIREMENTS") +load("@rules_python//python:pip.bzl", "compile_pip_requirements") package(default_visibility = ["//visibility:private"]) diff --git a/oss_scripts/pip_package/build_pip_package.sh b/oss_scripts/pip_package/build_pip_package.sh index 8e411ef17..ac4d94f54 100755 --- a/oss_scripts/pip_package/build_pip_package.sh +++ b/oss_scripts/pip_package/build_pip_package.sh @@ -98,11 +98,7 @@ main() { fi # Build pip package - if is_nightly; then - $installed_python setup.nightly.py bdist_wheel --universal $plat_name - else - $installed_python setup.py bdist_wheel --universal $plat_name - fi + $installed_python setup.nightly.py bdist_wheel --universal $plat_name cp dist/*.whl "${output_dir}" } diff --git a/oss_scripts/pip_package/requirements.in b/oss_scripts/pip_package/requirements.in index 9cc9f75d0..26dec2b4b 100644 --- a/oss_scripts/pip_package/requirements.in +++ b/oss_scripts/pip_package/requirements.in @@ -1,8 +1,7 @@ setuptools==70.0.0 dm-tree==0.1.8 # Limit for macos support. numpy -protobuf==4.25.3 # b/397977335 - Fix crash on python 3.9, 3.10. tensorflow tf-keras tensorflow-datasets -tensorflow-metadata +#tensorflow-metadata diff --git a/tensorflow_text/BUILD b/tensorflow_text/BUILD index 8078fd391..adda2d7ce 100644 --- a/tensorflow_text/BUILD +++ b/tensorflow_text/BUILD @@ -1,13 +1,15 @@ -# Placeholder: load py_library -# Placeholder: load py_test +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@org_tensorflow//tensorflow/lite:build_def.bzl", "tflite_cc_shared_object") -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("//tensorflow_text:tftext.bzl", "extra_py_deps", "py_tf_text_library") -# [internal] load build_test.bzl +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python:py_test.bzl", "py_test") +load(":tftext.bzl", "extra_py_deps", "py_tf_text_library") # Visibility rules package( - default_visibility = ["//visibility:public"], + default_visibility = [ + "//visibility:public", + ], ) licenses(["notice"]) @@ -77,6 +79,10 @@ py_library( "python/ops/__init__.py", "tools/__init__.py", ], + visibility = [ + "//tensorflow_text:__pkg__", + "//tensorflow_text:__subpackages__", + ], deps = [ ":bert_tokenizer", ":boise_offset_converter", @@ -122,13 +128,19 @@ py_library( ":whitespace_tokenizer_v2", ":wordpiece_tokenizer", ":wordshape_ops", - # python/util:all_util tensorflow dep, - "//tensorflow_text/core/pybinds:tflite_registrar", "//tensorflow_text/tools/wordpiece_vocab", + "@org_tensorflow//tensorflow/python/util:all_util", + "@org_tensorflow_text//tensorflow_text/core/pybinds:tflite_registrar_library", ] + extra_py_deps(), ) -# build_test +build_test( + name = "tf-text-build-test", + targets = [ + ":ops", + ":ops_lib", + ], +) # This is required for other external users to build tests using these models. exports_files( @@ -149,10 +161,10 @@ py_library( ":regex_split_ops", ":tokenization", ":wordpiece_tokenizer", - # python/eager:monitoring tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/eager:monitoring", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", ] + extra_py_deps(), ) @@ -164,17 +176,17 @@ py_test( deps = [ ":bert_tokenizer", "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -182,15 +194,15 @@ py_tf_text_library( name = "boise_offset_converter", srcs = ["python/ops/boise_offset_converter.py"], cc_op_defs = [ - "//tensorflow_text/core/ops:boise_offset_converter_op.cc", + "@org_tensorflow_text//tensorflow_text/core/ops:boise_offset_converter_op.cc", ], cc_op_kernels = [ - "//tensorflow_text/core/kernels:boise_offset_converter_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:boise_offset_converter_kernel", ], deps = [ - # python/framework:ops tensorflow dep, - # python/ops/ragged:ragged_conversion_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_conversion_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -200,8 +212,8 @@ py_test( srcs = ["python/ops/boise_offset_converter_test.py"], deps = [ ":tensorflow_text", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -209,14 +221,14 @@ py_tf_text_library( name = "byte_splitter", srcs = ["python/ops/byte_splitter.py"], cc_op_defs = [ - "//tensorflow_text/core/ops:byte_splitter_op.cc", + "@org_tensorflow_text//tensorflow_text/core/ops:byte_splitter_op.cc", ], cc_op_kernels = [ - "//tensorflow_text/core/kernels:byte_splitter_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:byte_splitter_kernel", ], deps = [ - # python/framework:ops tensorflow dep, - # python/ops/ragged tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops/ragged", ], ) @@ -226,23 +238,23 @@ py_test( srcs = ["python/ops/byte_splitter_test.py"], deps = [ ":tensorflow_text", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/keras/api", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "constrained_sequence_op", srcs = [], - cc_op_defs = ["//tensorflow_text/core/ops:constrained_sequence_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:constrained_sequence_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:constrained_sequence_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:constrained_sequence_kernel", ], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops/ragged tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops/ragged", ], ) @@ -250,13 +262,13 @@ py_library( name = "create_feature_bitmask_op", srcs = ["python/ops/create_feature_bitmask_op.py"], deps = [ - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:check_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:check_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", ] + extra_py_deps(), ) @@ -266,32 +278,32 @@ py_test( srcs = ["python/ops/create_feature_bitmask_op_test.py"], deps = [ ":create_feature_bitmask_op", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "fast_bert_normalizer", srcs = ["python/ops/fast_bert_normalizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:fast_bert_normalizer_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:fast_bert_normalizer_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:fast_bert_normalizer_tf_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:fast_bert_normalizer_tf_kernel", ], deps = [ - # python/eager:monitoring tensorflow dep, - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/module tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - "//tensorflow_text/core/pybinds:pywrap_fast_bert_normalizer_model_builder", + "@org_tensorflow//tensorflow/python/eager:monitoring", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/module", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow_text//tensorflow_text/core/pybinds:pywrap_fast_bert_normalizer_model_builder", ], ) @@ -301,19 +313,19 @@ py_test( srcs = ["python/ops/fast_bert_normalizer_test.py"], deps = [ ":tensorflow_text", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/compat tensorflow dep, - # python/data/kernel_tests:test_base tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, "//tensorflow_text:normalize_ops", - "//tensorflow_text/core/pybinds:pywrap_fast_bert_normalizer_model_builder", + "@absl_py//absl/testing:parameterized", + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/keras/api", + "@org_tensorflow//tensorflow/python/compat", + "@org_tensorflow//tensorflow/python/data/kernel_tests:test_base", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", + "@org_tensorflow_text//tensorflow_text/core/pybinds:pywrap_fast_bert_normalizer_model_builder", ], ) @@ -325,10 +337,10 @@ py_library( ":fast_bert_normalizer", ":fast_wordpiece_tokenizer", ":tokenization", - # python/eager:monitoring tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/ops:array_ops tensorflow dep, "//tensorflow_text:normalize_ops", + "@org_tensorflow//tensorflow/python/eager:monitoring", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/ops:array_ops", ], ) @@ -339,34 +351,34 @@ py_test( srcs = ["python/ops/fast_bert_tokenizer_test.py"], deps = [ ":tensorflow_text", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "fast_sentencepiece_tokenizer", srcs = ["python/ops/fast_sentencepiece_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:fast_sentencepiece_ops.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:fast_sentencepiece_ops.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels/sentencepiece:sentencepiece_tokenizer_kernel", - "//tensorflow_text/core/kernels/sentencepiece:sentencepiece_detokenizer_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels/sentencepiece:sentencepiece_tokenizer_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels/sentencepiece:sentencepiece_detokenizer_kernel", ], deps = [ - "//tensorflow_text/core/pybinds:pywrap_model_converter", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow_text//tensorflow_text/core/pybinds:pywrap_model_converter", ], ) @@ -385,29 +397,29 @@ py_test( "@absl_py//absl/flags", "@absl_py//absl/logging", "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/keras/api", ], ) py_tf_text_library( name = "fast_wordpiece_tokenizer", srcs = ["python/ops/fast_wordpiece_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:fast_wordpiece_tokenizer_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:fast_wordpiece_tokenizer_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:fast_wordpiece_tokenizer_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:fast_wordpiece_tokenizer_kernel", ], deps = [ ":tokenization", - # python/eager:monitoring tensorflow dep, - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - "//tensorflow_text/core/pybinds:pywrap_fast_wordpiece_tokenizer_model_builder", + "@org_tensorflow//tensorflow/python/eager:monitoring", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow_text//tensorflow_text/core/pybinds:pywrap_fast_wordpiece_tokenizer_model_builder", ], ) @@ -421,37 +433,37 @@ py_test( ], deps = [ ":tensorflow_text", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/compat tensorflow dep, - # python/data/kernel_tests:test_base tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/compat", + "@org_tensorflow//tensorflow/python/data/kernel_tests:test_base", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "phrase_tokenizer", srcs = ["python/ops/phrase_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:phrase_tokenizer_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:phrase_tokenizer_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:phrase_tokenizer_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:phrase_tokenizer_kernel", ], deps = [ ":tokenization", - # python/eager:monitoring tensorflow dep, - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - "//tensorflow_text/core/pybinds:pywrap_phrase_tokenizer_model_builder", + "@org_tensorflow//tensorflow/python/eager:monitoring", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow_text//tensorflow_text/core/pybinds:pywrap_phrase_tokenizer_model_builder", ], ) @@ -465,15 +477,15 @@ py_test( ], deps = [ ":tensorflow_text", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - # python/compat tensorflow dep, - # python/data/kernel_tests:test_base tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "//third_party/py/tensorflow", + "@org_tensorflow//tensorflow/python/compat", + "@org_tensorflow//tensorflow/python/data/kernel_tests:test_base", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -484,10 +496,10 @@ py_tf_text_library( ":constrained_sequence_op", ], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -497,10 +509,10 @@ py_test( srcs = ["python/ops/greedy_constrained_sequence_op_test.py"], deps = [ ":greedy_constrained_sequence_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", "@pypi//numpy:pkg", - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, ], ) @@ -509,10 +521,10 @@ py_library( srcs = ["python/ops/hub_module_splitter.py"], deps = [ ":splitter", - # python/eager:monitoring tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/saved_model:load tensorflow dep, + "@org_tensorflow//tensorflow/python/eager:monitoring", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/saved_model:load", ], ) @@ -525,12 +537,12 @@ py_test( ], deps = [ ":ops", - # python/framework:ops tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:variables tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:variables", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -552,12 +564,12 @@ py_test( ], deps = [ ":ops", - # python/framework:ops tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:variables tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:variables", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -565,26 +577,26 @@ py_library( name = "item_selector_ops", srcs = ["python/ops/item_selector_ops.py"], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:control_flow_ops tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:map_fn tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:random_ops tensorflow dep, - # python/ops:sort_ops tensorflow dep, - # python/ops/ragged:ragged_array_ops tensorflow dep, - # python/ops/ragged:ragged_batch_gather_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_functional_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_math_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/ops/ragged:ragged_tensor_shape tensorflow dep, - # python/ops/ragged:ragged_where_op tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:control_flow_ops", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:map_fn", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:random_ops", + "@org_tensorflow//tensorflow/python/ops:sort_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_batch_gather_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_functional_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor_shape", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_where_op", ] + extra_py_deps(), ) @@ -594,12 +606,12 @@ py_test( srcs = ["python/ops/item_selector_ops_test.py"], deps = [ ":item_selector_ops", - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -607,19 +619,19 @@ py_library( name = "masking_ops", srcs = ["python/ops/masking_ops.py"], deps = [ - # python/framework:dtypes tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:control_flow_assert tensorflow dep, - # python/ops:map_fn tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:random_ops tensorflow dep, - # python/ops:sort_ops tensorflow dep, - # python/ops/ragged:ragged_batch_gather_ops tensorflow dep, - # python/ops/ragged:ragged_functional_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_math_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/ops/ragged:ragged_where_op tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:control_flow_assert", + "@org_tensorflow//tensorflow/python/ops:map_fn", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:random_ops", + "@org_tensorflow//tensorflow/python/ops:sort_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_batch_gather_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_functional_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_where_op", ], ) @@ -630,29 +642,29 @@ py_test( deps = [ ":item_selector_ops", ":masking_ops", - "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "mst_ops", srcs = ["python/ops/mst_ops.py"], - cc_op_defs = ["//tensorflow_text/core/ops:mst_ops.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:mst_ops.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:mst_op_kernels", + "@org_tensorflow_text//tensorflow_text/core/kernels:mst_op_kernels", ], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:standard_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:standard_ops", ] + extra_py_deps(), ) @@ -661,30 +673,30 @@ py_test( srcs = ["python/ops/mst_ops_test.py"], deps = [ ":mst_ops", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "ngrams_op", srcs = ["python/ops/ngrams_op.py"], - cc_op_defs = ["//tensorflow_text/core/ops:ngrams_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:ngrams_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:ngrams_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:ngrams_kernel", ], deps = [ ":sliding_window_op", - # python/framework:errors tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_functional_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_functional_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -694,27 +706,27 @@ py_test( srcs = ["python/ops/ngrams_op_test.py"], deps = [ ":tensorflow_text", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:constant_op tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "normalize_ops", srcs = ["python/ops/normalize_ops.py"], - cc_op_defs = ["//tensorflow_text/core/ops:normalize_ops.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:normalize_ops.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:normalize_kernels", + "@org_tensorflow_text//tensorflow_text/core/kernels:normalize_kernels", ], deps = [ - # python/framework:dtypes tensorflow dep, - # python/ops/ragged:ragged_conversion_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_conversion_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -724,10 +736,10 @@ py_test( srcs = ["python/ops/normalize_ops_test.py"], deps = [ ":normalize_ops", - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -735,12 +747,12 @@ py_library( name = "pad_along_dimension_op", srcs = ["python/ops/pad_along_dimension_op.py"], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -750,15 +762,15 @@ py_test( srcs = ["python/ops/pad_along_dimension_op_test.py"], deps = [ ":pad_along_dimension_op", - "@absl_py//absl/testing:parameterized", - # python/eager:context tensorflow dep, - # python/framework:constant_op tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/eager:context", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -766,18 +778,18 @@ py_library( name = "pad_model_inputs_ops", srcs = ["python/ops/pad_model_inputs_ops.py"], deps = [ - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:control_flow_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:tensor_array_ops tensorflow dep, - # python/ops/ragged:ragged_array_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:control_flow_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:tensor_array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -786,11 +798,11 @@ py_test( srcs = ["python/ops/pad_model_inputs_ops_test.py"], deps = [ ":pad_model_inputs_ops", - "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -798,19 +810,19 @@ py_library( name = "pointer_ops", srcs = ["python/ops/pointer_ops.py"], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:check_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops/ragged:ragged_functional_ops tensorflow dep, - # python/ops/ragged:ragged_gather_ops tensorflow dep, - # python/ops/ragged:ragged_math_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/ops/ragged:ragged_where_op tensorflow dep, - # python/ops/ragged:segment_id_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:check_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_functional_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_gather_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_where_op", + "@org_tensorflow//tensorflow/python/ops/ragged:segment_id_ops", ] + extra_py_deps(), ) @@ -819,13 +831,13 @@ py_test( srcs = ["python/ops/gather_with_default_op_test.py"], deps = [ ":pointer_ops", - "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -834,10 +846,10 @@ py_test( srcs = ["python/ops/span_alignment_op_test.py"], deps = [ ":pointer_ops", - "@absl_py//absl/testing:parameterized", - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -846,40 +858,40 @@ py_test( srcs = ["python/ops/span_overlaps_op_test.py"], deps = [ ":pointer_ops", - "@absl_py//absl/testing:parameterized", - # python/eager:context tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/eager:context", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_test( name = "ragged_tensor_to_tensor_test", - srcs = ["python/ragged/ragged_tensor_to_tensor_test.py"], + srcs = ["//third_party/tensorflow_text/python/ragged:ragged_tensor_to_tensor_test.py"], deps = [ ":tensorflow_text", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "regex_split_ops", srcs = ["python/ops/regex_split_ops.py"], - cc_op_defs = ["//tensorflow_text/core/ops:regex_split_ops.cc"], - cc_op_kernels = ["//tensorflow_text/core/kernels:regex_split_kernels"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:regex_split_ops.cc"], + cc_op_kernels = ["@org_tensorflow_text//tensorflow_text/core/kernels:regex_split_kernels"], deps = [ ":splitter", - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -889,8 +901,8 @@ py_test( srcs = ["python/ops/regex_split_ops_test.py"], deps = [ ":regex_split_ops", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - # python/framework:test_lib tensorflow dep, + "//third_party/py/tensorflow", + "@org_tensorflow//tensorflow/python/framework:test_lib", ], ) @@ -898,10 +910,10 @@ py_library( name = "segment_combiner_ops", srcs = ["python/ops/segment_combiner_ops.py"], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", ] + extra_py_deps(), ) @@ -910,11 +922,11 @@ py_test( srcs = ["python/ops/segment_combiner_ops_test.py"], deps = [ ":segment_combiner_ops", - "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -922,20 +934,20 @@ py_tf_text_library( name = "sentence_breaking_ops", srcs = ["python/ops/sentence_breaking_ops.py"], cc_op_defs = [ - "//tensorflow_text/core/ops:sentence_breaking_ops.cc", + "@org_tensorflow_text//tensorflow_text/core/ops:sentence_breaking_ops.cc", ], cc_op_kernels = [ - "//tensorflow_text/core/kernels:sentence_breaking_kernels", + "@org_tensorflow_text//tensorflow_text/core/kernels:sentence_breaking_kernels", ], deps = [ ":regex_split_ops", - # tf:lib tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/util:deprecation tensorflow dep, + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/util:deprecation", ], ) @@ -945,36 +957,36 @@ py_test( srcs = ["python/ops/sentence_breaking_ops_test.py"], deps = [ ":sentence_breaking_ops", - "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "sentencepiece_tokenizer", srcs = ["python/ops/sentencepiece_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:sentencepiece_ops.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:sentencepiece_ops.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:sentencepiece_kernels", + "@org_tensorflow_text//tensorflow_text/core/kernels:sentencepiece_kernels", ], deps = [ ":tokenization", - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_conversion_ops tensorflow dep, - # python/ops/ragged:ragged_string_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_conversion_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -987,13 +999,13 @@ py_test( ], deps = [ ":sentencepiece_tokenizer", - "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_gather_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, - # python/saved_model tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_gather_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", + "@org_tensorflow//tensorflow/python/saved_model", ], ) @@ -1001,10 +1013,10 @@ py_library( name = "sliding_window_op", srcs = ["python/ops/sliding_window_op.py"], deps = [ - # python/framework:errors tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -1014,28 +1026,28 @@ py_test( srcs = ["python/ops/sliding_window_op_test.py"], deps = [ ":sliding_window_op", - "@absl_py//absl/testing:parameterized", - # python/framework:constant_op tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "split_merge_from_logits_tokenizer", srcs = ["python/ops/split_merge_from_logits_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:tokenizer_from_logits_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:tokenizer_from_logits_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:tokenizer_from_logits_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:tokenizer_from_logits_kernel", ], deps = [ ":tokenization", - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops/ragged tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops/ragged", ], ) @@ -1045,25 +1057,25 @@ py_test( srcs = ["python/ops/split_merge_from_logits_tokenizer_test.py"], deps = [ ":split_merge_from_logits_tokenizer", - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "split_merge_tokenizer", srcs = ["python/ops/split_merge_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:split_merge_tokenize_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:split_merge_tokenize_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:split_merge_tokenize_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:split_merge_tokenize_kernel", ], deps = [ ":tokenization", - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops/ragged tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops/ragged", ], ) @@ -1073,9 +1085,9 @@ py_test( srcs = ["python/ops/split_merge_tokenizer_test.py"], deps = [ ":split_merge_tokenizer", - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -1083,7 +1095,7 @@ py_library( name = "splitter", srcs = ["python/ops/splitter.py"], deps = [ - # python/module tensorflow dep, + "@org_tensorflow//tensorflow/python/module", ], ) @@ -1091,20 +1103,20 @@ py_tf_text_library( name = "state_based_sentence_breaker_op", srcs = ["python/ops/state_based_sentence_breaker_op.py"], cc_op_defs = [ - "//tensorflow_text/core/ops:sentence_fragmenter_v2_op.cc", + "@org_tensorflow_text//tensorflow_text/core/ops:sentence_fragmenter_v2_op.cc", ], cc_op_kernels = [ - "//tensorflow_text/core/kernels:sentence_fragmenter_v2_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:sentence_fragmenter_v2_kernel", ], deps = [ ":byte_splitter", ":sentence_breaking_ops", - # tf:lib tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -1115,19 +1127,19 @@ py_test( deps = [ ":state_based_sentence_breaker_op", ":tensorflow_text", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:errors tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:errors", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -1135,7 +1147,7 @@ py_library( name = "string_ops", srcs = ["python/ops/string_ops.py"], deps = [ - # python/ops:string_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/ops:string_ops", ] + extra_py_deps(), ) @@ -1145,22 +1157,22 @@ py_test( srcs = ["python/ops/coerce_to_valid_utf8_op_test.py"], deps = [ ":string_ops", - # python/framework:test_lib tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "text_similarity_metric_ops", srcs = ["python/metrics/text_similarity_metric_ops.py"], - cc_op_defs = ["//tensorflow_text/core/ops:rouge_l_op.cc"], - cc_op_kernels = ["//tensorflow_text/core/kernels:rouge_l_kernel"], - deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:rouge_l_op.cc"], + cc_op_kernels = ["@org_tensorflow_text//tensorflow_text/core/kernels:rouge_l_kernel"], + deps = [ + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -1170,14 +1182,14 @@ py_test( srcs = ["python/metrics/text_similarity_metric_ops_test.py"], deps = [ ":text_similarity_metric_ops", - "@absl_py//absl/testing:parameterized", - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -1185,8 +1197,7 @@ py_library( name = "todense_layer", srcs = ["python/keras/layers/todense.py"], deps = [ - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", + "@org_tensorflow//tensorflow:tensorflow_py", ], ) @@ -1198,10 +1209,10 @@ py_test( deps = [ ":todense_layer", "@absl_py//absl/testing:parameterized", + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/keras/api", + "@org_tensorflow//tensorflow/python/framework:test_lib", "@pypi//numpy:pkg", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:test_lib tensorflow dep, ], ) @@ -1210,7 +1221,7 @@ py_library( srcs = ["python/ops/tokenization.py"], deps = [ ":splitter", - # python/module tensorflow dep, + "@org_tensorflow//tensorflow/python/module", ] + extra_py_deps(), ) @@ -1221,10 +1232,9 @@ py_library( ":unicode_script_tokenizer", ":whitespace_tokenizer_v2", ":wordpiece_tokenizer", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/ops:lookup_ops tensorflow dep, - # python/ops/ragged:ragged_conversion_ops tensorflow dep, + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_conversion_ops", ], ) @@ -1236,31 +1246,31 @@ py_test( deps = [ ":tokenization_layers", "@absl_py//absl/testing:parameterized", + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/keras/api", "@pypi//numpy:pkg", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", ], ) py_tf_text_library( name = "trimmer_ops", srcs = ["python/ops/trimmer_ops.py"], - cc_op_defs = ["//tensorflow_text/core/ops:trimmer_ops.cc"], - cc_op_kernels = ["//tensorflow_text/core/kernels:round_robin_trimmer_kernel"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:trimmer_ops.cc"], + cc_op_kernels = ["@org_tensorflow_text//tensorflow_text/core/kernels:round_robin_trimmer_kernel"], deps = [ ":item_selector_ops", - # python/framework:constant_op tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:functional_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:while_loop tensorflow dep, - # python/ops/ragged:ragged_array_ops tensorflow dep, - # python/ops/ragged:ragged_map_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:functional_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:while_loop", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_map_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -1270,13 +1280,13 @@ py_test( deps = [ ":tensorflow_text", ":trimmer_ops", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -1285,6 +1295,7 @@ filegroup( srcs = glob([ "python/ops/test_data/segmenter_hub_module/**", ]), + visibility = ["//learning/tfx/users/nlx:__subpackages__"], ) py_library( @@ -1292,13 +1303,13 @@ py_library( srcs = ["python/ops/unicode_char_tokenizer.py"], deps = [ ":tokenization", - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_string_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -1309,30 +1320,30 @@ py_test( shard_count = 5, deps = [ ":unicode_char_tokenizer", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "unicode_script_tokenizer", srcs = ["python/ops/unicode_script_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:unicode_script_tokenize_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:unicode_script_tokenize_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:unicode_script_tokenize_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:unicode_script_tokenize_kernel", ], deps = [ ":tokenization", - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_conversion_ops tensorflow dep, - # python/ops/ragged:ragged_string_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_conversion_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -1343,24 +1354,24 @@ py_test( shard_count = 5, deps = [ ":unicode_script_tokenizer", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "utf8_binarize_op", srcs = ["python/ops/utf8_binarize_op.py"], - cc_op_defs = ["//tensorflow_text/core/ops:utf8_binarize_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:utf8_binarize_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:utf8_binarize_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:utf8_binarize_kernel", ], visibility = ["//visibility:private"], deps = [ - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", ], ) @@ -1371,10 +1382,10 @@ py_test( deps = [ ":tensorflow_text", ":utf8_binarize_op", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:test_lib tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -1385,10 +1396,10 @@ py_tf_text_library( ":constrained_sequence_op", ], deps = [ - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ] + extra_py_deps(), ) @@ -1399,10 +1410,10 @@ py_test( deps = [ ":viterbi_constrained_sequence_op", ":viterbi_decode", - "@pypi//numpy:pkg", - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", + "@org_tensorflow_text//tensorflow_text/python/numpy", ], ) @@ -1420,7 +1431,7 @@ py_test( srcs = ["python/numpy/viterbi_decode_test.py"], deps = [ ":viterbi_decode", - "@absl_py//absl/testing:absltest", + "//third_party/py/absl/testing:absltest", "@pypi//numpy:pkg", ], ) @@ -1428,42 +1439,42 @@ py_test( py_tf_text_library( name = "whitespace_tokenizer", srcs = [], - cc_op_defs = ["//tensorflow_text/core/ops:whitespace_tokenize_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:whitespace_tokenize_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:whitespace_tokenize_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:whitespace_tokenize_kernel", ], deps = [ ":tokenization", - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_conversion_ops tensorflow dep, - # python/ops/ragged:ragged_string_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_conversion_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) py_tf_text_library( name = "whitespace_tokenizer_v2", srcs = ["python/ops/whitespace_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:whitespace_tokenizer_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:whitespace_tokenizer_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:whitespace_tokenizer_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:whitespace_tokenizer_kernel", ], deps = [ ":tokenization", ":whitespace_tokenizer", - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_conversion_ops tensorflow dep, - # python/ops/ragged:ragged_string_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, - "//tensorflow_text/core/pybinds:pywrap_whitespace_tokenizer_config_builder", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_conversion_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", + "@org_tensorflow_text//tensorflow_text/core/pybinds:pywrap_whitespace_tokenizer_config_builder", ], ) @@ -1474,40 +1485,40 @@ py_test( shard_count = 4, deps = [ ":tensorflow_text", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - "@release_or_nightly//:tf_keras_pkg", - # python/framework:constant_op tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/tensorflow", + "//third_party/py/tf_keras/api:tf_keras_api", + "@org_tensorflow//tensorflow/python/framework:constant_op", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) py_tf_text_library( name = "wordpiece_tokenizer", srcs = ["python/ops/wordpiece_tokenizer.py"], - cc_op_defs = ["//tensorflow_text/core/ops:wordpiece_op.cc"], + cc_op_defs = ["@org_tensorflow_text//tensorflow_text/core/ops:wordpiece_op.cc"], cc_op_kernels = [ - "//tensorflow_text/core/kernels:wordpiece_kernel", + "@org_tensorflow_text//tensorflow_text/core/kernels:wordpiece_kernel", ], deps = [ ":tokenization", - # python/compat tensorflow dep, - # python/eager:monitoring tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:ops tensorflow dep, - # python/framework:tensor tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:check_ops tensorflow dep, - # python/ops:control_flow_assert tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops:sort_ops tensorflow dep, - # python/ops:string_ops tensorflow dep, - # python/ops/ragged:ragged_functional_ops tensorflow dep, - # python/ops/ragged:ragged_string_ops tensorflow dep, - # python/ops/ragged:ragged_tensor tensorflow dep, + "@org_tensorflow//tensorflow/python/compat", + "@org_tensorflow//tensorflow/python/eager:monitoring", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/framework:tensor", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:check_ops", + "@org_tensorflow//tensorflow/python/ops:control_flow_assert", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops:sort_ops", + "@org_tensorflow//tensorflow/python/ops:string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_functional_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_string_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_tensor", ], ) @@ -1517,15 +1528,15 @@ py_test( srcs = ["python/ops/wordpiece_tokenizer_test.py"], deps = [ ":wordpiece_tokenizer", - "@absl_py//absl/testing:parameterized", - # python/compat tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/framework:test_lib tensorflow dep, - # python/ops:array_ops tensorflow dep, - # python/ops:lookup_ops tensorflow dep, - # python/ops:math_ops tensorflow dep, - # python/ops/ragged:ragged_factory_ops tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "//third_party/py/absl/testing:parameterized", + "@org_tensorflow//tensorflow/python/compat", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/ops:array_ops", + "@org_tensorflow//tensorflow/python/ops:lookup_ops", + "@org_tensorflow//tensorflow/python/ops:math_ops", + "@org_tensorflow//tensorflow/python/ops/ragged:ragged_factory_ops", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -1533,9 +1544,9 @@ py_library( name = "wordshape_ops", srcs = ["python/ops/wordshape_ops.py"], deps = [ - # python/framework:ops tensorflow dep, - # python/ops:array_ops_stack tensorflow dep, - # python/ops:string_ops tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:ops", + "@org_tensorflow//tensorflow/python/ops:array_ops_stack", + "@org_tensorflow//tensorflow/python/ops:string_ops", ] + extra_py_deps(), ) @@ -1545,8 +1556,8 @@ py_test( srcs = ["python/ops/wordshape_ops_test.py"], deps = [ ":wordshape_ops", - # python/framework:test_lib tensorflow dep, - # python/platform:client_testlib tensorflow dep, + "@org_tensorflow//tensorflow/python/framework:test_lib", + "@org_tensorflow//tensorflow/python/platform:client_testlib", ], ) @@ -1561,21 +1572,32 @@ py_test( # regular expression that is matched against the benchmark methods' name. # e.g. --benchmark_filter="wordpiece" will run the Wordpiece Tokenizer benchmarks. -py_library(name = "profiler_lib") +py_library( + name = "profiler_lib", + deps = [ + "//perftools/gputools/profiler:dsprof_lib", + "//perftools/gputools/profiler:gpuprof_lib", + "//perftools/gputools/profiler:xprof_lib", + "//perftools/gputools/profiler:xprofilez", + "//perftools/gputools/profiler:xprofilez_with_server", + ], +) py_library( name = "benchmark_utils", srcs = ["python/benchmarks/benchmark_utils.py"], deps = [ + ":profiler_lib", + "//perftools/accelerators/xprof/api/python:xprof_session", + "//third_party/py/tensorflow", + "//third_party/py/tensorflow_datasets", + "@org_tensorflow//tensorflow/python/client:session", + "@org_tensorflow//tensorflow/python/data/ops:dataset_ops", + "@org_tensorflow//tensorflow/python/eager:context", + "@org_tensorflow//tensorflow/python/eager:def_function", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/util:tf_inspect", "@pypi//numpy:pkg", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - # tensorflow datasets dep, - # python/client:session tensorflow dep, - # python/data/ops:dataset_ops tensorflow dep, - # python/eager:context tensorflow dep, - # python/eager:def_function tensorflow dep, - # python/framework:dtypes tensorflow dep, - # python/util:tf_inspect tensorflow dep, ], ) @@ -1592,11 +1614,11 @@ py_test( deps = [ ":benchmark_utils", ":ops", - "@absl_py//absl:app", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - # python/framework:dtypes tensorflow dep, - # python/user_ops:ops tensorflow dep, + "//third_party/py/absl:app", + "//third_party/py/absl/testing:parameterized", + "//third_party/py/tensorflow", + "@org_tensorflow//tensorflow/python/framework:dtypes", + "@org_tensorflow//tensorflow/python/user_ops:ops", ], ) @@ -1607,9 +1629,9 @@ py_test( deps = [ ":benchmark_utils", ":ops", - "@absl_py//absl:app", - "@absl_py//absl/testing:parameterized", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep - # python/user_ops:ops tensorflow dep, + "//third_party/py/absl:app", + "//third_party/py/absl/testing:parameterized", + "//third_party/py/tensorflow", + "@org_tensorflow//tensorflow/python/user_ops:ops", ], ) diff --git a/tensorflow_text/__init__.py b/tensorflow_text/__init__.py index d40d3b6cc..a6b4e8bb4 100644 --- a/tensorflow_text/__init__.py +++ b/tensorflow_text/__init__.py @@ -110,4 +110,4 @@ ] remove_undocumented(__name__, _allowed_symbols) -__version__ = "2.13.0" +__version__ = "2.20.0" diff --git a/tensorflow_text/__pycache__/__init__.cpython-311.pyc b/tensorflow_text/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 000000000..77767e649 Binary files /dev/null and b/tensorflow_text/__pycache__/__init__.cpython-311.pyc differ diff --git a/tensorflow_text/core/kernels/BUILD b/tensorflow_text/core/kernels/BUILD index 8e6fc14d2..b77af8388 100644 --- a/tensorflow_text/core/kernels/BUILD +++ b/tensorflow_text/core/kernels/BUILD @@ -337,6 +337,7 @@ cc_library( deps = [ "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@darts_clone", ], ) diff --git a/tensorflow_text/core/kernels/darts_clone_trie_builder.cc b/tensorflow_text/core/kernels/darts_clone_trie_builder.cc index b7fafefaa..4eb492b14 100644 --- a/tensorflow_text/core/kernels/darts_clone_trie_builder.cc +++ b/tensorflow_text/core/kernels/darts_clone_trie_builder.cc @@ -19,6 +19,7 @@ #include #include "absl/container/flat_hash_set.h" +#include "absl/strings/str_cat.h" #include "include/darts.h" namespace tensorflow { diff --git a/tensorflow_text/core/kernels/whitespace_tokenizer_config_builder.cc b/tensorflow_text/core/kernels/whitespace_tokenizer_config_builder.cc index e3a4d026b..6e79c2573 100644 --- a/tensorflow_text/core/kernels/whitespace_tokenizer_config_builder.cc +++ b/tensorflow_text/core/kernels/whitespace_tokenizer_config_builder.cc @@ -14,53 +14,62 @@ #include "tensorflow_text/core/kernels/whitespace_tokenizer_config_builder.h" +#include #include +#include "icu4c/source/common/unicode/appendable.h" +#include "icu4c/source/common/unicode/bytestream.h" +#include "icu4c/source/common/unicode/edits.h" +#include "icu4c/source/common/unicode/normalizer2.h" +#include "icu4c/source/common/unicode/schriter.h" +#include "icu4c/source/common/unicode/stringoptions.h" +#include "icu4c/source/common/unicode/stringpiece.h" #include "icu4c/source/common/unicode/uchar.h" +#include "icu4c/source/common/unicode/ucnv.h" +#include "icu4c/source/common/unicode/ucnv_err.h" #include "icu4c/source/common/unicode/umachine.h" #include "icu4c/source/common/unicode/uniset.h" +#include "icu4c/source/common/unicode/unistr.h" #include "icu4c/source/common/unicode/uset.h" +#include "icu4c/source/common/unicode/utf.h" #include "icu4c/source/common/unicode/utf8.h" #include "icu4c/source/common/unicode/utypes.h" namespace tensorflow { namespace text { -namespace { - -const icu::UnicodeSet& WhiteSpaceSet() { - // Will not fail because the data is hardcoded in the ICU library. - UErrorCode error_code = U_ZERO_ERROR; - const USet* c_set = u_getBinaryPropertySet(UCHAR_WHITE_SPACE, &error_code); - // assert(U_SUCCESS(error_code)); - const icu::UnicodeSet* set = icu::UnicodeSet::fromUSet(c_set); - return *set; -} - -} // namespace - std::string BuildWhitespaceString() { - std::string str; - char buf[U8_MAX_LENGTH]; - for (auto cp : WhiteSpaceSet().codePoints()) { - int len = 0; - U8_APPEND_UNSAFE(buf, len, cp); - str.append(buf, len); + icu::UnicodeString unicode_string; + icu::UnicodeStringAppendable appendable_unicode_string(unicode_string); + // The maximum codepoint in Unicode is 0x0010FFFF. + for (UChar32 cp = 0; cp <= 0x0010FFFF; ++cp) { + if (U_IS_UNICODE_CHAR(cp) && u_isUWhiteSpace(cp)) { + appendable_unicode_string.appendCodePoint(cp); + } } + std::string str; + unicode_string.toUTF8String(str); return str; } std::string BuildWhitespaceTokenizerConfig() { - const icu::UnicodeSet& set = WhiteSpaceSet(); - int range_count = set.getRangeCount(); - UChar32 largest_whitespace = set.getRangeEnd(range_count - 1); + // The maximum codepoint in Unicode is 0x0010FFFF. + UChar32 max_unicode_char = 0x0010FFFF; // The string will hold our bit array - std::string bitset((largest_whitespace >> 3) + 1, 0); - for (auto cp : set.codePoints()) { - int index = cp >> 3; - bitset[index] |= 1 << (cp & 7); + std::string bitset((max_unicode_char >> 3) + 1, 0); + auto bitdata = bitset.begin(); + UChar32 largest_whitespace = 0; + int shift = 0; + for (UChar32 cp = 0; cp <= max_unicode_char; ++cp, ++shift) { + if (shift == 8) { + ++bitdata; + shift = 0; + } + bool is_whitespace = U_IS_UNICODE_CHAR(cp) && u_isUWhiteSpace(cp); + largest_whitespace = is_whitespace ? cp : largest_whitespace; + *bitdata |= is_whitespace << shift; } - return bitset; + return bitset.substr(0, (largest_whitespace >> 3) + 1); } } // namespace text diff --git a/tensorflow_text/core/pybinds/BUILD b/tensorflow_text/core/pybinds/BUILD index 68b03161b..3a777f068 100644 --- a/tensorflow_text/core/pybinds/BUILD +++ b/tensorflow_text/core/pybinds/BUILD @@ -1,7 +1,7 @@ # Code that exposes C++ libraries to Python via pybind11. -# Placeholder: load py_test load("@org_tensorflow//tensorflow:tensorflow.bzl", "pybind_extension") +load("@rules_python//python:py_test.bzl", "py_test") licenses(["notice"]) @@ -12,7 +12,7 @@ package(default_visibility = [ ]) pybind_extension( - name = "tflite_registrar", + name = "tflite_registrar_library", srcs = [ "tflite_registrar.cc", ], @@ -39,11 +39,12 @@ pybind_extension( "tflite_registrar.pyi", ], deps = [ - "@pybind11", - # lite:framework tensorflow dep, - # lite/c:common tensorflow dep, - # lite/kernels:builtin_ops tensorflow dep, "//tensorflow_text/core/kernels:tflite_ops", + "@org_tensorflow//tensorflow/lite:framework", + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", + "@pybind11", + "@pypi//numpy:pkg", ], ) @@ -63,6 +64,7 @@ pybind_extension( deps = [ "//tensorflow_text/core/kernels:fast_bert_normalizer_model_builder", "@pybind11", + "@pypi//numpy:pkg", ], ) @@ -75,7 +77,7 @@ py_test( ], deps = [ ":pywrap_fast_bert_normalizer_model_builder", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep + "@org_tensorflow//tensorflow", ], ) @@ -94,7 +96,9 @@ pybind_extension( module_name = "pywrap_fast_wordpiece_tokenizer_model_builder", deps = [ "//tensorflow_text/core/kernels:fast_wordpiece_tokenizer_model_builder", + "@org_tensorflow//tensorflow/core:framework", "@pybind11", + "@pypi//numpy:pkg", ], ) @@ -106,7 +110,7 @@ py_test( ], deps = [ ":pywrap_fast_wordpiece_tokenizer_model_builder", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep + "@org_tensorflow//tensorflow", ], ) @@ -130,6 +134,7 @@ pybind_extension( deps = [ "//tensorflow_text/core/kernels:phrase_tokenizer_model_builder", "@pybind11", + "@pypi//numpy:pkg", ], ) @@ -141,7 +146,7 @@ py_test( ], deps = [ ":pywrap_phrase_tokenizer_model_builder", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep + "@org_tensorflow//tensorflow", ], ) @@ -163,6 +168,7 @@ pybind_extension( deps = [ "//tensorflow_text/core/kernels/sentencepiece:model_converter", "@pybind11", + "@pypi//numpy:pkg", ], ) @@ -182,6 +188,7 @@ pybind_extension( deps = [ "//tensorflow_text/core/kernels:whitespace_tokenizer_config_builder", "@pybind11", + "@pypi//numpy:pkg", ], ) @@ -190,6 +197,6 @@ py_test( srcs = ["pywrap_whitespace_tokenizer_config_builder_test.py"], deps = [ ":pywrap_whitespace_tokenizer_config_builder", - "@release_or_nightly//:tensorflow_pkg", # tensorflow package dep + "@org_tensorflow//tensorflow", ], ) diff --git a/tensorflow_text/py_deps_profile.bzl b/tensorflow_text/py_deps_profile.bzl new file mode 100644 index 000000000..b13937f11 --- /dev/null +++ b/tensorflow_text/py_deps_profile.bzl @@ -0,0 +1,49 @@ +""" +Repository rule for open source dependencies. +""" + +def _py_deps_profile_impl(ctx): + is_switch = False + for var_name, var_val in ctx.attr.switch.items(): + is_switch = is_switch or ctx.os.environ.get(var_name, "") == var_val + + prefix = ctx.attr.pip_repo_name + reqirements_name = ctx.attr.requirements_in.name + requirements_in_substitutions = {} + build_content = ['exports_files(["{}"])'.format(reqirements_name)] + for k, v in ctx.attr.deps_map.items(): + repo_name = v[0] if is_switch else k + requirements_in_substitutions[k + "\n"] = repo_name + "\n" + requirements_in_substitutions[k + "\r\n"] = repo_name + "\r\n" + aliased_targets = ["pkg"] + v[1:] + norm_repo_name = repo_name.replace("-", "_") + norm_alas_name = k.replace("-", "_") + for target in aliased_targets: + alias_name = "{}_{}".format(norm_alas_name, target) + alias_value = "@{}_{}//:{}".format(prefix, norm_repo_name, target) + build_content.append(""" +alias( + name = "{}", + actual = "{}", + visibility = ["//visibility:public"] +) +""".format(alias_name, alias_value)) + + ctx.file("BUILD", "".join(build_content)) + ctx.template( + reqirements_name, + ctx.attr.requirements_in, + executable = False, + substitutions = requirements_in_substitutions, + ) + +py_deps_profile = repository_rule( + implementation = _py_deps_profile_impl, + attrs = { + "requirements_in": attr.label(mandatory = True), + "deps_map": attr.string_list_dict(mandatory = True), + "pip_repo_name": attr.string(mandatory = True), + "switch": attr.string_dict(mandatory = True), + }, + local = True, +) diff --git a/tensorflow_text/tftext.bzl b/tensorflow_text/tftext.bzl index 8343de92e..9a560e556 100644 --- a/tensorflow_text/tftext.bzl +++ b/tensorflow_text/tftext.bzl @@ -1,104 +1,106 @@ """ -Build rule for open source tf.text libraries. +Build rule for tf.text libraries. + +BEGIN GOOGLE-INTERNAL +Example usage: + +py_tf_text_library( + name = "string_ops", + srcs = ["python/ops/string_ops.py"], + cc_op_defs = ["core/ops/string_ops.cc"], + cc_op_kernels = ["//third_party/tensorflow_text/core/kernels:string_kernels"], + deps = ["//third_party/tensorflow/python/framework:dtypes"], +) +END GOOGLE-INTERNAL """ +load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py", "tf_opts_nortti_if_mobile") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_python//python:py_library.bzl", "py_library") + +# Generates a Python library for the ops defined in `op_defs` and `py_srcs`. +# +# Defines three targets: +# +# +# Python library that exposes all ops defined in `cc_op_defs` and `py_srcs`. +# _cc +# C++ library that registers any c++ ops in `cc_op_defs`, and includes the +# kernels from `cc_op_kernels`. +# gen__py +# Python library that exposes any c++ ops. +# +# Args: +# name: The name for the python library target build by this rule. +# srcs: Python source files for the Python library. +# deps: Dependencies for the Python library. +# compatible_with: Standard blaze cc_library for the cc_library. +# visibility: Visibility for the Python library. +# cc_op_defs: A list of c++ src files containing REGISTER_OP definitions. +# cc_op_kernels: A list of c++ targets containing kernels that are used +# by the Python library. def py_tf_text_library( name, srcs = [], deps = [], + compatible_with = None, visibility = None, cc_op_defs = [], cc_op_kernels = []): - """Creates build rules for TF.Text ops as shared libraries. - - Defines three targets: - - - Python library that exposes all ops defined in `cc_op_defs` and `py_srcs`. - _cc - C++ library that registers any c++ ops in `cc_op_defs`, and includes the - kernels from `cc_op_kernels`. - python/ops/_.so - Shared library exposing the _cc library. - - Args: - name: The name for the python library target build by this rule. - srcs: Python source files for the Python library. - deps: Dependencies for the Python library. - visibility: Visibility for the Python library. - cc_op_defs: A list of c++ src files containing REGISTER_OP definitions. - cc_op_kernels: A list of c++ targets containing kernels that are used - by the Python library. - """ - binary_path = "python/ops" - if srcs: - binary_path_end_pos = srcs[0].rfind("/") - binary_path = srcs[0][0:binary_path_end_pos] - binary_name = binary_path + "/_" + cc_op_kernels[0][1:] + ".so" if cc_op_defs: - binary_name = binary_path + "/_" + name + ".so" - library_name = name + "_cc" - native.cc_library( - name = library_name, + # C++ library that registers ops. + op_def_lib_name = name + "_cc" + cc_library( + name = op_def_lib_name, srcs = cc_op_defs, - copts = select({ - # Android supports pthread natively, -pthread is not needed. - "@org_tensorflow//tensorflow:mobile": [], - "//conditions:default": ["-pthread"], - }), - alwayslink = 1, deps = cc_op_kernels + ["@org_tensorflow//tensorflow/lite/kernels/shim:tf_op_shim"] + select({ "@org_tensorflow//tensorflow:mobile": [ "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", ], - "//conditions:default": [], + "//conditions:default": [ + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:framework_headers_lib", + ], }), - ) - - native.cc_binary( - name = binary_name, - copts = select({ - "@org_tensorflow//tensorflow:mobile": [], - "//conditions:default": ["-pthread"], - }), - linkshared = 1, - linkopts = select({ - "@org_tensorflow//tensorflow:macos": [ - "-Wl,-exported_symbols_list,$(location //tensorflow_text:exported_symbols.lds)", - ], - "@org_tensorflow//tensorflow:windows": [], - "//conditions:default": [ - "-Wl,--version-script,$(location //tensorflow_text:version_script.lds)", - ], - }), - deps = [ - ":" + library_name, - "//tensorflow_text:version_script.lds", - "//tensorflow_text:exported_symbols.lds", - ] + select({ - "@org_tensorflow//tensorflow:mobile": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", + features = select({ + "@org_tensorflow//tensorflow:android": [ + "-layering_check", ], "//conditions:default": [], }), + alwayslink = 1, + ) + deps = deps + [":" + op_def_lib_name] + + # Python wrapper that exposes c++ ops. + gen_py_lib_name = "gen_" + name + "_py" + gen_py_out = "gen_" + name + ".py" + tf_gen_op_wrapper_py( + name = gen_py_lib_name, + out = gen_py_out, + deps = [":" + op_def_lib_name], ) + deps = deps + [":" + gen_py_lib_name] - native.py_library( + # Python library that exposes all ops. + py_library( name = name, srcs = srcs, srcs_version = "PY2AND3", visibility = visibility, - data = [":" + binary_name], - deps = deps, + deps = cc_op_kernels + deps + [ + "@org_tensorflow//tensorflow/python/framework:for_generated_wrappers", + ], ) -def _dedupe(list, item): - if item not in list: - return [item] - else: - return [] +# Enable build_cleaner to update py_tf_text_library targets. +# GOOGLE-INTERNAL: See: go/build-cleaner-build-extensions +# register_extension_info( +# extension = py_tf_text_library, +# label_regex_for_dep = "{extension_name}", +# ) def tf_cc_library( name, @@ -107,6 +109,7 @@ def tf_cc_library( deps = [], tf_deps = [], copts = [], + features = [], compatible_with = None, testonly = 0, alwayslink = 0): @@ -114,6 +117,7 @@ def tf_cc_library( Just like cc_library, but: * Adds alwayslink=1 for kernels (name has kernel in it) + * Passes -DGOOGLE_CUDA=1 if we're building with --config=cuda. * Separates out TF deps for when building for Android. Args: @@ -125,36 +129,30 @@ def tf_cc_library( copts: C options compatible_with: List of environments target can be built for testonly: If library is only for testing - alwayslink: If symbols should be exported + alwayslink: If symbols should be exported. """ if "kernel" in name: alwayslink = 1 - - # These are "random" deps likely needed by each library (http://b/142433427) - oss_deps = [] - oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/container:flat_hash_map") - oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/status") - oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/status:statusor") - oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/strings:cord") - oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/time") - oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/types:variant") - oss_deps = oss_deps + _dedupe(deps, "@com_google_absl//absl/functional:any_invocable") - deps += select({ - "@org_tensorflow//tensorflow:mobile": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - "//conditions:default": [ - "@release_or_nightly//:tensorflow_libtensorflow_framework", - "@release_or_nightly//:tensorflow_tf_header_lib", - ] + tf_deps + oss_deps, - }) - native.cc_library( + if tf_deps: + deps += select({ + "@org_tensorflow//tensorflow:mobile": [ + "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", + ], + "//conditions:default": tf_deps, + }) + features += select({ + "@org_tensorflow//tensorflow:android": [ + "-layering_check", + ], + "//conditions:default": [], + }) + cc_library( name = name, srcs = srcs, hdrs = hdrs, deps = deps, - copts = copts, - compatible_with = compatible_with, + copts = copts + tf_opts_nortti_if_mobile(), + features = features, testonly = testonly, alwayslink = alwayslink, ) @@ -168,7 +166,7 @@ def tflite_cc_library( compatible_with = None, testonly = 0, alwayslink = 0): - """ A rule to build a TensorFlow library or OpKernel. + """ A rule to build a TensorFlow Lite library or OpKernel. Args: name: Name of library @@ -178,98 +176,24 @@ def tflite_cc_library( copts: C options compatible_with: List of environments target can be built for testonly: If library is only for testing - alwayslink: If symbols should be exported + alwayslink: If symbols should be exported. """ - - # Necessary build deps for tflite ops - tflite_deps = [ - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core/util:ragged_to_dense_util_common", - "@org_tensorflow//tensorflow/lite:framework", - "@org_tensorflow//tensorflow/lite:mutable_op_resolver", - "@org_tensorflow//tensorflow/lite/c:common", - "@org_tensorflow//tensorflow/lite/kernels/shim:tflite_op_shim", - "@org_tensorflow//tensorflow/lite/kernels/shim:tflite_op_wrapper", - ] - - # These are "random" deps likely needed by each library (http://b/142433427) - oss_deps = [ - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/strings:cord", - "@com_google_absl//absl/time", - "@com_google_absl//absl/types:variant", - ] - deps += tflite_deps + select({ - "@org_tensorflow//tensorflow:mobile": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - "//conditions:default": [ - "@release_or_nightly//:tensorflow_libtensorflow_framework", - "@release_or_nightly//:tensorflow_tf_header_lib", - ] + oss_deps, - }) - native.cc_library( + cc_library( name = name, srcs = srcs, hdrs = hdrs, deps = deps, - copts = copts, - compatible_with = compatible_with, + copts = copts + tf_opts_nortti_if_mobile(), testonly = testonly, alwayslink = alwayslink, ) -def extra_py_deps(): - return [ - "@release_or_nightly//:tensorflow_pkg", - "@release_or_nightly//:tf_keras_pkg", - "@pypi_tensorflow_datasets//:pkg", - "@pypi_tensorflow_metadata//:pkg", - ] - -def _py_deps_profile_impl(ctx): - is_switch = False - for var_name, var_val in ctx.attr.switch.items(): - is_switch = is_switch or ctx.os.environ.get(var_name, "") == var_val - - prefix = ctx.attr.pip_repo_name - reqirements_name = ctx.attr.requirements_in.name - requirements_in_substitutions = {} - build_content = ['exports_files(["{}"])'.format(reqirements_name)] - for k, v in ctx.attr.deps_map.items(): - repo_name = v[0] if is_switch else k - requirements_in_substitutions[k + "\n"] = repo_name + "\n" - requirements_in_substitutions[k + "\r\n"] = repo_name + "\r\n" - aliased_targets = ["pkg"] + v[1:] - norm_repo_name = repo_name.replace("-", "_") - norm_alas_name = k.replace("-", "_") - for target in aliased_targets: - alias_name = "{}_{}".format(norm_alas_name, target) - alias_value = "@{}_{}//:{}".format(prefix, norm_repo_name, target) - build_content.append(""" -alias( - name = "{}", - actual = "{}", - visibility = ["//visibility:public"] -) -""".format(alias_name, alias_value)) +# Allow build_cleaner to manage dependencies of tf_cc_library build rules. +# GOOGLE-INTERNAL: See: go/build-cleaner-build-extensions +# register_extension_info( +# extension = tf_cc_library, +# label_regex_for_dep = "{extension_name}", +# ) - ctx.file("BUILD", "".join(build_content)) - ctx.template( - reqirements_name, - ctx.attr.requirements_in, - executable = False, - substitutions = requirements_in_substitutions, - ) - -py_deps_profile = repository_rule( - implementation = _py_deps_profile_impl, - attrs = { - "requirements_in": attr.label(mandatory = True), - "deps_map": attr.string_list_dict(mandatory = True), - "pip_repo_name": attr.string(mandatory = True), - "switch": attr.string_dict(mandatory = True), - }, - local = True, -) +def extra_py_deps(): + return [] diff --git a/tensorflow_text/tools/wordpiece_vocab/BUILD b/tensorflow_text/tools/wordpiece_vocab/BUILD index 608f3c276..62093ff9d 100644 --- a/tensorflow_text/tools/wordpiece_vocab/BUILD +++ b/tensorflow_text/tools/wordpiece_vocab/BUILD @@ -1,5 +1,5 @@ -# Placeholder: load py_library -# Placeholder: load py_test +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python:py_test.bzl", "py_test") licenses(["notice"]) diff --git a/third_party/tensorflow/tf_configure.bzl b/third_party/tensorflow/tf_configure.bzl index e224a2e7a..36c4f5ecb 100644 --- a/third_party/tensorflow/tf_configure.bzl +++ b/third_party/tensorflow/tf_configure.bzl @@ -4,7 +4,10 @@ This is used for the generation of the dynamic libraries used for custom ops. See: http://github.com/tensorflow/custom-op """ -load("@python//:defs.bzl", "interpreter") +load( + "@org_tensorflow//third_party/py:python_init_toolchains.bzl", + "get_toolchain_name_per_python_version", +) load("@python_version_repo//:py_version.bzl", "REQUIREMENTS_WITH_LOCAL_WHEELS") load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse") @@ -55,6 +58,11 @@ cc_library( additive_build_content = tensorflow_annotation, ), }, - python_interpreter_target = interpreter, + python_interpreter_target = _get_python_interpreter(), requirements_lock = REQUIREMENTS_WITH_LOCAL_WHEELS, ) + +def _get_python_interpreter(): + return "@{}_host//:python".format( + get_toolchain_name_per_python_version("python"), + )