Remove tf1 stragglers from preprocessing lookup layers
PiperOrigin-RevId: 363792344
Change-Id: I61fb2791300e49112d41861dc7d1980c79ba16a0
diff --git a/tensorflow/python/keras/layers/preprocessing/BUILD b/tensorflow/python/keras/layers/preprocessing/BUILD
index b18a580..6884071 100644
--- a/tensorflow/python/keras/layers/preprocessing/BUILD
+++ b/tensorflow/python/keras/layers/preprocessing/BUILD
@@ -213,7 +213,6 @@
"//tensorflow/python:platform",
"//tensorflow/python:sparse_tensor",
"//tensorflow/python:string_ops",
- "//tensorflow/python/keras:backend",
"//tensorflow/python/keras/utils:tf_utils",
"//tensorflow/python/ops/ragged:ragged_functional_ops",
"//tensorflow/python/ops/ragged:ragged_tensor",
diff --git a/tensorflow/python/keras/layers/preprocessing/index_lookup.py b/tensorflow/python/keras/layers/preprocessing/index_lookup.py
index 88d7c36..9e452b4 100644
--- a/tensorflow/python/keras/layers/preprocessing/index_lookup.py
+++ b/tensorflow/python/keras/layers/preprocessing/index_lookup.py
@@ -217,14 +217,13 @@
value_index=value_index,
value_index_offset=total_offset)
- self._table = self._static_table_class()(
+ self._table = lookup_ops.StaticHashTable(
initializer, default_value=default_value)
self._table_handler = table_utils.TableHandler(
table=self._table,
mask_token=self._mask_key,
mask_value=self._mask_value,
- oov_tokens=oov_indices,
- use_v1_apis=self._use_v1_apis())
+ oov_tokens=oov_indices)
self.max_tokens = (
self._table_handler.table_size() + self.num_oov_indices +
(0 if mask_token is None else 1))
@@ -236,8 +235,7 @@
name=(self._name + "_index_table"))
self._table_handler = table_utils.TableHandler(
table=self._table,
- oov_tokens=oov_indices,
- use_v1_apis=self._use_v1_apis())
+ oov_tokens=oov_indices)
if vocabulary is not None:
self.set_vocabulary(vocabulary)
@@ -543,12 +541,6 @@
def _convert_to_ndarray(self, x):
return np.array(x) if isinstance(x, (list, tuple)) else x
- def _use_v1_apis(self):
- return False
-
- def _static_table_class(self):
- return lookup_ops.StaticHashTable
-
def _oov_start_index(self):
return 1 if self.mask_token is not None and self.output_mode == INT else 0
diff --git a/tensorflow/python/keras/layers/preprocessing/index_lookup_test.py b/tensorflow/python/keras/layers/preprocessing/index_lookup_test.py
index c73da2c..d8363e3 100644
--- a/tensorflow/python/keras/layers/preprocessing/index_lookup_test.py
+++ b/tensorflow/python/keras/layers/preprocessing/index_lookup_test.py
@@ -43,10 +43,6 @@
from tensorflow.python.platform import test
-def get_layer_class():
- return index_lookup.IndexLookup
-
-
def _get_end_to_end_test_cases():
test_cases = (
{
@@ -310,7 +306,7 @@
def test_layer_end_to_end_with_adapt(self, vocab_data, input_data, kwargs,
use_dataset, expected_output,
input_dtype):
- cls = get_layer_class()
+ cls = index_lookup.IndexLookup
if "invert" in kwargs and kwargs["invert"]:
expected_output_dtype = kwargs["dtype"]
elif "output_mode" in kwargs and kwargs["output_mode"] != index_lookup.INT:
@@ -371,7 +367,7 @@
expected_dense_shape = [3, 4]
input_data = keras.Input(shape=(None,), dtype=dtypes.string, sparse=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -397,7 +393,7 @@
expected_dense_shape = [3, 4]
input_data = keras.Input(shape=(None,), dtype=dtypes.int64, sparse=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
dtype=dtypes.int64,
num_oov_indices=1,
@@ -418,7 +414,7 @@
expected_output = [[2, 3, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string, ragged=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -437,7 +433,7 @@
expected_output = [[2, 3, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.int64, ragged=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
dtype=dtypes.int64,
num_oov_indices=1,
@@ -456,7 +452,7 @@
expected_output = [[2, 3, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.int32, ragged=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
dtype=dtypes.int64,
num_oov_indices=1,
@@ -486,7 +482,7 @@
expected_dense_shape = [3, 4]
input_data = keras.Input(shape=(None,), dtype=dtypes.string, sparse=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=2,
mask_token="",
@@ -512,7 +508,7 @@
expected_dense_shape = [3, 4]
input_data = keras.Input(shape=(None,), dtype=dtypes.int64, sparse=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
dtype=dtypes.int64,
num_oov_indices=2,
@@ -533,7 +529,7 @@
expected_output = [[3, 4, 6], [6, 5, 3, 2]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string, ragged=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=2,
mask_token="",
@@ -552,7 +548,7 @@
expected_output = [[3, 4, 6], [6, 5, 3, 2]]
input_data = keras.Input(shape=(None,), dtype=dtypes.int64, ragged=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
dtype=dtypes.int64,
num_oov_indices=2,
@@ -577,7 +573,7 @@
dense_shape=[3, 4])
vocab_dataset = dataset_ops.Dataset.from_tensors(vocab_data)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -592,7 +588,7 @@
["fire", "michigan"]])
vocab_dataset = dataset_ops.Dataset.from_tensors(vocab_data)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -614,7 +610,7 @@
expected_dense_shape = [3, 4]
input_data = keras.Input(shape=(None,), dtype=dtypes.int64, sparse=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
dtype=dtypes.int64,
num_oov_indices=1,
@@ -635,7 +631,7 @@
expected_output = [[2, 3, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string, ragged=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -654,7 +650,7 @@
expected_output = [[2, 3, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.int64, ragged=True)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
dtype=dtypes.int64,
num_oov_indices=1,
@@ -676,7 +672,7 @@
tensor_shape.TensorShape([]))
batched_ds = ds.take(2)
input_t = keras.Input(shape=(), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=10,
num_oov_indices=0,
mask_token=None,
@@ -706,7 +702,7 @@
expected_output = [[2, 3, 4, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -720,7 +716,7 @@
def test_int_output_shape(self):
input_data = keras.Input(batch_size=16, shape=(4,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=2,
num_oov_indices=1,
mask_token="",
@@ -736,7 +732,7 @@
expected_output = [[1, 2, 3, 4], [4, 3, 1, 0]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=None,
@@ -755,7 +751,7 @@
expected_output = [[1, 2, 3, -1], [4, 3, 1, -1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=0,
mask_token="",
@@ -774,7 +770,7 @@
expected_output = [[2, 3, 4, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -797,7 +793,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=6,
num_oov_indices=1,
mask_token="",
@@ -822,7 +818,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=5,
num_oov_indices=0,
mask_token="",
@@ -854,7 +850,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=5,
num_oov_indices=1,
mask_token="",
@@ -885,7 +881,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -900,7 +896,7 @@
def test_binary_output_shape(self):
input_data = keras.Input(batch_size=16, shape=(4,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=2,
num_oov_indices=1,
mask_token="",
@@ -921,7 +917,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=6,
num_oov_indices=1,
mask_token="",
@@ -946,7 +942,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -961,7 +957,7 @@
def test_count_output_shape(self):
input_data = keras.Input(batch_size=16, shape=(4,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=2,
num_oov_indices=1,
mask_token="",
@@ -984,7 +980,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=6,
num_oov_indices=1,
mask_token="",
@@ -1011,7 +1007,7 @@
]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1026,7 +1022,7 @@
def test_ifidf_output_shape(self):
input_data = keras.Input(batch_size=16, shape=(4,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=2,
num_oov_indices=1,
mask_token="",
@@ -1045,7 +1041,7 @@
vocab_file = self._write_to_temp_file("temp", vocab_data)
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
vocabulary=vocab_file,
max_tokens=None,
num_oov_indices=1,
@@ -1066,7 +1062,7 @@
vocab_file = self._write_to_temp_file("temp", vocab_data)
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
vocabulary=vocab_file,
max_tokens=None,
num_oov_indices=1,
@@ -1085,7 +1081,7 @@
vocab_file = self._write_to_temp_file("temp", vocab_data)
input_data = keras.Input(shape=(None,), dtype=dtypes.int64)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
vocabulary=vocab_file,
max_tokens=None,
num_oov_indices=1,
@@ -1110,7 +1106,7 @@
expected_output = [[2, 3, 4, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -1129,7 +1125,7 @@
expected_output = [[2, 3, 4, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -1143,7 +1139,7 @@
def test_vocab_with_max_cap(self):
vocab_data = ["", "[OOV]", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=5,
num_oov_indices=1,
mask_token="",
@@ -1156,7 +1152,7 @@
def test_int_vocab_with_max_cap(self):
vocab_data = [0, -1, 42, 1276, 1138]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=5,
num_oov_indices=1,
mask_token=0,
@@ -1169,7 +1165,7 @@
def test_vocab_with_multiple_oov_indices(self):
vocab_data = ["", "[OOV]", "[OOV]", "[OOV]", "wind"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=3,
mask_token="",
@@ -1181,7 +1177,7 @@
def test_int_vocab_with_multiple_oov_indices(self):
vocab_data = [0, -1, -1, -1, 42]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=3,
mask_token=0,
@@ -1194,7 +1190,7 @@
def test_non_unique_vocab_fails(self):
vocab_data = ["earth", "wind", "and", "fire", "fire"]
with self.assertRaisesRegex(ValueError, ".*repeated term.*fire.*"):
- _ = get_layer_class()(
+ _ = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -1204,7 +1200,7 @@
def test_vocab_with_oov_and_wrong_mask_fails(self):
vocab_data = ["custom_mask", "[OOV]", "earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1215,7 +1211,7 @@
def test_vocab_with_oov_and_no_mask_fails(self):
vocab_data = ["[OOV]", "earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1226,7 +1222,7 @@
def test_vocab_with_mask_but_no_oov_fails(self):
vocab_data = ["", "earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1237,7 +1233,7 @@
def test_vocab_with_repeated_element_fails(self):
vocab_data = ["earth", "earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1248,7 +1244,7 @@
def test_vocab_with_reserved_oov_element_fails(self):
vocab_data = ["earth", "test", "[OOV]", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1259,7 +1255,7 @@
def test_vocab_with_reserved_mask_element_fails(self):
vocab_data = ["earth", "mask_token", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="mask_token",
@@ -1270,7 +1266,7 @@
def test_vocab_set_after_call_pad_to_max_false_fails(self):
vocab_data = ["earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1287,7 +1283,7 @@
def test_vocab_with_idf_weights_non_tfidf_output_fails(self):
vocab_data = ["earth", "wind", "and", "fire"]
weight_data = [1, 1, 1, 1, 1]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1301,7 +1297,7 @@
def test_vocab_with_idf_weights_length_mismatch_fails(self):
vocab_data = ["earth", "wind", "and", "fire"]
weight_data = [1, 1, 1, 1, 1] # too long
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1314,7 +1310,7 @@
def test_vocab_without_idf_weights_tfidf_output_fails(self):
vocab_data = ["earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1328,7 +1324,7 @@
def test_non_unique_int_vocab_fails(self):
vocab_data = [12, 13, 14, 15, 15]
with self.assertRaisesRegex(ValueError, "repeated term.*15"):
- _ = get_layer_class()(
+ _ = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -1338,7 +1334,7 @@
def test_int_vocab_with_oov_and_wrong_mask_fails(self):
vocab_data = [1234, -1, 11, 21, 13, 14]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=0,
@@ -1349,7 +1345,7 @@
def test_int_vocab_with_oov_and_no_mask_fails(self):
vocab_data = [-1, 11, 12, 13, 14]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=0,
@@ -1360,7 +1356,7 @@
def test_int_vocab_with_mask_but_no_oov_fails(self):
vocab_data = [0, 11, 12, 13, 14]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=0,
@@ -1371,7 +1367,7 @@
def test_int_vocab_with_repeated_element_fails(self):
vocab_data = [11, 11, 34, 23, 124]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=0,
@@ -1382,7 +1378,7 @@
def test_int_vocab_with_reserved_oov_element_fails(self):
vocab_data = [14, 38, -1, 34, 3, 84]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=0,
@@ -1393,7 +1389,7 @@
def test_int_vocab_with_reserved_mask_element_fails(self):
vocab_data = [125, 0, 3, 4, 94]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=0,
@@ -1404,7 +1400,7 @@
def test_no_vocab_file_string_fails(self):
with self.assertRaisesRegex(ValueError, ".*non_existant_file.*"):
- _ = get_layer_class()(
+ _ = index_lookup.IndexLookup(
vocabulary="non_existant_file",
max_tokens=None,
num_oov_indices=1,
@@ -1425,7 +1421,7 @@
["fire", "and", "earth", "[OOV]"]])
input_data = keras.Input(shape=(None,), dtype=dtypes.int64)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -1440,7 +1436,7 @@
def test_vocab_with_max_cap(self):
vocab_data = ["", "[OOV]", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=5,
num_oov_indices=1,
mask_token="",
@@ -1453,7 +1449,7 @@
def test_int_vocab_with_max_cap(self):
vocab_data = [0, -1, 42, 1276, 1138]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=5,
num_oov_indices=1,
mask_token=0,
@@ -1467,7 +1463,7 @@
def test_non_unique_vocab_fails(self):
vocab_data = ["earth", "wind", "and", "fire", "fire"]
with self.assertRaisesRegex(ValueError, ".*repeated term.*fire.*"):
- _ = get_layer_class()(
+ _ = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -1478,7 +1474,7 @@
def test_non_int_output_fails(self):
with self.assertRaisesRegex(ValueError, "`output_mode` must be int"):
- _ = get_layer_class()(
+ _ = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1489,7 +1485,7 @@
def test_vocab_with_repeated_element_fails(self):
vocab_data = ["earth", "earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1501,7 +1497,7 @@
def test_vocab_with_reserved_mask_element_fails(self):
vocab_data = ["earth", "mask_token", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="mask_token",
@@ -1514,7 +1510,7 @@
def test_non_unique_int_vocab_fails(self):
vocab_data = [12, 13, 14, 15, 15]
with self.assertRaisesRegex(ValueError, ".*repeated term.*15.*"):
- _ = get_layer_class()(
+ _ = index_lookup.IndexLookup(
vocabulary=vocab_data,
max_tokens=None,
num_oov_indices=1,
@@ -1525,7 +1521,7 @@
def test_int_vocab_with_repeated_element_fails(self):
vocab_data = [11, 11, 34, 23, 124]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token=0,
@@ -1543,7 +1539,7 @@
def test_too_long_vocab_fails_in_single_setting(self):
vocab_data = ["earth", "wind", "and", "fire"]
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=4,
num_oov_indices=1,
mask_token="",
@@ -1555,7 +1551,7 @@
def test_zero_max_tokens_fails(self):
with self.assertRaisesRegex(ValueError, ".*max_tokens.*"):
- _ = get_layer_class()(
+ _ = index_lookup.IndexLookup(
max_tokens=0,
num_oov_indices=1,
mask_token="",
@@ -1575,7 +1571,7 @@
# Build and validate a golden model.
input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
+ layer = index_lookup.IndexLookup(
max_tokens=None,
num_oov_indices=1,
mask_token="",
@@ -1598,7 +1594,7 @@
keras.backend.clear_session()
loaded_model = keras.models.load_model(
- output_path, custom_objects={"IndexLookup": get_layer_class()})
+ output_path, custom_objects={"IndexLookup": index_lookup.IndexLookup})
# Ensure that the loaded model is unique (so that the save/load is real)
self.assertIsNot(model, loaded_model)
diff --git a/tensorflow/python/keras/layers/preprocessing/table_utils.py b/tensorflow/python/keras/layers/preprocessing/table_utils.py
index 69eeb49..f0e69e5 100644
--- a/tensorflow/python/keras/layers/preprocessing/table_utils.py
+++ b/tensorflow/python/keras/layers/preprocessing/table_utils.py
@@ -23,7 +23,6 @@
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.keras import backend as K
from tensorflow.python.keras.utils import tf_utils
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import lookup_ops
@@ -42,23 +41,12 @@
table,
oov_tokens=None,
mask_token=None,
- mask_value=0,
- use_v1_apis=False):
+ mask_value=0):
self.table = table
-
- # If we are using V1 APIs, and the table has an initializer, we need to run
- # it. However, not all tables have initializers, so we try-except here.
- if use_v1_apis:
- try:
- K.get_session().run(self.table.initializer)
- except AttributeError:
- pass
-
self.mutable = isinstance(table, lookup_ops.MutableHashTable)
self.mask_token = mask_token
self.mask_value = mask_value
- self.use_v1_apis = use_v1_apis
if oov_tokens is None:
self.oov_tokens = oov_tokens
else:
@@ -68,17 +56,17 @@
def data(self):
keys, values = self.table.export()
- return (self._eval(keys), self._eval(values))
+ return (keys.numpy(), values.numpy())
def table_size(self):
- return self._eval(self.table.size())
+ return self.table.size().numpy()
def clear(self):
if not self.mutable:
return RuntimeError("Unable to clear a statically-backed table.")
keys, _ = self.table.export()
- self._run(self.table.remove(keys))
+ self.table.remove(keys)
def insert(self, keys, values):
"""Insert values into the backed table."""
@@ -96,7 +84,7 @@
if values.shape.ndims != 1:
raise ValueError("`values` must be 1-dimensional, got an input with "
" %s dimensions." % values.shape.ndims)
- self._run(self.table.insert(keys, values))
+ self.table.insert(keys, values)
def _replace_oov_buckets(self, inputs, lookups):
"""Replace the default OOV value with one of the OOV bucket values."""
@@ -193,16 +181,6 @@
inputs = ops.convert_to_tensor_v2_with_dispatch(inputs)
return self._tensor_lookup(inputs)
- def _eval(self, tensor):
- if self.use_v1_apis:
- return K.get_session().run(tensor)
- else:
- return tensor.numpy()
-
- def _run(self, op):
- if self.use_v1_apis:
- K.get_session().run(op)
-
def get_vocabulary_from_file(vocabulary_path, encoding="utf-8"):
"""Read a vocabulary in from a file."""
diff --git a/tensorflow/python/keras/layers/preprocessing/table_utils_test.py b/tensorflow/python/keras/layers/preprocessing/table_utils_test.py
index d23eb97..f6509f7 100644
--- a/tensorflow/python/keras/layers/preprocessing/table_utils_test.py
+++ b/tensorflow/python/keras/layers/preprocessing/table_utils_test.py
@@ -23,7 +23,6 @@
import numpy as np
-from tensorflow.python.eager import context
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.keras import keras_parameterized
@@ -41,8 +40,7 @@
value_dtype=dtypes.int64,
default_value=-7,
name="index_table")
- return table_utils.TableHandler(
- table, oov_tokens, use_v1_apis=(not context.executing_eagerly()))
+ return table_utils.TableHandler(table, oov_tokens)
def get_static_table(tmpdir,
@@ -68,19 +66,14 @@
dtypes.int64,
lookup_ops.TextFileIndex.LINE_NUMBER,
value_index_offset=offset)
- if context.executing_eagerly():
- table = lookup_ops.StaticHashTable(init, default_value=-7)
- else:
- table = lookup_ops.StaticHashTableV1(init, default_value=-7)
-
+ table = lookup_ops.StaticHashTable(init, default_value=-7)
return table_utils.TableHandler(
table,
oov_tokens,
- mask_token=mask_token,
- use_v1_apis=(not context.executing_eagerly()))
+ mask_token=mask_token)
-@keras_parameterized.run_all_keras_modes
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
class CategoricalEncodingInputTest(
keras_parameterized.TestCase,
preprocessing_test_utils.PreprocessingLayerTest):
@@ -157,7 +150,7 @@
table.insert(key_data, value_data)
-@keras_parameterized.run_all_keras_modes
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
class CategoricalEncodingMultiOOVTest(
keras_parameterized.TestCase,
preprocessing_test_utils.PreprocessingLayerTest):
@@ -247,7 +240,7 @@
self.assertAllEqual(expected_output, output_data)
-@keras_parameterized.run_all_keras_modes
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
class IndexLookupOutputTest(keras_parameterized.TestCase,
preprocessing_test_utils.PreprocessingLayerTest):
@@ -287,7 +280,7 @@
self.assertAllEqual(expected_output, output_data)
-@keras_parameterized.run_all_keras_modes
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
class StaticIndexLookupOutputTest(
keras_parameterized.TestCase,
preprocessing_test_utils.PreprocessingLayerTest):
@@ -331,7 +324,7 @@
self.assertAllEqual(expected_output, output_data)
-@keras_parameterized.run_all_keras_modes
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
class CategoricalEncodingStaticInputTest(
keras_parameterized.TestCase,
preprocessing_test_utils.PreprocessingLayerTest):
diff --git a/tensorflow/python/keras/layers/preprocessing/text_vectorization.py b/tensorflow/python/keras/layers/preprocessing/text_vectorization.py
index 41a8a4c..d980e1d 100644
--- a/tensorflow/python/keras/layers/preprocessing/text_vectorization.py
+++ b/tensorflow/python/keras/layers/preprocessing/text_vectorization.py
@@ -207,21 +207,6 @@
This example instantiates a TextVectorization layer by passing a list
of vocabulary terms to the layer's __init__ method.
- input_array = np.array([["earth", "wind", "and", "fire"],
- ["fire", "and", "earth", "michigan"]])
- expected_output = [[2, 3, 4, 5], [5, 4, 2, 1]]
-
- input_data = keras.Input(shape=(None,), dtype=dtypes.string)
- layer = get_layer_class()(
- max_tokens=None,
- standardize=None,
- split=None,
- output_mode=text_vectorization.INT,
- vocabulary=vocab_data)
- int_data = layer(input_data)
- model = keras.Model(inputs=input_data, outputs=int_data)
-
- output_dataset = model.predict(input_array)
>>> vocab_data = ["earth", "wind", "and", "fire"]
>>> max_len = 4 # Sequence length to pad the outputs to.
>>>
@@ -332,17 +317,13 @@
base_preprocessing_layer.keras_kpl_gauge.get_cell(
"TextVectorization").set(True)
- self._index_lookup_layer = self._get_index_lookup_class()(
+ self._index_lookup_layer = string_lookup.StringLookup(
max_tokens=max_tokens,
vocabulary=vocabulary,
pad_to_max_tokens=pad_to_max_tokens,
output_mode=output_mode if output_mode is not None else INT,
vocabulary_size=vocabulary_size)
- def _get_index_lookup_class(self):
- return string_lookup.StringLookup
- # End of V1/V2 shim points.
-
def _assert_same_type(self, expected_type, values, value_name):
if dtypes.as_dtype(expected_type) != dtypes.as_dtype(values.dtype):
raise RuntimeError("Expected %s type %s, got %s" %
