tensorflow2.0 中英翻译 对文本的预处理（文本转换为文本向量 tfds.features.text.SubwordT...

from __future__ import absolute_import, division, print_function, unicode_literals import tensorflow_datasets as tfds import tensorflow as tf def encode(lang1, lang2): lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode( lang1.numpy()) + [tokenizer_pt.vocab_size+1] lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode( lang2.numpy()) + [tokenizer_en.vocab_size+1] return lang1, lang2 MAX_LENGTH = 40 def filter_long_sent(x, y, max_length=MAX_LENGTH): return tf.logical_and(tf.size(x) <= max_length, tf.size(y) <= max_length) def tf_encode(pt, en): return tf.py_function(encode, [pt, en], [tf.int64, tf.int64]) BUFFER_SIZE = 20000 BATCH_SIZE = 64 examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True, as_supervised=True) train_examples, val_examples = examples['train'], examples['validation'] #tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus( # (en.numpy() for pt, en in train_examples), target_vocab_size=2 ** 13) #tokenizer_pt = tfds.features.text.SubwordTextEncoder.build_from_corpus( # (pt.numpy() for pt, en in train_examples), target_vocab_size=2 ** 13) tokenizer_en= tfds.features.text.SubwordTextEncoder.load_from_file('./vocab_filename') tokenizer_pt = tfds.features.text.SubwordTextEncoder.load_from_file('./vocab') # 使用.map()运行相关图操作 train_dataset = train_examples.map(tf_encode) # 过滤过长的数据 train_dataset = train_dataset.filter(filter_long_sent) # 使用缓存数据加速读入 train_dataset = train_dataset.cache() # 打乱并获取批数据 train_dataset = train_dataset.padded_batch(BATCH_SIZE, padded_shapes=([40], [40])) # 填充为最大长度-90 # 设置预取数据 train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE) # 验证集数据 val_dataset = val_examples.map(tf_encode) val_dataset = val_dataset.filter(filter_long_sent).padded_batch( BATCH_SIZE, padded_shapes=([40], [40]))