二、分字处理及文本清洗

  该部分涉及tokenization.py

1. CharacterRecognition类

  该类中存放一些字符识别函数, 判断是否为whitespace、control、punctuation、chinese_char。

class CharacterRecognition:@staticmethoddef is_whitespace(char):"""\t, \n, \r 通通认为是 whitespace. """if char == " " or char == "\t" or char == "\n" or char == "\r":return Truecat = unicodedata.category(char)if cat == "Zs":return Truereturn False@staticmethoddef is_control(char):"""控制符 """if char == "\t" or char == "\n" or char == "\r":return Falsecat = unicodedata.category(char)if cat in ("Cc", "Cf"):return Truereturn False@staticmethoddef is_punctuation(char):"""所有非 字母/数字 ASCII都看作是 标点符号"""# 包括 "^", "$", "`"cp = ord(char)if ((33 <= cp <= 47) or (58 <= cp <= 64) or(91 <= cp <= 96) or (123 <= cp <= 126)):return Truecat = unicodedata.category(char)if cat.startswith("P"):return Truereturn False@staticmethoddef is_chinese_char(char):cp = ord(char)if ((0x4E00 <= cp <= 0x9FFF) or(0x3400 <= cp <= 0x4DBF) or(0x20000 <= cp <= 0x2A6DF) or(0x2A700 <= cp <= 0x2B73F) or(0x2B740 <= cp <= 0x2B81F) or(0x2B820 <= cp <= 0x2CEAF) or(0xF900 <= cp <= 0xFAFF) or(0x2F800 <= cp <= 0x2FA1F)):return Truereturn False

2.BasicTokenizer类

  该类主要包含以下功能：

• 文本清洗
    移除无效字符； whitespace替换；
• 中文处理
    对于文本中的每个中文字符，前后都添加一个空格。
• 按whitespace分词
    这样分词后，还可能存在"hello," 这种情况
• 按punctuation分词
• " ".join()后再进行一次whitespace分词

  最后结果是list[char]

class BasicTokenizer(object):def __init__(self, do_lower_case=True):"""文本中的英文是否需要小写化"""self.character_recognition = CharacterRecognition()self.do_lower_case = do_lower_casedef tokenizer(self, text):"""文本的清洗、数据处理, 以及分词"""text = self._clean_text(text)text = self._tokenize_chinese_chars(text)orig_tokens = self._whitespace_tokenize(text)split_tokens = []for token in orig_tokens:if self.do_lower_case:token = token.lower()split_tokens.extend(self._run_split_on_punc(token))output_tokens = self._whitespace_tokenize(" ".join(split_tokens))return output_tokensdef _run_split_on_punc(self, text):"""切分 小片段文本中的标点符号, 可能有 "hello,"这种 """chars = list(text)i = 0start_new_word = Trueoutput = []while i < len(chars):char = chars[i]if self.character_recognition.is_whitespace(char):output.append([char])start_new_word = Trueelse:if start_new_word:output.append([])start_new_word = Falseoutput[-1].append(char)i += 1return ["".join(x) for x in output]@classmethoddef _whitespace_tokenize(cls, text):"""空格处理, 移除多余的, 并分词"""text = text.strip()if not text:return []tokens = text.split()return tokensdef _tokenize_chinese_chars(self, text):"""中文处理, 每个中文周围加一个 分隔符."""output = []for char in text:if self.character_recognition.is_chinese_char(char):output.append(" ")output.append(char)output.append(" ")else:output.append(char)return "".join(output)# _开头的, 仅仅是概念上的私有。def _clean_text(self, text):"""移除无效字符, whitespace替换"""output = []for char in text:cp = ord(char)if cp == 0 or cp == 0xfffd or \self.character_recognition.is_control(char):continueif self.character_recognition.is_whitespace(char):output.append(" ")else:output.append(char)return "".join(output)

3. FullTokenizer类

  该类主要包含以下功能：

• 建立词表
  word2id
• 建立反词表
  id2word
• 分词
  调用BasicTokenizer, 返回list[char]

class FullTokenizer(object):"""端到端的 tokenizer. """def __init__(self, vocab_file, do_lower_case=True):self.vocab = load_vocab(vocab_file)# inv_vocab 是 vocab的反转. {id:word, id:word}self.inv_vocab = {v: k for k, v in self.vocab.items()}# 数据清洗, 以及分词处理self.basic_tokenizer = BasicTokenizer()def tokenize(self, text):split_tokens = []for token in self.basic_tokenizer.tokenizer(text):split_tokens.append(token)return split_tokensdef convert_tokens_to_ids(self, tokens):return convert_by_vocab(self.vocab, tokens)def convert_ids_to_tokens(self, ids):return convert_by_vocab(self.inv_vocab, ids)

4.通用函数

a、加载词表

遇到空行, 结束加载。

def load_vocab(vocab_file):vocab = collections.OrderedDict()index = 0with tf.gfile.GFile(vocab_file, "r") as reader:while True:if not token:# 遇到空行, 或结尾.breaktoken = reader.readline()token = token.strip()vocab[token] = indexindex += 1return vocab

b、利用字典转换.

被token2id和id2token调用.

def convert_by_vocab(vocab, items):# 被 convert_tokens_to_ids 和 convert_ids_to_tokens调用.output = []for item in items:output.append(vocab[item])return output

c、token2id.

def convert_tokens_to_ids(vocab, tokens):return convert_by_vocab(vocab, tokens)

d、id2token.

def convert_ids_to_tokens(inv_vocab, ids):return convert_by_vocab(inv_vocab, ids)