test/python/dummy_tokenizer.py

   1 """
   2 Tokenizer for testing.
   3 """
   4
   5 def create(dsn, data_dir):
   6     """ Create a new instance of the tokenizer provided by this module.
   7     """
   8     return DummyTokenizer(dsn, data_dir)
   9
  10 class DummyTokenizer:
  11
  12     def __init__(self, dsn, data_dir):
  13         self.dsn = dsn
  14         self.data_dir = data_dir
  15         self.init_state = None
  16         self.analyser_cache = {}
  17
  18
  19     def init_new_db(self, *args, **kwargs):
  20         assert self.init_state == None
  21         self.init_state = "new"
  22
  23
  24     def init_from_project(self):
  25         assert self.init_state == None
  26         self.init_state = "loaded"
  27
  28
  29     def finalize_import(self, _):
  30         pass
  31
  32
  33     def name_analyzer(self):
  34         return DummyNameAnalyzer(self.analyser_cache)
  35
  36
  37 class DummyNameAnalyzer:
  38
  39     def __enter__(self):
  40         return self
  41
  42     def __exit__(self, exc_type, exc_value, traceback):
  43         self.close()
  44
  45
  46     def __init__(self, cache):
  47         self.analyser_cache = cache
  48         cache['countries'] = []
  49
  50
  51     def close(self):
  52         pass
  53
  54     def normalize_postcode(self, postcode):
  55         return postcode
  56
  57     def update_postcodes_from_db(self):
  58         pass
  59
  60     def update_special_phrases(self, phrases, should_replace):
  61         self.analyser_cache['special_phrases'] = phrases
  62
  63     def add_country_names(self, code, names):
  64         self.analyser_cache['countries'].append((code, names))
  65
  66     def process_place(self, place):
  67         return {}