test/python/dummy_tokenizer.py

   1 """
   2 Tokenizer for testing.
   3 """
   4
   5 def create(dsn, data_dir):
   6     """ Create a new instance of the tokenizer provided by this module.
   7     """
   8     return DummyTokenizer(dsn, data_dir)
   9
  10 class DummyTokenizer:
  11
  12     def __init__(self, dsn, data_dir):
  13         self.dsn = dsn
  14         self.data_dir = data_dir
  15         self.init_state = None
  16         self.analyser_cache = {}
  17
  18
  19     def init_new_db(self, *args, **kwargs):
  20         assert self.init_state == None
  21         self.init_state = "new"
  22
  23
  24     def init_from_project(self):
  25         assert self.init_state == None
  26         self.init_state = "loaded"
  27
  28
  29     def name_analyzer(self):
  30         return DummyNameAnalyzer(self.analyser_cache)
  31
  32
  33 class DummyNameAnalyzer:
  34
  35     def __enter__(self):
  36         return self
  37
  38     def __exit__(self, exc_type, exc_value, traceback):
  39         self.close()
  40
  41
  42     def __init__(self, cache):
  43         self.analyser_cache = cache
  44         cache['countries'] = []
  45
  46
  47     def close(self):
  48         pass
  49
  50     def add_postcodes_from_db(self):
  51         pass
  52
  53     def update_special_phrases(self, phrases):
  54         self.analyser_cache['special_phrases'] = phrases
  55
  56     def add_country_names(self, code, names):
  57         self.analyser_cache['countries'].append((code, names))
  58
  59     def process_place(self, place):
  60         return {}