test/python/dummy_tokenizer.py

   1 """
   2 Tokenizer for testing.
   3 """
   4
   5 def create(dsn, data_dir):
   6     """ Create a new instance of the tokenizer provided by this module.
   7     """
   8     return DummyTokenizer(dsn, data_dir)
   9
  10 class DummyTokenizer:
  11
  12     def __init__(self, dsn, data_dir):
  13         self.dsn = dsn
  14         self.data_dir = data_dir
  15         self.init_state = None
  16         self.analyser_cache = {}
  17
  18
  19     def init_new_db(self, *args, **kwargs):
  20         assert self.init_state is None
  21         self.init_state = "new"
  22
  23
  24     def init_from_project(self):
  25         assert self.init_state is None
  26         self.init_state = "loaded"
  27
  28
  29     @staticmethod
  30     def finalize_import(_):
  31         pass
  32
  33
  34     def name_analyzer(self):
  35         return DummyNameAnalyzer(self.analyser_cache)
  36
  37
  38 class DummyNameAnalyzer:
  39
  40     def __enter__(self):
  41         return self
  42
  43     def __exit__(self, exc_type, exc_value, traceback):
  44         self.close()
  45
  46
  47     def __init__(self, cache):
  48         self.analyser_cache = cache
  49         cache['countries'] = []
  50
  51
  52     def close(self):
  53         pass
  54
  55     @staticmethod
  56     def normalize_postcode(postcode):
  57         return postcode
  58
  59     @staticmethod
  60     def update_postcodes_from_db():
  61         pass
  62
  63     def update_special_phrases(self, phrases, should_replace):
  64         self.analyser_cache['special_phrases'] = phrases
  65
  66     def add_country_names(self, code, names):
  67         self.analyser_cache['countries'].append((code, names))
  68
  69     @staticmethod
  70     def process_place(place):
  71         return {}