nominatim/clicmd/setup.py

   1 """
   2 Implementation of the 'import' subcommand.
   3 """
   4 import logging
   5 from pathlib import Path
   6
   7 import psutil
   8
   9 from ..tools.exec_utils import run_legacy_script
  10 from ..db.connection import connect
  11 from ..db import status, properties
  12 from ..version import NOMINATIM_VERSION
  13 from ..errors import UsageError
  14
  15 # Do not repeat documentation of subcommand classes.
  16 # pylint: disable=C0111
  17 # Using non-top-level imports to avoid eventually unused imports.
  18 # pylint: disable=E0012,C0415
  19
  20 LOG = logging.getLogger()
  21
  22 class SetupAll:
  23     """\
  24     Create a new Nominatim database from an OSM file.
  25     """
  26
  27     @staticmethod
  28     def add_args(parser):
  29         group_name = parser.add_argument_group('Required arguments')
  30         group = group_name.add_mutually_exclusive_group(required=True)
  31         group.add_argument('--osm-file', metavar='FILE',
  32                            help='OSM file to be imported.')
  33         group.add_argument('--continue', dest='continue_at',
  34                            choices=['load-data', 'indexing', 'db-postprocess'],
  35                            help='Continue an import that was interrupted')
  36         group = parser.add_argument_group('Optional arguments')
  37         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
  38                            help='Size of cache to be used by osm2pgsql (in MB)')
  39         group.add_argument('--reverse-only', action='store_true',
  40                            help='Do not create tables and indexes for searching')
  41         group.add_argument('--no-partitions', action='store_true',
  42                            help="""Do not partition search indices
  43                                    (speeds up import of single country extracts)""")
  44         group.add_argument('--no-updates', action='store_true',
  45                            help="""Do not keep tables that are only needed for
  46                                    updating the database later""")
  47         group = parser.add_argument_group('Expert options')
  48         group.add_argument('--ignore-errors', action='store_true',
  49                            help='Continue import even when errors in SQL are present')
  50         group.add_argument('--index-noanalyse', action='store_true',
  51                            help='Do not perform analyse operations during index')
  52
  53
  54     @staticmethod
  55     def run(args): # pylint: disable=too-many-statements
  56         from ..tools import database_import
  57         from ..tools import refresh
  58         from ..indexer.indexer import Indexer
  59
  60         if args.osm_file and not Path(args.osm_file).is_file():
  61             LOG.fatal("OSM file '%s' does not exist.", args.osm_file)
  62             raise UsageError('Cannot access file.')
  63
  64         if args.continue_at is None:
  65             database_import.setup_database_skeleton(args.config.get_libpq_dsn(),
  66                                                     args.data_dir,
  67                                                     args.no_partitions,
  68                                                     rouser=args.config.DATABASE_WEBUSER)
  69
  70             LOG.warning('Installing database module')
  71             with connect(args.config.get_libpq_dsn()) as conn:
  72                 database_import.install_module(args.module_dir, args.project_dir,
  73                                                args.config.DATABASE_MODULE_PATH,
  74                                                conn=conn)
  75
  76             LOG.warning('Importing OSM data file')
  77             database_import.import_osm_data(Path(args.osm_file),
  78                                             args.osm2pgsql_options(0, 1),
  79                                             drop=args.no_updates,
  80                                             ignore_errors=args.ignore_errors)
  81
  82             with connect(args.config.get_libpq_dsn()) as conn:
  83                 LOG.warning('Create functions (1st pass)')
  84                 refresh.create_functions(conn, args.config, args.sqllib_dir,
  85                                          False, False)
  86                 LOG.warning('Create tables')
  87                 database_import.create_tables(conn, args.config, args.sqllib_dir,
  88                                               reverse_only=args.reverse_only)
  89                 refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
  90                 LOG.warning('Create functions (2nd pass)')
  91                 refresh.create_functions(conn, args.config, args.sqllib_dir,
  92                                          False, False)
  93                 LOG.warning('Create table triggers')
  94                 database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
  95                 LOG.warning('Create partition tables')
  96                 database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
  97                 LOG.warning('Create functions (3rd pass)')
  98                 refresh.create_functions(conn, args.config, args.sqllib_dir,
  99                                          False, False)
 100
 101             LOG.warning('Importing wikipedia importance data')
 102             data_path = Path(args.config.WIKIPEDIA_DATA_PATH or args.project_dir)
 103             if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
 104                                                  data_path) > 0:
 105                 LOG.error('Wikipedia importance dump file not found. '
 106                           'Will be using default importances.')
 107
 108             LOG.warning('Initialise tables')
 109             with connect(args.config.get_libpq_dsn()) as conn:
 110                 database_import.truncate_data_tables(conn, args.config.MAX_WORD_FREQUENCY)
 111
 112         if args.continue_at is None or args.continue_at == 'load-data':
 113             LOG.warning('Load data into placex table')
 114             database_import.load_data(args.config.get_libpq_dsn(),
 115                                       args.data_dir,
 116                                       args.threads or psutil.cpu_count() or 1)
 117
 118             LOG.warning('Calculate postcodes')
 119             run_legacy_script('setup.php', '--calculate-postcodes',
 120                               nominatim_env=args, throw_on_fail=not args.ignore_errors)
 121
 122         if args.continue_at is None or args.continue_at in ('load-data', 'indexing'):
 123             LOG.warning('Indexing places')
 124             indexer = Indexer(args.config.get_libpq_dsn(),
 125                               args.threads or psutil.cpu_count() or 1)
 126             indexer.index_full(analyse=not args.index_noanalyse)
 127
 128         LOG.warning('Post-process tables')
 129         with connect(args.config.get_libpq_dsn()) as conn:
 130             database_import.create_search_indices(conn, args.config,
 131                                                   args.sqllib_dir,
 132                                                   drop=args.no_updates)
 133         run_legacy_script('setup.php', '--create-country-names',
 134                           nominatim_env=args, throw_on_fail=not args.ignore_errors)
 135
 136         webdir = args.project_dir / 'website'
 137         LOG.warning('Setup website at %s', webdir)
 138         refresh.setup_website(webdir, args.phplib_dir, args.config)
 139
 140         with connect(args.config.get_libpq_dsn()) as conn:
 141             try:
 142                 dbdate = status.compute_database_date(conn)
 143                 status.set_status(conn, dbdate)
 144                 LOG.info('Database is at %s.', dbdate)
 145             except Exception as exc: # pylint: disable=broad-except
 146                 LOG.error('Cannot determine date of database: %s', exc)
 147
 148             properties.set_property(conn, 'database_version',
 149                                     '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION))
 150
 151         return 0