Merge remote-tracking branch 'upstream/master'

[nominatim.git] / nominatim / clicmd / setup.py

diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py
index 8bab6f3896f8826770a01bbb6ea2409b20c2ec0a..73095468ab2fe30b4e50b9d2b889346030a1a92c 100644 (file)
--- a/nominatim/clicmd/setup.py
+++ b/nominatim/clicmd/setup.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Implementation of the 'import' subcommand.
 """
 """
 Implementation of the 'import' subcommand.
 """


@@ -8,18 +14,22 @@ import psutil
 
 from nominatim.db.connection import connect
 from nominatim.db import status, properties
 
 from nominatim.db.connection import connect
 from nominatim.db import status, properties

-from nominatim.version import NOMINATIM_VERSION
+from nominatim.version import version_str

 
 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111
 # Using non-top-level imports to avoid eventually unused imports.
 
 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111
 # Using non-top-level imports to avoid eventually unused imports.

-# pylint: disable=E0012,C0415
+# pylint: disable=C0415

 
 LOG = logging.getLogger()
 
 class SetupAll:
     """\
     Create a new Nominatim database from an OSM file.
 
 LOG = logging.getLogger()
 
 class SetupAll:
     """\
     Create a new Nominatim database from an OSM file.

+
+    This sub-command sets up a new Nominatim database from scratch starting
+    with creating a new database in Postgresql. The user running this command
+    needs superuser rights on the database.

     """
 
     @staticmethod
     """
 
     @staticmethod


@@ -28,7 +38,7 @@ class SetupAll:
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file', metavar='FILE', action='append',
                            help='OSM file to be imported'
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file', metavar='FILE', action='append',
                            help='OSM file to be imported'

-                                ' (repeat for importing multiple files.')
+                                ' (repeat for importing multiple files)')

         group.add_argument('--continue', dest='continue_at',
                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group.add_argument('--continue', dest='continue_at',
                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')


@@ -43,26 +53,35 @@ class SetupAll:
         group.add_argument('--no-updates', action='store_true',
                            help="Do not keep tables that are only needed for "
                                 "updating the database later")
         group.add_argument('--no-updates', action='store_true',
                            help="Do not keep tables that are only needed for "
                                 "updating the database later")

+        group.add_argument('--offline', action='store_true',
+                           help="Do not attempt to load any additional data from the internet")

         group = parser.add_argument_group('Expert options')
         group.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')
         group.add_argument('--index-noanalyse', action='store_true',
         group = parser.add_argument_group('Expert options')
         group.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')
         group.add_argument('--index-noanalyse', action='store_true',

-                           help='Do not perform analyse operations during index')
+                           help='Do not perform analyse operations during index (expert only)')

 
 
     @staticmethod
 
 
     @staticmethod

-    def run(args):
+    def run(args): # pylint: disable=too-many-statements
+        from ..data import country_info

         from ..tools import database_import, refresh, postcodes, freeze
         from ..indexer.indexer import Indexer
 
         from ..tools import database_import, refresh, postcodes, freeze
         from ..indexer.indexer import Indexer
 

+        country_info.setup_country_config(args.config)
+

         if args.continue_at is None:
             files = args.get_osm_file_list()
 
         if args.continue_at is None:
             files = args.get_osm_file_list()
 

+            LOG.warning('Creating database')

             database_import.setup_database_skeleton(args.config.get_libpq_dsn(),
             database_import.setup_database_skeleton(args.config.get_libpq_dsn(),

-                                                    args.data_dir,
-                                                    args.no_partitions,

                                                     rouser=args.config.DATABASE_WEBUSER)
 
                                                     rouser=args.config.DATABASE_WEBUSER)
 

+            LOG.warning('Setting up country tables')
+            country_info.setup_country_tables(args.config.get_libpq_dsn(),
+                                              args.data_dir,
+                                              args.no_partitions)
+

             LOG.warning('Importing OSM data file')
             database_import.import_osm_data(files,
                                             args.osm2pgsql_options(0, 1),
             LOG.warning('Importing OSM data file')
             database_import.import_osm_data(files,
                                             args.osm2pgsql_options(0, 1),


@@ -109,23 +128,21 @@ class SetupAll:
             database_import.create_search_indices(conn, args.config,
                                                   drop=args.no_updates)
             LOG.warning('Create search index for default country names.')
             database_import.create_search_indices(conn, args.config,
                                                   drop=args.no_updates)
             LOG.warning('Create search index for default country names.')

-            database_import.create_country_names(conn, tokenizer,
-                                                 args.config.LANGUAGES)
-            conn.commit()
+            country_info.create_country_names(conn, tokenizer,
+                                              args.config.get_str_list('LANGUAGES'))

             if args.no_updates:
                 freeze.drop_update_tables(conn)
         tokenizer.finalize_import(args.config)
 
             if args.no_updates:
                 freeze.drop_update_tables(conn)
         tokenizer.finalize_import(args.config)
 

+        LOG.warning('Recompute word counts')
+        tokenizer.update_statistics()

 
         webdir = args.project_dir / 'website'
         LOG.warning('Setup website at %s', webdir)
         with connect(args.config.get_libpq_dsn()) as conn:
             refresh.setup_website(webdir, args.config, conn)
 
 
         webdir = args.project_dir / 'website'
         LOG.warning('Setup website at %s', webdir)
         with connect(args.config.get_libpq_dsn()) as conn:
             refresh.setup_website(webdir, args.config, conn)
 

-        with connect(args.config.get_libpq_dsn()) as conn:
-            SetupAll._set_database_date(conn)
-            properties.set_property(conn, 'database_version',
-                                    '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION))
+        SetupAll._finalize_database(args.config.get_libpq_dsn(), args.offline)

 
         return 0
 
 
         return 0
 


@@ -141,7 +158,7 @@ class SetupAll:
             refresh.create_functions(conn, config, False, False)
             LOG.warning('Create tables')
             database_import.create_tables(conn, config, reverse_only=reverse_only)
             refresh.create_functions(conn, config, False, False)
             LOG.warning('Create tables')
             database_import.create_tables(conn, config, reverse_only=reverse_only)

-            refresh.load_address_levels_from_file(conn, Path(config.ADDRESS_LEVEL_CONFIG))
+            refresh.load_address_levels_from_config(conn, config)

             LOG.warning('Create functions (2nd pass)')
             refresh.create_functions(conn, config, False, False)
             LOG.warning('Create table triggers')
             LOG.warning('Create functions (2nd pass)')
             refresh.create_functions(conn, config, False, False)
             LOG.warning('Create table triggers')


@@ -180,20 +197,24 @@ class SetupAll:
             LOG.warning('Creating support index')
             if tablespace:
                 tablespace = 'TABLESPACE ' + tablespace
             LOG.warning('Creating support index')
             if tablespace:
                 tablespace = 'TABLESPACE ' + tablespace

-            cur.execute("""CREATE INDEX idx_placex_pendingsector
-                           ON placex USING BTREE (rank_address,geometry_sector)
-                           {} WHERE indexed_status > 0
-                        """.format(tablespace))
+            cur.execute(f"""CREATE INDEX idx_placex_pendingsector
+                            ON placex USING BTREE (rank_address,geometry_sector)
+                            {tablespace} WHERE indexed_status > 0
+                         """)

         conn.commit()
 
 
     @staticmethod
         conn.commit()
 
 
     @staticmethod

-    def _set_database_date(conn):
+    def _finalize_database(dsn, offline):

         """ Determine the database date and set the status accordingly.
         """
         """ Determine the database date and set the status accordingly.
         """

-        try:
-            dbdate = status.compute_database_date(conn)
-            status.set_status(conn, dbdate)
-            LOG.info('Database is at %s.', dbdate)
-        except Exception as exc: # pylint: disable=broad-except
-            LOG.error('Cannot determine date of database: %s', exc)
+        with connect(dsn) as conn:
+            if not offline:
+                try:
+                    dbdate = status.compute_database_date(conn)
+                    status.set_status(conn, dbdate)
+                    LOG.info('Database is at %s.', dbdate)
+                except Exception as exc: # pylint: disable=broad-except
+                    LOG.error('Cannot determine date of database: %s', exc)
+
+            properties.set_property(conn, 'database_version', version_str())