document new status fields

[nominatim.git] / nominatim / tools / refresh.py

diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py
index f09c0cede2cc62e8603fc905501a4b71469f2259..5cfa1ab00e92a46d941bd1c55a3d7effbf6d38f1 100644 (file)
--- a/nominatim/tools/refresh.py
+++ b/nominatim/tools/refresh.py
@@ -9,20 +9,21 @@ from textwrap import dedent
 from psycopg2.extras import execute_values
 
 from ..db.utils import execute_file
+from ..version import NOMINATIM_VERSION
 
 LOG = logging.getLogger()
 
-def update_postcodes(conn, sql_dir):
+def update_postcodes(dsn, sql_dir):
     """ Recalculate postcode centroids and add, remove and update entries in the
         location_postcode table. `conn` is an opne connection to the database.
     """
-    execute_file(conn, sql_dir / 'update-postcodes.sql')
+    execute_file(dsn, sql_dir / 'update-postcodes.sql')
 
 
-def recompute_word_counts(conn, sql_dir):
+def recompute_word_counts(dsn, sql_dir):
     """ Compute the frequency of full-word search terms.
     """
-    execute_file(conn, sql_dir / 'words_from_search_name.sql')
+    execute_file(dsn, sql_dir / 'words_from_search_name.sql')
 
 
 def _add_address_level_rows_from_entry(rows, entry):
@@ -200,6 +201,53 @@ PHP_CONST_DEFS = (
 )
 
 
+def import_wikipedia_articles(dsn, data_path, ignore_errors=False):
+    """ Replaces the wikipedia importance tables with new data.
+        The import is run in a single transaction so that the new data
+        is replace seemlessly.
+
+        Returns 0 if all was well and 1 if the importance file could not
+        be found. Throws an exception if there was an error reading the file.
+    """
+    datafile = data_path / 'wikimedia-importance.sql.gz'
+
+    if not datafile.exists():
+        return 1
+
+    pre_code = """BEGIN;
+                  DROP TABLE IF EXISTS "wikipedia_article";
+                  DROP TABLE IF EXISTS "wikipedia_redirect"
+               """
+    post_code = "COMMIT"
+    execute_file(dsn, datafile, ignore_errors=ignore_errors,
+                 pre_code=pre_code, post_code=post_code)
+
+    return 0
+
+
+def recompute_importance(conn):
+    """ Recompute wikipedia links and importance for all entries in placex.
+        This is a long-running operations that must not be executed in
+        parallel with updates.
+    """
+    with conn.cursor() as cur:
+        cur.execute('ALTER TABLE placex DISABLE TRIGGER ALL')
+        cur.execute("""
+            UPDATE placex SET (wikipedia, importance) =
+               (SELECT wikipedia, importance
+                FROM compute_importance(extratags, country_code, osm_type, osm_id))
+            """)
+        cur.execute("""
+            UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
+             FROM placex d
+             WHERE s.place_id = d.linked_place_id and d.wikipedia is not null
+                   and (s.wikipedia is null or s.importance < d.importance);
+            """)
+
+        cur.execute('ALTER TABLE placex ENABLE TRIGGER ALL')
+    conn.commit()
+
+
 def setup_website(basedir, phplib_dir, config):
     """ Create the website script stubs.
     """
@@ -211,9 +259,10 @@ def setup_website(basedir, phplib_dir, config):
                       <?php
 
                       @define('CONST_Debug', $_GET['debug'] ?? false);
-                      @define('CONST_LibDir', '{}');
+                      @define('CONST_LibDir', '{0}');
+                      @define('CONST_NominatimVersion', '{1[0]}.{1[1]}.{1[2]}-{1[3]}');
 
-                      """.format(phplib_dir))
+                      """.format(phplib_dir, NOMINATIM_VERSION))
 
     for php_name, conf_name, var_type in PHP_CONST_DEFS:
         if var_type == bool: