X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/7c66fef63f5066674e0e76d378c8ace2754ee54c..6e688a01138dd18c643cf3a5e3541506d9f63b20:/nominatim/api/logging.py?ds=sidebyside diff --git a/nominatim/api/logging.py b/nominatim/api/logging.py index 6c8b1b38..30999a3f 100644 --- a/nominatim/api/logging.py +++ b/nominatim/api/logging.py @@ -7,11 +7,13 @@ """ Functions for specialised logging with HTML output. """ -from typing import Any, Iterator, Optional, List, Tuple, cast +from typing import Any, Iterator, Optional, List, Tuple, cast, Union, Mapping, Sequence from contextvars import ContextVar import datetime as dt import textwrap import io +import re +import html import sqlalchemy as sa from sqlalchemy.ext.asyncio import AsyncConnection @@ -74,23 +76,57 @@ class BaseLogger: """ - def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None: + def sql(self, conn: AsyncConnection, statement: 'sa.Executable', + params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None]) -> None: """ Print the SQL for the given statement. """ - def format_sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> str: - """ Return the comiled version of the statement. + def format_sql(self, conn: AsyncConnection, statement: 'sa.Executable', + extra_params: Union[Mapping[str, Any], + Sequence[Mapping[str, Any]], None]) -> str: + """ Return the compiled version of the statement. """ - try: - return str(cast('sa.ClauseElement', statement) - .compile(conn.sync_engine, compile_kwargs={"literal_binds": True})) - except sa.exc.CompileError: - pass - except NotImplementedError: - pass - - return str(cast('sa.ClauseElement', statement).compile(conn.sync_engine)) - + compiled = cast('sa.ClauseElement', statement).compile(conn.sync_engine) + + params = dict(compiled.params) + if isinstance(extra_params, Mapping): + for k, v in extra_params.items(): + if hasattr(v, 'to_wkt'): + params[k] = v.to_wkt() + elif isinstance(v, (int, float)): + params[k] = v + else: + params[k] = str(v) + elif isinstance(extra_params, Sequence) and extra_params: + for k in extra_params[0]: + params[k] = f':{k}' + + sqlstr = str(compiled) + + if conn.dialect.name == 'postgresql': + if sa.__version__.startswith('1'): + try: + sqlstr = re.sub(r'__\[POSTCOMPILE_[^]]*\]', '%s', sqlstr) + return sqlstr % tuple((repr(params.get(name, None)) + for name in compiled.positiontup)) # type: ignore + except TypeError: + return sqlstr + + # Fixes an odd issue with Python 3.7 where percentages are not + # quoted correctly. + sqlstr = re.sub(r'%(?!\()', '%%', sqlstr) + sqlstr = re.sub(r'__\[POSTCOMPILE_([^]]*)\]', r'%(\1)s', sqlstr) + return sqlstr % params + + assert conn.dialect.name == 'sqlite' + + # params in positional order + pparams = (repr(params.get(name, None)) for name in compiled.positiontup) # type: ignore + + sqlstr = re.sub(r'__\[POSTCOMPILE_([^]]*)\]', '?', sqlstr) + sqlstr = re.sub(r"\?", lambda m: next(pparams), sqlstr) + + return sqlstr class HTMLLogger(BaseLogger): """ Logger that formats messages in HTML. @@ -183,15 +219,16 @@ class HTMLLogger(BaseLogger): self._write(f'TOTAL: {total}
') - def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None: + def sql(self, conn: AsyncConnection, statement: 'sa.Executable', + params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None]) -> None: self._timestamp() - sqlstr = self.format_sql(conn, statement) + sqlstr = self.format_sql(conn, statement, params) if CODE_HIGHLIGHT: sqlstr = highlight(sqlstr, PostgresLexer(), HtmlFormatter(nowrap=True, lineseparator='{sqlstr}
{sqlstr}
')
+ self._write(f'{html.escape(sqlstr)}
')
def _python_var(self, var: Any) -> str:
@@ -199,7 +236,7 @@ class HTMLLogger(BaseLogger):
fmt = highlight(str(var), PythonLexer(), HtmlFormatter(nowrap=True))
return f'{fmt}
{str(var)}
'
+ return f'{html.escape(str(var))}
'
def _write(self, text: str) -> None:
@@ -215,6 +252,10 @@ class TextLogger(BaseLogger):
self.buffer = io.StringIO()
+ def _timestamp(self) -> None:
+ self._write(f'[{dt.datetime.now()}]\n')
+
+
def get_buffer(self) -> str:
return self.buffer.getvalue()
@@ -227,6 +268,7 @@ class TextLogger(BaseLogger):
def section(self, heading: str) -> None:
+ self._timestamp()
self._write(f"\n# {heading}\n\n")
@@ -263,6 +305,7 @@ class TextLogger(BaseLogger):
def result_dump(self, heading: str, results: Iterator[Tuple[Any, Any]]) -> None:
+ self._timestamp()
self._write(f'{heading}:\n')
total = 0
for rank, res in results:
@@ -276,8 +319,10 @@ class TextLogger(BaseLogger):
self._write(f'TOTAL: {total}\n\n')
- def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None:
- sqlstr = '\n| '.join(textwrap.wrap(self.format_sql(conn, statement), width=78))
+ def sql(self, conn: AsyncConnection, statement: 'sa.Executable',
+ params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None]) -> None:
+ self._timestamp()
+ sqlstr = '\n| '.join(textwrap.wrap(self.format_sql(conn, statement, params), width=78))
self._write(f"| {sqlstr}\n\n")