dddddd', wkb[17:65])\n else:\n raise ValueError(\"WKB has wrong header\")\n\n return Bbox(min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2))\n\n @staticmethod\n def from_point(pt: Point, buffer: float) -> 'Bbox':\n \"\"\" Return a Bbox around the point with the buffer added to all sides.\n \"\"\"\n return Bbox(pt[0] - buffer, pt[1] - buffer,\n pt[0] + buffer, pt[1] + buffer)\n\n @staticmethod\n def from_param(inp: Any) -> 'Bbox':\n \"\"\" Return a Bbox from an input parameter. The box may be\n given as a Bbox, a string or a list or strings or integer.\n Raises a UsageError if the format is incorrect.\n \"\"\"\n if isinstance(inp, Bbox):\n return inp\n\n seq: Sequence[str]\n if isinstance(inp, str):\n seq = inp.split(',')\n elif isinstance(inp, abc.Sequence):\n seq = inp\n\n if len(seq) != 4:\n raise UsageError('Bounding box parameter needs 4 coordinates.')\n try:\n x1, y1, x2, y2 = filter(math.isfinite, map(float, seq))\n except ValueError as exc:\n raise UsageError('Bounding box parameter needs to be numbers.') from exc\n\n x1 = min(180, max(-180, x1))\n x2 = min(180, max(-180, x2))\n y1 = min(90, max(-90, y1))\n y2 = min(90, max(-90, y2))\n\n if x1 == x2 or y1 == y2:\n raise UsageError('Bounding box with invalid parameters.')\n\n return Bbox(min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2))\n\n\nclass GeometryFormat(enum.Flag):\n \"\"\" All search functions support returning the full geometry of a place in\n various formats. The internal geometry is converted by PostGIS to\n the desired format and then returned as a string. It is possible to\n request multiple formats at the same time.\n \"\"\"\n NONE = 0\n \"\"\" No geometry requested. Alias for a empty flag.\n \"\"\"\n GEOJSON = enum.auto()\n \"\"\"\n [GeoJSON](https://geojson.org/) format\n \"\"\"\n KML = enum.auto()\n \"\"\"\n [KML](https://en.wikipedia.org/wiki/Keyhole_Markup_Language) format\n \"\"\"\n SVG = enum.auto()\n \"\"\"\n [SVG](http://www.w3.org/TR/SVG/paths.html) format\n \"\"\"\n TEXT = enum.auto()\n \"\"\"\n [WKT](https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry) format\n \"\"\"\n\n\nclass DataLayer(enum.Flag):\n \"\"\" The `DataLayer` flag type defines the layers that can be selected\n for reverse and forward search.\n \"\"\"\n ADDRESS = enum.auto()\n \"\"\" The address layer contains all places relevant for addresses:\n fully qualified addresses with a house number (or a house name equivalent,\n for some addresses) and places that can be part of an address like\n roads, cities, states.\n \"\"\"\n POI = enum.auto()\n \"\"\" Layer for points of interest like shops, restaurants but also\n recycling bins or postboxes.\n \"\"\"\n RAILWAY = enum.auto()\n \"\"\" Layer with railway features including tracks and other infrastructure.\n Note that in Nominatim's standard configuration, only very few railway\n features are imported into the database. Thus a custom configuration\n is required to make full use of this layer.\n \"\"\"\n NATURAL = enum.auto()\n \"\"\" Layer with natural features like rivers, lakes and mountains.\n \"\"\"\n MANMADE = enum.auto()\n \"\"\" Layer with other human-made features and boundaries. This layer is\n the catch-all and includes all features not covered by the other\n layers. A typical example for this layer are national park boundaries.\n \"\"\"\n\n\nclass QueryStatistics(dict[str, Any]):\n \"\"\" A specialised dictionary for collecting query statistics.\n \"\"\"\n\n def __enter__(self) -> 'QueryStatistics':\n self.log_time('start')\n return self\n\n def __exit__(self, *_: Any) -> None:\n self.log_time('end')\n self['total_time'] = (self['end'] - self['start']).total_seconds()\n if 'start_query' in self:\n self['wait_time'] = (self['start_query'] - self['start']).total_seconds()\n else:\n self['wait_time'] = self['total_time']\n self['start_query'] = self['end']\n self['query_time'] = self['total_time'] - self['wait_time']\n\n def __missing__(self, key: str) -> str:\n return ''\n\n def log_time(self, key: str) -> None:\n self[key] = dt.datetime.now(tz=dt.timezone.utc)\n\n\nclass NoQueryStats:\n \"\"\" Null object to use, when no query statistics are requested.\n \"\"\"\n\n def __enter__(self) -> 'NoQueryStats':\n return self\n\n def __exit__(self, *_: Any) -> None:\n pass\n\n def __setitem__(self, key: str, value: Any) -> None:\n pass\n\n def __getitem__(self, key: str) -> Any:\n return None\n\n def __contains__(self, key: str, default: Any = None) -> bool:\n return False\n\n def log_time(self, key: str) -> None:\n pass\n\n\ndef format_country(cc: Any) -> List[str]:\n \"\"\" Extract a list of country codes from the input which may be either\n a string or list of strings. Filters out all values that are not\n a two-letter string.\n \"\"\"\n clist: Sequence[str]\n if isinstance(cc, str):\n clist = cc.split(',')\n elif isinstance(cc, abc.Sequence):\n clist = cc\n else:\n raise UsageError(\"Parameter 'country' needs to be a comma-separated list \"\n \"or a Python list of strings.\")\n\n return [cc.lower() for cc in clist if isinstance(cc, str) and len(cc) == 2]\n\n\ndef format_excluded(ids: Any) -> List[PlaceRef]:\n \"\"\" Extract a list of place IDs and OSM IDs from the input.\n \"\"\"\n if not ids:\n return []\n\n plist: Sequence[str]\n if isinstance(ids, str):\n plist = [s.strip() for s in ids.split(',')]\n elif isinstance(ids, abc.Sequence):\n plist = ids\n else:\n raise UsageError(\"Parameter 'excluded' needs to be a comma-separated list \"\n \"or a Python list of place IDs or OSM IDs.\")\n\n result: List[PlaceRef] = []\n for i in plist:\n if not i:\n continue\n if isinstance(i, (PlaceID, OsmID)):\n result.append(i)\n elif isinstance(i, int):\n if i > 0:\n result.append(PlaceID(i))\n elif isinstance(i, str):\n if i.isdigit():\n if int(i) > 0:\n result.append(PlaceID(int(i)))\n elif len(i) > 1 and i[0].upper() in ('N', 'W', 'R') and i[1:].isdigit():\n if int(i[1:]) > 0:\n result.append(OsmID(i[0].upper(), int(i[1:])))\n else:\n raise UsageError(f\"Invalid exclude ID: {i}\")\n else:\n raise UsageError(\"Parameter 'excluded' contains invalid types.\")\n\n return result\n\n\ndef format_categories(categories: List[Tuple[str, str]]) -> List[Tuple[str, str]]:\n \"\"\" Extract a list of categories. Currently a noop.\n \"\"\"\n return categories\n\n\nTParam = TypeVar('TParam', bound='LookupDetails')\n\n\n@dataclasses.dataclass\nclass LookupDetails:\n \"\"\" Collection of parameters that define which kind of details are\n returned with a lookup or details result.\n \"\"\"\n geometry_output: GeometryFormat = GeometryFormat.NONE\n \"\"\" Add the full geometry of the place to the result. Multiple\n formats may be selected. Note that geometries can become quite large.\n \"\"\"\n address_details: bool = False\n \"\"\" Get detailed information on the places that make up the address\n for the result.\n \"\"\"\n linked_places: bool = False\n \"\"\" Get detailed information on the places that link to the result.\n \"\"\"\n parented_places: bool = False\n \"\"\" Get detailed information on all places that this place is a parent\n for, i.e. all places for which it provides the address details.\n Only POI places can have parents.\n \"\"\"\n entrances: bool = False\n \"\"\" Get detailed information about the tagged entrances for the result.\n \"\"\"\n keywords: bool = False\n \"\"\" Add information about the search terms used for this place.\n \"\"\"\n geometry_simplification: float = 0.0\n \"\"\" Simplification factor for a geometry in degrees WGS. A factor of\n 0.0 means the original geometry is kept. The higher the value, the\n more the geometry gets simplified.\n \"\"\"\n query_stats: Union[QueryStatistics, NoQueryStats] = \\\n dataclasses.field(default_factory=NoQueryStats)\n \"\"\" Optional QueryStatistics object collecting information about\n runtime behaviour of the call.\n \"\"\"\n\n @classmethod\n def from_kwargs(cls: Type[TParam], kwargs: Dict[str, Any]) -> TParam:\n \"\"\" Load the data fields of the class from a dictionary.\n Unknown entries in the dictionary are ignored, missing ones\n get the default setting.\n\n The function supports type checking and throws a UsageError\n when the value does not fit.\n \"\"\"\n def _check_field(v: Any, field: 'dataclasses.Field[Any]') -> Any:\n if v is None:\n return field.default_factory() \\\n if field.default_factory != dataclasses.MISSING \\\n else field.default\n if field.metadata and 'transform' in field.metadata:\n return field.metadata['transform'](v)\n if not isinstance(v, field.type): # type: ignore[arg-type]\n raise UsageError(f\"Parameter '{field.name}' needs to be of {field.type!s}.\")\n return v\n\n return cls(**{f.name: _check_field(kwargs[f.name], f)\n for f in dataclasses.fields(cls) if f.name in kwargs})\n\n\n@dataclasses.dataclass\nclass ReverseDetails(LookupDetails):\n \"\"\" Collection of parameters for the reverse call.\n \"\"\"\n\n max_rank: int = dataclasses.field(default=30,\n metadata={'transform': lambda v: max(0, min(v, 30))})\n \"\"\" Highest address rank to return.\n \"\"\"\n\n layers: DataLayer = DataLayer.ADDRESS | DataLayer.POI\n \"\"\" Filter which kind of data to include.\n \"\"\"\n\n\n@dataclasses.dataclass\nclass SearchDetails(LookupDetails):\n \"\"\" Collection of parameters for the search call.\n \"\"\"\n max_results: int = 10\n \"\"\" Maximum number of results to be returned. The actual number of results\n may be less.\n \"\"\"\n\n min_rank: int = dataclasses.field(default=0,\n metadata={'transform': lambda v: max(0, min(v, 30))})\n \"\"\" Lowest address rank to return.\n \"\"\"\n\n max_rank: int = dataclasses.field(default=30,\n metadata={'transform': lambda v: max(0, min(v, 30))})\n \"\"\" Highest address rank to return.\n \"\"\"\n\n layers: Optional[DataLayer] = dataclasses.field(default=None,\n metadata={'transform': lambda r: r})\n \"\"\" Filter which kind of data to include. When 'None' (the default) then\n filtering by layers is disabled.\n \"\"\"\n\n countries: List[str] = dataclasses.field(default_factory=list,\n metadata={'transform': format_country})\n \"\"\" Restrict search results to the given countries. An empty list (the\n default) will disable this filter.\n \"\"\"\n\n excluded: List[PlaceRef] = dataclasses.field(default_factory=list,\n metadata={'transform': format_excluded})\n \"\"\" List of OSM objects to exclude from the results,\n provided as either internal Place IDs or OSM IDs.\n \"\"\"\n\n viewbox: Optional[Bbox] = dataclasses.field(default=None,\n metadata={'transform': Bbox.from_param})\n \"\"\" Focus the search on a given map area.\n \"\"\"\n\n bounded_viewbox: bool = False\n \"\"\" Use 'viewbox' as a filter and restrict results to places within the\n given area.\n \"\"\"\n\n near: Optional[Point] = dataclasses.field(default=None,\n metadata={'transform': Point.from_param})\n \"\"\" Order results by distance to the given point.\n \"\"\"\n\n near_radius: Optional[float] = dataclasses.field(default=None,\n metadata={'transform': lambda r: r})\n \"\"\" Use near point as a filter and drop results outside the given\n radius. Radius is given in degrees WSG84.\n \"\"\"\n\n categories: List[Tuple[str, str]] = dataclasses.field(default_factory=list,\n metadata={'transform': format_categories})\n \"\"\" Restrict search to places with one of the given class/type categories.\n An empty list (the default) will disable this filter.\n \"\"\"\n\n viewbox_x2: Optional[Bbox] = None\n\n locales: Optional['Locales'] = dataclasses.field(\n default=None, metadata={'transform': lambda v: v})\n \"\"\" Locale preferences of the caller.\n Used during result re-ranking to prefer results that match the\n caller's locale over results that only match in an alternate language.\n \"\"\"\n\n def __post_init__(self) -> None:\n if self.viewbox is not None:\n xext = (self.viewbox.maxlon - self.viewbox.minlon)/2\n yext = (self.viewbox.maxlat - self.viewbox.minlat)/2\n self.viewbox_x2 = Bbox(self.viewbox.minlon - xext, self.viewbox.minlat - yext,\n self.viewbox.maxlon + xext, self.viewbox.maxlat + yext)\n\n @property\n def excluded_place_ids(self) -> List[int]:\n \"\"\" Return excluded entries as a plain list of place ID integers.\n Only includes PlaceID entries. Returns [0] if empty to\n ensure SQL NOT IN clauses work correctly.\n \"\"\"\n return [e.place_id for e in self.excluded if isinstance(e, PlaceID)] or [0]\n\n def restrict_min_max_rank(self, new_min: int, new_max: int) -> None:\n \"\"\" Change the min_rank and max_rank fields to respect the\n given boundaries.\n \"\"\"\n assert new_min <= new_max\n self.min_rank = max(self.min_rank, new_min)\n self.max_rank = min(self.max_rank, new_max)\n\n def is_impossible(self) -> bool:\n \"\"\" Check if the parameter configuration is contradictionary and\n cannot yield any results.\n \"\"\"\n return (self.min_rank > self.max_rank\n or (self.bounded_viewbox\n and self.viewbox is not None and self.near is not None\n and self.viewbox.contains(self.near))\n or (self.layers is not None and not self.layers)\n or (self.max_rank <= 4 and\n self.layers is not None and not self.layers & DataLayer.ADDRESS))\n\n def layer_enabled(self, layer: DataLayer) -> bool:\n \"\"\" Check if the given layer has been chosen. Also returns\n true when layer restriction has been disabled completely.\n \"\"\"\n return self.layers is None or bool(self.layers & layer)\n\n\n@dataclasses.dataclass\nclass EntranceDetails:\n \"\"\" Reference a place by its OSM ID and potentially the basic category.\n\n The OSM ID may refer to places in the main table placex and OSM\n interpolation lines.\n \"\"\"\n osm_id: int\n \"\"\" The OSM ID of the object.\n \"\"\"\n type: str\n \"\"\" The value of the OSM entrance tag (i.e. yes, main, secondary, etc.).\n \"\"\"\n location: Point\n \"\"\" The location of the entrance node.\n \"\"\"\n extratags: Dict[str, str]\n \"\"\" The other tags associated with the entrance node.\n \"\"\"\n"
},
{
"path": "src/nominatim_api/typing.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nType definitions for typing annotations.\n\nComplex type definitions are moved here, to keep the source files readable.\n\"\"\"\nfrom typing import Union, TYPE_CHECKING\n\n# flake8: noqa\n\n# SQLAlchemy introduced generic types in version 2.0 making typing\n# incompatible with older versions. Add wrappers here so we don't have\n# to litter the code with bare-string types.\n\nif TYPE_CHECKING:\n from typing import Any\n import sqlalchemy as sa\n import os\n from typing_extensions import (TypeAlias as TypeAlias,\n Protocol as Protocol)\nelse:\n TypeAlias = str\n Protocol = object\n\nStrPath = Union[str, 'os.PathLike[str]']\n\nSaLambdaSelect: TypeAlias = 'Union[sa.Select[Any], sa.StatementLambdaElement]'\nSaSelect: TypeAlias = 'sa.Select[Any]'\nSaScalarSelect: TypeAlias = 'sa.ScalarSelect[Any]'\nSaRow: TypeAlias = 'sa.Row[Any]'\nSaColumn: TypeAlias = 'sa.ColumnElement[Any]'\nSaExpression: TypeAlias = 'sa.ColumnElement[bool]'\nSaLabel: TypeAlias = 'sa.Label[Any]'\nSaFromClause: TypeAlias = 'sa.FromClause'\nSaSelectable: TypeAlias = 'sa.Selectable'\nSaBind: TypeAlias = 'sa.BindParameter[Any]'\nSaDialect: TypeAlias = 'sa.Dialect'\n"
},
{
"path": "src/nominatim_api/utils/__init__.py",
"content": ""
},
{
"path": "src/nominatim_api/utils/json_writer.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nStreaming JSON encoder.\n\"\"\"\nfrom typing import Any, TypeVar, Optional, Callable\nimport io\ntry:\n import ujson as json\nexcept ModuleNotFoundError:\n import json # type: ignore[no-redef]\n\nT = TypeVar('T')\n\n\nclass JsonWriter:\n \"\"\" JSON encoder that renders the output directly into an output\n stream. This is a very simple writer which produces JSON in a\n compact as possible form.\n\n The writer does not check for syntactic correctness. It is the\n responsibility of the caller to call the write functions in an\n order that produces correct JSON.\n\n All functions return the writer object itself so that function\n calls can be chained.\n \"\"\"\n\n def __init__(self) -> None:\n self.data = io.StringIO()\n self.pending = ''\n\n def __call__(self) -> str:\n \"\"\" Return the rendered JSON content as a string.\n The writer remains usable after calling this function.\n \"\"\"\n if self.pending:\n assert self.pending in (']', '}')\n self.data.write(self.pending)\n self.pending = ''\n return self.data.getvalue()\n\n def start_object(self) -> 'JsonWriter':\n \"\"\" Write the open bracket of a JSON object.\n \"\"\"\n if self.pending:\n self.data.write(self.pending)\n self.pending = '{'\n return self\n\n def end_object(self) -> 'JsonWriter':\n \"\"\" Write the closing bracket of a JSON object.\n \"\"\"\n assert self.pending in (',', '{', '')\n if self.pending == '{':\n self.data.write(self.pending)\n self.pending = '}'\n return self\n\n def start_array(self) -> 'JsonWriter':\n \"\"\" Write the opening bracket of a JSON array.\n \"\"\"\n if self.pending:\n self.data.write(self.pending)\n self.pending = '['\n return self\n\n def end_array(self) -> 'JsonWriter':\n \"\"\" Write the closing bracket of a JSON array.\n \"\"\"\n assert self.pending in (',', '[', ']', ')', '')\n if self.pending not in (',', ''):\n self.data.write(self.pending)\n self.pending = ']'\n return self\n\n def key(self, name: str) -> 'JsonWriter':\n \"\"\" Write the key string of a JSON object.\n \"\"\"\n assert self.pending\n self.data.write(self.pending)\n self.data.write(json.dumps(name, ensure_ascii=False))\n self.pending = ':'\n return self\n\n def value(self, value: Any) -> 'JsonWriter':\n \"\"\" Write out a value as JSON. The function uses the json.dumps()\n function for encoding the JSON. Thus any value that can be\n encoded by that function is permissible here.\n \"\"\"\n return self.raw(json.dumps(value, ensure_ascii=False))\n\n def float(self, value: float, precision: int) -> 'JsonWriter':\n \"\"\" Write out a float value with the given precision.\n \"\"\"\n return self.raw(f\"{value:0.{precision}f}\")\n\n def next(self) -> 'JsonWriter':\n \"\"\" Write out a delimiter comma between JSON object or array elements.\n \"\"\"\n if self.pending:\n self.data.write(self.pending)\n self.pending = ','\n return self\n\n def raw(self, raw_json: str) -> 'JsonWriter':\n \"\"\" Write out the given value as is. This function is useful if\n a value is already available in JSON format.\n \"\"\"\n if self.pending:\n self.data.write(self.pending)\n self.pending = ''\n self.data.write(raw_json)\n return self\n\n def keyval(self, key: str, value: Any) -> 'JsonWriter':\n \"\"\" Write out an object element with the given key and value.\n This is a shortcut for calling 'key()', 'value()' and 'next()'.\n \"\"\"\n self.key(key)\n self.value(value)\n return self.next()\n\n def keyval_not_none(self, key: str, value: Optional[T],\n transform: Optional[Callable[[T], Any]] = None) -> 'JsonWriter':\n \"\"\" Write out an object element only if the value is not None.\n If 'transform' is given, it must be a function that takes the\n value type and returns a JSON encodable type. The transform\n function will be called before the value is written out.\n \"\"\"\n if value is not None:\n self.key(key)\n self.value(transform(value) if transform else value)\n self.next()\n return self\n"
},
{
"path": "src/nominatim_api/v1/__init__.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of API version v1 (aka the legacy version).\n\"\"\"\n\nfrom .server_glue import get_routes as get_routes\n"
},
{
"path": "src/nominatim_api/v1/classtypes.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHard-coded information about tag categories.\n\nThese tables have been copied verbatim from the old PHP code. For future\nversion a more flexible formatting is required.\n\"\"\"\nfrom typing import Tuple, Optional, Mapping, Union\n\nfrom ..results import ReverseResult, SearchResult\nfrom ..types import Bbox\n\n\ndef get_label_tag(category: Tuple[str, str], extratags: Optional[Mapping[str, str]],\n rank: int, country: Optional[str]) -> str:\n \"\"\" Create a label tag for the given place that can be used as an XML name.\n \"\"\"\n if category in (('place', 'postcode'), ('boundary', 'postal_code')):\n label = 'postcode'\n elif rank < 26 and extratags and 'place' in extratags:\n label = extratags['place']\n elif rank < 26 and extratags and 'linked_place' in extratags:\n label = extratags['linked_place']\n elif category == ('boundary', 'administrative'):\n label = ADMIN_LABELS.get((country or '', rank // 2))\\\n or ADMIN_LABELS.get(('', rank // 2))\\\n or 'Administrative'\n elif rank < 26:\n label = category[1] if category[1] != 'yes' else category[0]\n elif rank < 28:\n label = 'road'\n elif (category[0] == 'place'\n and category[1] in ('house_number', 'house_name', 'country_code')):\n label = category[1]\n else:\n label = category[0]\n\n return label.lower().replace(' ', '_')\n\n\ndef bbox_from_result(result: Union[ReverseResult, SearchResult]) -> Bbox:\n \"\"\" Compute a bounding box for the result. For ways and relations\n a given boundingbox is used. For all other object, a box is computed\n around the centroid according to dimensions derived from the\n search rank.\n \"\"\"\n if result.category == ('place', 'postcode') and result.bbox is None:\n return Bbox.from_point(result.centroid,\n 0.05 - 0.012 * (result.rank_search - 21))\n\n if (result.osm_object and result.osm_object[0] == 'N') or result.bbox is None:\n\n extent = NODE_EXTENT.get(result.category, 0.00005)\n return Bbox.from_point(result.centroid, extent)\n\n return result.bbox\n\n\nOSM_ATTRIBUTION = 'Data © OpenStreetMap contributors, ODbL 1.0. http://osm.org/copyright'\n\n\nOSM_TYPE_NAME = {\n 'N': 'node',\n 'W': 'way',\n 'R': 'relation'\n}\n\n\nADMIN_LABELS = {\n ('', 1): 'Continent',\n ('', 2): 'Country',\n ('', 3): 'Region',\n ('', 4): 'State',\n ('', 5): 'State District',\n ('', 6): 'County',\n ('', 7): 'Municipality',\n ('', 8): 'City',\n ('', 9): 'City District',\n ('', 10): 'Suburb',\n ('', 11): 'Neighbourhood',\n ('', 12): 'City Block',\n ('no', 3): 'State',\n ('no', 4): 'County',\n ('se', 3): 'State',\n ('se', 4): 'County'\n}\n\n\nICONS = {\n ('boundary', 'administrative'): 'poi_boundary_administrative',\n ('place', 'city'): 'poi_place_city',\n ('place', 'town'): 'poi_place_town',\n ('place', 'village'): 'poi_place_village',\n ('place', 'hamlet'): 'poi_place_village',\n ('place', 'suburb'): 'poi_place_village',\n ('place', 'locality'): 'poi_place_village',\n ('place', 'airport'): 'transport_airport2',\n ('aeroway', 'aerodrome'): 'transport_airport2',\n ('railway', 'station'): 'transport_train_station2',\n ('amenity', 'place_of_worship'): 'place_of_worship_unknown3',\n ('amenity', 'pub'): 'food_pub',\n ('amenity', 'bar'): 'food_bar',\n ('amenity', 'university'): 'education_university',\n ('tourism', 'museum'): 'tourist_museum',\n ('amenity', 'arts_centre'): 'tourist_art_gallery2',\n ('tourism', 'zoo'): 'tourist_zoo',\n ('tourism', 'theme_park'): 'poi_point_of_interest',\n ('tourism', 'attraction'): 'poi_point_of_interest',\n ('leisure', 'golf_course'): 'sport_golf',\n ('historic', 'castle'): 'tourist_castle',\n ('amenity', 'hospital'): 'health_hospital',\n ('amenity', 'school'): 'education_school',\n ('amenity', 'theatre'): 'tourist_theatre',\n ('amenity', 'library'): 'amenity_library',\n ('amenity', 'fire_station'): 'amenity_firestation3',\n ('amenity', 'police'): 'amenity_police2',\n ('amenity', 'bank'): 'money_bank2',\n ('amenity', 'post_office'): 'amenity_post_office',\n ('tourism', 'hotel'): 'accommodation_hotel2',\n ('amenity', 'cinema'): 'tourist_cinema',\n ('tourism', 'artwork'): 'tourist_art_gallery2',\n ('historic', 'archaeological_site'): 'tourist_archaeological2',\n ('amenity', 'doctors'): 'health_doctors',\n ('leisure', 'sports_centre'): 'sport_leisure_centre',\n ('leisure', 'swimming_pool'): 'sport_swimming_outdoor',\n ('shop', 'supermarket'): 'shopping_supermarket',\n ('shop', 'convenience'): 'shopping_convenience',\n ('amenity', 'restaurant'): 'food_restaurant',\n ('amenity', 'fast_food'): 'food_fastfood',\n ('amenity', 'cafe'): 'food_cafe',\n ('tourism', 'guest_house'): 'accommodation_bed_and_breakfast',\n ('amenity', 'pharmacy'): 'health_pharmacy_dispensing',\n ('amenity', 'fuel'): 'transport_fuel',\n ('natural', 'peak'): 'poi_peak',\n ('natural', 'wood'): 'landuse_coniferous_and_deciduous',\n ('shop', 'bicycle'): 'shopping_bicycle',\n ('shop', 'clothes'): 'shopping_clothes',\n ('shop', 'hairdresser'): 'shopping_hairdresser',\n ('shop', 'doityourself'): 'shopping_diy',\n ('shop', 'estate_agent'): 'shopping_estateagent2',\n ('shop', 'car'): 'shopping_car',\n ('shop', 'garden_centre'): 'shopping_garden_centre',\n ('shop', 'car_repair'): 'shopping_car_repair',\n ('shop', 'bakery'): 'shopping_bakery',\n ('shop', 'butcher'): 'shopping_butcher',\n ('shop', 'apparel'): 'shopping_clothes',\n ('shop', 'laundry'): 'shopping_laundrette',\n ('shop', 'beverages'): 'shopping_alcohol',\n ('shop', 'alcohol'): 'shopping_alcohol',\n ('shop', 'optician'): 'health_opticians',\n ('shop', 'chemist'): 'health_pharmacy',\n ('shop', 'gallery'): 'tourist_art_gallery2',\n ('shop', 'jewelry'): 'shopping_jewelry',\n ('tourism', 'information'): 'amenity_information',\n ('historic', 'ruins'): 'tourist_ruin',\n ('amenity', 'college'): 'education_school',\n ('historic', 'monument'): 'tourist_monument',\n ('historic', 'memorial'): 'tourist_monument',\n ('historic', 'mine'): 'poi_mine',\n ('tourism', 'caravan_site'): 'accommodation_caravan_park',\n ('amenity', 'bus_station'): 'transport_bus_station',\n ('amenity', 'atm'): 'money_atm2',\n ('tourism', 'viewpoint'): 'tourist_view_point',\n ('tourism', 'guesthouse'): 'accommodation_bed_and_breakfast',\n ('railway', 'tram'): 'transport_tram_stop',\n ('amenity', 'courthouse'): 'amenity_court',\n ('amenity', 'recycling'): 'amenity_recycling',\n ('amenity', 'dentist'): 'health_dentist',\n ('natural', 'beach'): 'tourist_beach',\n ('railway', 'tram_stop'): 'transport_tram_stop',\n ('amenity', 'prison'): 'amenity_prison',\n ('highway', 'bus_stop'): 'transport_bus_stop2'\n}\n\nNODE_EXTENT = {\n ('place', 'continent'): 25,\n ('place', 'country'): 7,\n ('place', 'state'): 2.6,\n ('place', 'province'): 2.6,\n ('place', 'region'): 1.0,\n ('place', 'county'): 0.7,\n ('place', 'city'): 0.16,\n ('place', 'municipality'): 0.16,\n ('place', 'island'): 0.32,\n ('place', 'postcode'): 0.16,\n ('place', 'town'): 0.04,\n ('place', 'village'): 0.02,\n ('place', 'hamlet'): 0.02,\n ('place', 'district'): 0.02,\n ('place', 'borough'): 0.02,\n ('place', 'suburb'): 0.02,\n ('place', 'locality'): 0.01,\n ('place', 'neighbourhood'): 0.01,\n ('place', 'quarter'): 0.01,\n ('place', 'city_block'): 0.01,\n ('landuse', 'farm'): 0.01,\n ('place', 'farm'): 0.01,\n ('place', 'airport'): 0.015,\n ('aeroway', 'aerodrome'): 0.015,\n ('railway', 'station'): 0.005\n}\n"
},
{
"path": "src/nominatim_api/v1/format.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nOutput formatters for API version v1.\n\"\"\"\nfrom typing import List, Dict, Mapping, Any\nimport collections\nimport datetime as dt\n\nfrom ..utils.json_writer import JsonWriter\nfrom ..status import StatusResult\nfrom ..results import DetailedResult, ReverseResults, SearchResults, \\\n AddressLines, AddressLine\nfrom ..localization import Locales\nfrom ..result_formatting import FormatDispatcher\nfrom .classtypes import ICONS\nfrom . import format_json, format_xml\nfrom .helpers import _add_admin_level\nfrom .. import logging as loglib\nfrom ..server import content_types as ct\n\n\nclass RawDataList(List[Dict[str, Any]]):\n \"\"\" Data type for formatting raw data lists 'as is' in json.\n \"\"\"\n\n\ndispatch = FormatDispatcher({'text': ct.CONTENT_TEXT,\n 'xml': ct.CONTENT_XML,\n 'debug': ct.CONTENT_HTML})\n\n\n@dispatch.error_format_func\ndef _format_error(content_type: str, msg: str, status: int) -> str:\n if content_type == ct.CONTENT_XML:\n return f\"\"\"\n \n {status}\n {msg}\n \n \"\"\"\n\n if content_type == ct.CONTENT_JSON:\n return f\"\"\"{{\"error\":{{\"code\":{status},\"message\":\"{msg}\"}}}}\"\"\"\n\n if content_type == ct.CONTENT_HTML:\n loglib.log().section('Execution error')\n loglib.log().var_dump('Status', status)\n loglib.log().var_dump('Message', msg)\n return loglib.get_and_disable()\n\n return f\"ERROR {status}: {msg}\"\n\n\n@dispatch.format_func(StatusResult, 'text')\ndef _format_status_text(result: StatusResult, _: Mapping[str, Any]) -> str:\n if result.status:\n return f\"ERROR: {result.message}\"\n\n return 'OK'\n\n\n@dispatch.format_func(StatusResult, 'json')\ndef _format_status_json(result: StatusResult, _: Mapping[str, Any]) -> str:\n out = JsonWriter()\n\n out.start_object()\\\n .keyval('status', result.status)\\\n .keyval('message', result.message)\\\n .keyval_not_none('data_updated', result.data_updated,\n lambda v: v.isoformat())\\\n .keyval('software_version', str(result.software_version))\\\n .keyval_not_none('database_version', result.database_version, str)\\\n .end_object()\n\n return out()\n\n\ndef _add_address_row(writer: JsonWriter, row: AddressLine,\n locales: Locales) -> None:\n writer.start_object()\\\n .keyval('localname', locales.display_name(row.names))\\\n .keyval_not_none('place_id', row.place_id)\n\n if row.osm_object is not None:\n writer.keyval('osm_id', row.osm_object[1])\\\n .keyval('osm_type', row.osm_object[0])\n\n if row.extratags:\n writer.keyval_not_none('place_type', row.extratags.get('place_type'))\n\n writer.keyval('class', row.category[0])\\\n .keyval('type', row.category[1])\\\n .keyval_not_none('admin_level', row.admin_level)\\\n .keyval('rank_address', row.rank_address)\\\n .keyval('distance', row.distance)\\\n .keyval('isaddress', row.isaddress)\\\n .end_object()\n\n\ndef _add_address_rows(writer: JsonWriter, section: str, rows: AddressLines,\n locales: Locales) -> None:\n writer.key(section).start_array()\n for row in rows:\n _add_address_row(writer, row, locales)\n writer.next()\n writer.end_array().next()\n\n\ndef _add_parent_rows_grouped(writer: JsonWriter, rows: AddressLines,\n locales: Locales) -> None:\n # group by category type\n data = collections.defaultdict(list)\n for row in rows:\n sub = JsonWriter()\n _add_address_row(sub, row, locales)\n data[row.category[1]].append(sub())\n\n writer.key('hierarchy').start_object()\n for group, grouped in data.items():\n writer.key(group).start_array()\n grouped.sort() # sorts alphabetically by local name\n for line in grouped:\n writer.raw(line).next()\n writer.end_array().next()\n\n writer.end_object().next()\n\n\n@dispatch.format_func(DetailedResult, 'json')\ndef _format_details_json(result: DetailedResult, options: Mapping[str, Any]) -> str:\n locales = options.get('locales') or Locales()\n geom = result.geometry.get('geojson')\n centroid = result.centroid.to_geojson()\n\n out = JsonWriter()\n out.start_object()\\\n .keyval_not_none('place_id', result.place_id)\\\n .keyval_not_none('parent_place_id', result.parent_place_id)\n\n if result.osm_object is not None:\n out.keyval('osm_type', result.osm_object[0])\\\n .keyval('osm_id', result.osm_object[1])\n\n out.keyval('category', result.category[0])\\\n .keyval('type', result.category[1])\\\n .keyval('admin_level', result.admin_level)\\\n .keyval('localname', result.locale_name or '')\\\n .keyval('names', result.names or {})\\\n .keyval('addresstags', result.address or {})\\\n .keyval_not_none('housenumber', result.housenumber)\\\n .keyval_not_none('calculated_postcode', result.postcode)\\\n .keyval_not_none('country_code', result.country_code)\\\n .keyval_not_none('indexed_date', result.indexed_date, lambda v: v.isoformat())\\\n .keyval_not_none('importance', result.importance)\\\n .keyval('calculated_importance', result.calculated_importance())\\\n .keyval('extratags', _add_admin_level(result) or {})\\\n .keyval_not_none('calculated_wikipedia', result.wikipedia)\\\n .keyval('rank_address', result.rank_address)\\\n .keyval('rank_search', result.rank_search)\\\n .keyval('isarea', 'Polygon' in (geom or result.geometry.get('type') or ''))\\\n .key('centroid').raw(centroid).next()\\\n .key('geometry').raw(geom or centroid).next()\n\n if options.get('icon_base_url', None):\n icon = ICONS.get(result.category)\n if icon:\n out.keyval('icon', f\"{options['icon_base_url']}/{icon}.p.20.png\")\n\n if result.address_rows is not None:\n _add_address_rows(out, 'address', result.address_rows, locales)\n\n if result.linked_rows:\n _add_address_rows(out, 'linked_places', result.linked_rows, locales)\n\n if result.name_keywords is not None or result.address_keywords is not None:\n out.key('keywords').start_object()\n\n for sec, klist in (('name', result.name_keywords), ('address', result.address_keywords)):\n out.key(sec).start_array()\n for word in (klist or []):\n out.start_object()\\\n .keyval('id', word.word_id)\\\n .keyval('token', word.word_token)\\\n .end_object().next()\n out.end_array().next()\n\n out.end_object().next()\n\n if result.parented_rows is not None:\n if options.get('group_hierarchy', False):\n _add_parent_rows_grouped(out, result.parented_rows, locales)\n else:\n _add_address_rows(out, 'hierarchy', result.parented_rows, locales)\n\n if options.get('entrances', False):\n format_json.write_entrances(out, result.entrances)\n\n out.end_object()\n\n return out()\n\n\n@dispatch.format_func(ReverseResults, 'xml')\ndef _format_reverse_xml(results: ReverseResults, options: Mapping[str, Any]) -> str:\n return format_xml.format_base_xml(results,\n options, True, 'reversegeocode',\n {'querystring': options.get('query', '')})\n\n\n@dispatch.format_func(ReverseResults, 'geojson')\ndef _format_reverse_geojson(results: ReverseResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_geojson(results, options, True)\n\n\n@dispatch.format_func(ReverseResults, 'geocodejson')\ndef _format_reverse_geocodejson(results: ReverseResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_geocodejson(results, options, True)\n\n\n@dispatch.format_func(ReverseResults, 'json')\ndef _format_reverse_json(results: ReverseResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_json(results, options, True,\n class_label='class')\n\n\n@dispatch.format_func(ReverseResults, 'jsonv2')\ndef _format_reverse_jsonv2(results: ReverseResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_json(results, options, True,\n class_label='category')\n\n\n@dispatch.format_func(SearchResults, 'xml')\ndef _format_search_xml(results: SearchResults, options: Mapping[str, Any]) -> str:\n extra = {'querystring': options.get('query', '')}\n for attr in ('more_url', 'exclude_place_ids', 'viewbox'):\n if options.get(attr):\n extra[attr] = options[attr]\n return format_xml.format_base_xml(results, options, False, 'searchresults',\n extra)\n\n\n@dispatch.format_func(SearchResults, 'geojson')\ndef _format_search_geojson(results: SearchResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_geojson(results, options, False)\n\n\n@dispatch.format_func(SearchResults, 'geocodejson')\ndef _format_search_geocodejson(results: SearchResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_geocodejson(results, options, False)\n\n\n@dispatch.format_func(SearchResults, 'json')\ndef _format_search_json(results: SearchResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_json(results, options, False,\n class_label='class')\n\n\n@dispatch.format_func(SearchResults, 'jsonv2')\ndef _format_search_jsonv2(results: SearchResults,\n options: Mapping[str, Any]) -> str:\n return format_json.format_base_json(results, options, False,\n class_label='category')\n\n\n@dispatch.format_func(RawDataList, 'json')\ndef _format_raw_data_json(results: RawDataList, _: Mapping[str, Any]) -> str:\n out = JsonWriter()\n out.start_array()\n for res in results:\n out.start_object()\n for k, v in res.items():\n if isinstance(v, dt.datetime):\n out.keyval(k, v.isoformat(sep=' ', timespec='seconds'))\n else:\n out.keyval(k, v)\n out.end_object().next()\n\n out.end_array()\n\n return out()\n"
},
{
"path": "src/nominatim_api/v1/format_json.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelper functions for output of results in json formats.\n\"\"\"\nfrom typing import Mapping, Any, Optional, Tuple, Union, List\n\nfrom ..utils.json_writer import JsonWriter\nfrom ..results import AddressLines, ReverseResults, SearchResults\nfrom . import classtypes as cl\nfrom .helpers import _add_admin_level\nfrom ..types import EntranceDetails\n\n\ndef _write_osm_id(out: JsonWriter, osm_object: Optional[Tuple[str, int]]) -> None:\n if osm_object is not None:\n out.keyval_not_none('osm_type', cl.OSM_TYPE_NAME.get(osm_object[0], None))\\\n .keyval('osm_id', osm_object[1])\n\n\ndef _write_typed_address(out: JsonWriter, address: Optional[AddressLines],\n country_code: Optional[str]) -> None:\n parts = {}\n for line in (address or []):\n if line.isaddress:\n if line.local_name:\n label = cl.get_label_tag(line.category, line.extratags,\n line.rank_address, country_code)\n if label not in parts:\n parts[label] = line.local_name\n if line.names and 'ISO3166-2' in line.names and line.admin_level:\n parts[f\"ISO3166-2-lvl{line.admin_level}\"] = line.names['ISO3166-2']\n\n for k, v in parts.items():\n out.keyval(k, v)\n\n if country_code:\n out.keyval('country_code', country_code)\n\n\ndef _write_geocodejson_address(out: JsonWriter,\n address: Optional[AddressLines],\n obj_place_id: Optional[int],\n country_code: Optional[str]) -> None:\n extra = {}\n for line in (address or []):\n if line.isaddress and line.local_name:\n if line.category[1] in ('postcode', 'postal_code'):\n out.keyval('postcode', line.local_name)\n elif line.category[1] == 'house_number':\n out.keyval('housenumber', line.local_name)\n elif ((obj_place_id is None or obj_place_id != line.place_id)\n and line.rank_address >= 4 and line.rank_address < 28):\n rank_name = GEOCODEJSON_RANKS[line.rank_address]\n if rank_name not in extra:\n extra[rank_name] = line.local_name\n\n for k, v in extra.items():\n out.keyval(k, v)\n\n if country_code:\n out.keyval('country_code', country_code)\n\n\ndef write_entrances(out: JsonWriter, entrances: Optional[List[EntranceDetails]]) -> None:\n if entrances is None:\n out.keyval('entrances', None)\n return\n\n out.key('entrances')\\\n .start_array()\n\n for entrance in entrances:\n out.start_object()\\\n .keyval('osm_id', entrance.osm_id)\\\n .keyval('type', entrance.type)\\\n .keyval('lat', f\"{entrance.location.lat:0.7f}\")\\\n .keyval('lon', f\"{entrance.location.lon:0.7f}\")\n\n if entrance.extratags:\n out.keyval('extratags', entrance.extratags)\n out.end_object().next()\n\n out.end_array().next()\n\n\ndef format_base_json(results: Union[ReverseResults, SearchResults],\n options: Mapping[str, Any], simple: bool,\n class_label: str) -> str:\n \"\"\" Return the result list as a simple json string in custom Nominatim format.\n \"\"\"\n out = JsonWriter()\n\n if simple:\n if not results:\n return '{\"error\":\"Unable to geocode\"}'\n else:\n out.start_array()\n\n for result in results:\n out.start_object()\\\n .keyval_not_none('place_id', result.place_id)\\\n .keyval('licence', cl.OSM_ATTRIBUTION)\\\n\n _write_osm_id(out, result.osm_object)\n\n # lat and lon must be string values\n out.keyval('lat', f\"{result.centroid.lat:0.7f}\")\\\n .keyval('lon', f\"{result.centroid.lon:0.7f}\")\\\n .keyval(class_label, result.category[0])\\\n .keyval('type', result.category[1])\\\n .keyval('place_rank', result.rank_search)\\\n .keyval('importance', result.calculated_importance())\\\n .keyval('addresstype', cl.get_label_tag(result.category, result.extratags,\n result.rank_address,\n result.country_code))\\\n .keyval('name', result.locale_name or '')\\\n .keyval('display_name', result.display_name or '')\n\n if options.get('icon_base_url', None):\n icon = cl.ICONS.get(result.category)\n if icon:\n out.keyval('icon', f\"{options['icon_base_url']}/{icon}.p.20.png\")\n\n if options.get('addressdetails', False):\n out.key('address').start_object()\n _write_typed_address(out, result.address_rows, result.country_code)\n out.end_object().next()\n\n if options.get('entrances', False):\n write_entrances(out, result.entrances)\n\n if options.get('extratags', False):\n out.keyval('extratags', _add_admin_level(result))\n\n if options.get('namedetails', False):\n out.keyval('namedetails', result.names)\n\n # must be string values\n bbox = cl.bbox_from_result(result)\n out.key('boundingbox').start_array()\\\n .value(f\"{bbox.minlat:0.7f}\").next()\\\n .value(f\"{bbox.maxlat:0.7f}\").next()\\\n .value(f\"{bbox.minlon:0.7f}\").next()\\\n .value(f\"{bbox.maxlon:0.7f}\").next()\\\n .end_array().next()\n\n if result.geometry:\n for key in ('text', 'kml'):\n out.keyval_not_none('geo' + key, result.geometry.get(key))\n if 'geojson' in result.geometry:\n out.key('geojson').raw(result.geometry['geojson']).next()\n out.keyval_not_none('svg', result.geometry.get('svg'))\n\n out.end_object()\n\n if simple:\n return out()\n\n out.next()\n\n out.end_array()\n\n return out()\n\n\ndef format_base_geojson(results: Union[ReverseResults, SearchResults],\n options: Mapping[str, Any],\n simple: bool) -> str:\n \"\"\" Return the result list as a geojson string.\n \"\"\"\n if not results and simple:\n return '{\"error\":\"Unable to geocode\"}'\n\n out = JsonWriter()\n\n out.start_object()\\\n .keyval('type', 'FeatureCollection')\\\n .keyval('licence', cl.OSM_ATTRIBUTION)\\\n .key('features').start_array()\n\n for result in results:\n out.start_object()\\\n .keyval('type', 'Feature')\\\n .key('properties').start_object()\n\n out.keyval_not_none('place_id', result.place_id)\n\n _write_osm_id(out, result.osm_object)\n\n out.keyval('place_rank', result.rank_search)\\\n .keyval('category', result.category[0])\\\n .keyval('type', result.category[1])\\\n .keyval('importance', result.calculated_importance())\\\n .keyval('addresstype', cl.get_label_tag(result.category, result.extratags,\n result.rank_address,\n result.country_code))\\\n .keyval('name', result.locale_name or '')\\\n .keyval('display_name', result.display_name or '')\n\n if options.get('addressdetails', False):\n out.key('address').start_object()\n _write_typed_address(out, result.address_rows, result.country_code)\n out.end_object().next()\n\n if options.get('entrances', False):\n write_entrances(out, result.entrances)\n\n if options.get('extratags', False):\n out.keyval('extratags', _add_admin_level(result))\n\n if options.get('namedetails', False):\n out.keyval('namedetails', result.names)\n\n out.end_object().next() # properties\n\n out.key('bbox').start_array()\n for coord in cl.bbox_from_result(result).coords:\n out.float(coord, 7).next()\n out.end_array().next()\n\n out.key('geometry').raw(result.geometry.get('geojson')\n or result.centroid.to_geojson()).next()\n\n out.end_object().next()\n\n out.end_array().next().end_object()\n\n return out()\n\n\ndef format_base_geocodejson(results: Union[ReverseResults, SearchResults],\n options: Mapping[str, Any], simple: bool) -> str:\n \"\"\" Return the result list as a geocodejson string.\n \"\"\"\n if not results and simple:\n return '{\"error\":\"Unable to geocode\"}'\n\n out = JsonWriter()\n\n out.start_object()\\\n .keyval('type', 'FeatureCollection')\\\n .key('geocoding').start_object()\\\n .keyval('version', '0.1.0')\\\n .keyval('attribution', cl.OSM_ATTRIBUTION)\\\n .keyval('licence', 'ODbL')\\\n .keyval_not_none('query', options.get('query'))\\\n .end_object().next()\\\n .key('features').start_array()\n\n for result in results:\n out.start_object()\\\n .keyval('type', 'Feature')\\\n .key('properties').start_object()\\\n .key('geocoding').start_object()\n\n out.keyval_not_none('place_id', result.place_id)\n\n _write_osm_id(out, result.osm_object)\n\n out.keyval('osm_key', result.category[0])\\\n .keyval('osm_value', result.category[1])\\\n .keyval('type', GEOCODEJSON_RANKS[max(3, min(28, result.rank_address))])\\\n .keyval_not_none('accuracy', getattr(result, 'distance', None), transform=int)\\\n .keyval('label', result.display_name or '')\\\n .keyval_not_none('name', result.locale_name or None)\\\n\n if options.get('addressdetails', False):\n _write_geocodejson_address(out, result.address_rows, result.place_id,\n result.country_code)\n\n out.key('admin').start_object()\n if result.address_rows:\n for line in result.address_rows:\n if line.isaddress and (line.admin_level or 15) < 15 and line.local_name \\\n and line.category[0] == 'boundary' and line.category[1] == 'administrative':\n out.keyval(f\"level{line.admin_level}\", line.local_name)\n out.end_object().next()\n\n if options.get('entrances', False):\n write_entrances(out, result.entrances)\n\n if options.get('extratags', False):\n out.keyval('extra', _add_admin_level(result))\n\n out.end_object().next().end_object().next()\n\n out.key('geometry').raw(result.geometry.get('geojson')\n or result.centroid.to_geojson()).next()\n\n out.end_object().next()\n\n out.end_array().next().end_object()\n\n return out()\n\n\nGEOCODEJSON_RANKS = {\n 3: 'locality',\n 4: 'country',\n 5: 'state', 6: 'state', 7: 'state', 8: 'state', 9: 'state',\n 10: 'county', 11: 'county', 12: 'county',\n 13: 'city', 14: 'city', 15: 'city', 16: 'city',\n 17: 'district', 18: 'district', 19: 'district', 20: 'district', 21: 'district',\n 22: 'locality', 23: 'locality', 24: 'locality',\n 25: 'street', 26: 'street', 27: 'street', 28: 'house'}\n"
},
{
"path": "src/nominatim_api/v1/format_xml.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelper functions for output of results in XML format.\n\"\"\"\nfrom typing import Mapping, Any, Optional, Union\nimport datetime as dt\nimport xml.etree.ElementTree as ET\n\nfrom ..results import AddressLines, ReverseResult, ReverseResults, \\\n SearchResult, SearchResults\nfrom . import classtypes as cl\nfrom .helpers import _add_admin_level\nfrom ..types import EntranceDetails\n\n\ndef _write_xml_address(root: ET.Element, address: AddressLines,\n country_code: Optional[str]) -> None:\n parts = {}\n for line in address:\n if line.isaddress:\n if line.local_name:\n label = cl.get_label_tag(line.category, line.extratags,\n line.rank_address, country_code)\n if label not in parts:\n parts[label] = line.local_name\n if line.names and 'ISO3166-2' in line.names and line.admin_level:\n parts[f\"ISO3166-2-lvl{line.admin_level}\"] = line.names['ISO3166-2']\n\n for k, v in parts.items():\n ET.SubElement(root, k).text = v\n\n if country_code:\n ET.SubElement(root, 'country_code').text = country_code\n\n\ndef _create_base_entry(result: Union[ReverseResult, SearchResult],\n root: ET.Element, simple: bool) -> ET.Element:\n place = ET.SubElement(root, 'result' if simple else 'place')\n if result.place_id is not None:\n place.set('place_id', str(result.place_id))\n if result.osm_object:\n osm_type = cl.OSM_TYPE_NAME.get(result.osm_object[0], None)\n if osm_type is not None:\n place.set('osm_type', osm_type)\n place.set('osm_id', str(result.osm_object[1]))\n if result.names and 'ref' in result.names:\n place.set('ref', result.names['ref'])\n elif result.locale_name:\n # bug reproduced from PHP\n place.set('ref', result.locale_name)\n place.set('lat', f\"{result.centroid.lat:.7f}\")\n place.set('lon', f\"{result.centroid.lon:.7f}\")\n\n bbox = cl.bbox_from_result(result)\n place.set('boundingbox',\n f\"{bbox.minlat:.7f},{bbox.maxlat:.7f},{bbox.minlon:.7f},{bbox.maxlon:.7f}\")\n\n place.set('place_rank', str(result.rank_search))\n place.set('address_rank', str(result.rank_address))\n\n if result.geometry:\n for key in ('text', 'svg'):\n if key in result.geometry:\n place.set('geo' + key, result.geometry[key])\n if 'kml' in result.geometry:\n ET.SubElement(root if simple else place, 'geokml')\\\n .append(ET.fromstring(result.geometry['kml']))\n if 'geojson' in result.geometry:\n place.set('geojson', result.geometry['geojson'])\n\n if simple:\n place.text = result.display_name or ''\n else:\n place.set('display_name', result.display_name or '')\n place.set('class', result.category[0])\n place.set('type', result.category[1])\n place.set('importance', str(result.calculated_importance()))\n\n return place\n\n\ndef _create_entrance(root: ET.Element, entrance: EntranceDetails) -> None:\n entrance_node = ET.SubElement(root, 'entrance', attrib={\n \"osm_id\": str(entrance.osm_id),\n \"type\": entrance.type,\n \"lat\": f\"{entrance.location.lat:0.7f}\",\n \"lon\": f\"{entrance.location.lon:0.7f}\",\n })\n if entrance.extratags:\n for k, v in entrance.extratags.items():\n ET.SubElement(entrance_node, 'tag', attrib={'key': k, 'value': v})\n\n\ndef format_base_xml(results: Union[ReverseResults, SearchResults],\n options: Mapping[str, Any],\n simple: bool, xml_root_tag: str,\n xml_extra_info: Mapping[str, str]) -> str:\n \"\"\" Format the result into an XML response. With 'simple' exactly one\n result will be output, otherwise a list.\n \"\"\"\n root = ET.Element(xml_root_tag)\n root.set('timestamp', dt.datetime.now(dt.timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +00:00'))\n root.set('attribution', cl.OSM_ATTRIBUTION)\n for k, v in xml_extra_info.items():\n root.set(k, v)\n\n if simple and not results:\n ET.SubElement(root, 'error').text = 'Unable to geocode'\n\n for result in results:\n place = _create_base_entry(result, root, simple)\n\n if not simple and options.get('icon_base_url', None):\n icon = cl.ICONS.get(result.category)\n if icon:\n place.set('icon', icon)\n\n if options.get('addressdetails', False) and result.address_rows:\n _write_xml_address(ET.SubElement(root, 'addressparts') if simple else place,\n result.address_rows, result.country_code)\n\n if options.get('extratags', False):\n eroot = ET.SubElement(root if simple else place, 'extratags')\n tags = _add_admin_level(result)\n if tags:\n for k, v in tags.items():\n ET.SubElement(eroot, 'tag', attrib={'key': k, 'value': v})\n\n if options.get('namedetails', False):\n eroot = ET.SubElement(root if simple else place, 'namedetails')\n if result.names:\n for k, v in result.names.items():\n ET.SubElement(eroot, 'name', attrib={'desc': k}).text = v\n\n if options.get('entrances', False):\n eroot = ET.SubElement(root if simple else place, 'entrances')\n if result.entrances:\n for entrance in result.entrances:\n _create_entrance(eroot, entrance)\n\n return '\\n' + ET.tostring(root, encoding='unicode')\n"
},
{
"path": "src/nominatim_api/v1/helpers.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelper function for parsing parameters and and outputting data\nspecifically for the v1 version of the API.\n\"\"\"\nfrom typing import Tuple, Optional, Any, Dict, Iterable\nfrom itertools import chain\nimport re\n\nfrom ..results import SearchResults, SourceTable, BaseResult\nfrom ..types import SearchDetails, GeometryFormat\n\n\ndef _add_admin_level(result: BaseResult) -> Optional[Dict[str, str]]:\n \"\"\" Inject admin_level into extratags for boundary=administrative results.\n \"\"\"\n tags = result.extratags\n if result.category == ('boundary', 'administrative') and result.admin_level < 15:\n tags = dict(tags) if tags else {}\n tags['admin_level'] = str(result.admin_level)\n return tags\n\n\nREVERSE_MAX_RANKS = [2, 2, 2, # 0-2 Continent/Sea\n 4, 4, # 3-4 Country\n 8, # 5 State\n 10, 10, # 6-7 Region\n 12, 12, # 8-9 County\n 16, 17, # 10-11 City\n 18, # 12 Town\n 19, # 13 Village/Suburb\n 22, # 14 Hamlet/Neighbourhood\n 25, # 15 Localities\n 26, # 16 Major Streets\n 27, # 17 Minor Streets\n 30 # 18 Building\n ]\n\n\ndef zoom_to_rank(zoom: int) -> int:\n \"\"\" Convert a zoom parameter into a rank according to the v1 API spec.\n \"\"\"\n return REVERSE_MAX_RANKS[max(0, min(18, zoom))]\n\n\nFEATURE_TYPE_TO_RANK: Dict[Optional[str], Tuple[int, int]] = {\n 'country': (4, 4),\n 'state': (8, 8),\n 'city': (14, 16),\n 'settlement': (8, 20)\n}\n\n\ndef feature_type_to_rank(feature_type: Optional[str]) -> Tuple[int, int]:\n \"\"\" Convert a feature type parameter to a tuple of\n feature type name, minimum rank and maximum rank.\n \"\"\"\n return FEATURE_TYPE_TO_RANK.get(feature_type, (0, 30))\n\n\ndef extend_query_parts(queryparts: Dict[str, Any], details: Dict[str, Any],\n feature_type: Optional[str],\n namedetails: bool, extratags: bool,\n excluded: Iterable[str]) -> None:\n \"\"\" Add parameters from details dictionary to the query parts\n dictionary which is suitable as URL parameter dictionary.\n \"\"\"\n parsed = SearchDetails.from_kwargs(details)\n if parsed.geometry_output != GeometryFormat.NONE:\n if GeometryFormat.GEOJSON & parsed.geometry_output:\n queryparts['polygon_geojson'] = '1'\n if GeometryFormat.KML & parsed.geometry_output:\n queryparts['polygon_kml'] = '1'\n if GeometryFormat.SVG & parsed.geometry_output:\n queryparts['polygon_svg'] = '1'\n if GeometryFormat.TEXT & parsed.geometry_output:\n queryparts['polygon_text'] = '1'\n if parsed.address_details:\n queryparts['addressdetails'] = '1'\n if parsed.entrances:\n queryparts['entrances'] = '1'\n if namedetails:\n queryparts['namedetails'] = '1'\n if extratags:\n queryparts['extratags'] = '1'\n if parsed.geometry_simplification > 0.0:\n queryparts['polygon_threshold'] = f\"{parsed.geometry_simplification:.6g}\"\n if parsed.max_results != 10:\n queryparts['limit'] = str(parsed.max_results)\n if parsed.countries:\n queryparts['countrycodes'] = ','.join(parsed.countries)\n queryparts['exclude_place_ids'] = \\\n ','.join(chain(excluded,\n (str(e) for e in parsed.excluded)))\n if parsed.viewbox:\n queryparts['viewbox'] = ','.join(f\"{c:.7g}\" for c in parsed.viewbox.coords)\n if parsed.bounded_viewbox:\n queryparts['bounded'] = '1'\n if not details['dedupe']:\n queryparts['dedupe'] = '0'\n if feature_type in FEATURE_TYPE_TO_RANK:\n queryparts['featureType'] = feature_type\n\n\ndef deduplicate_results(results: SearchResults, max_results: int) -> SearchResults:\n \"\"\" Remove results that look like duplicates.\n\n Two results are considered the same if they have the same OSM ID\n or if they have the same category, display name and rank.\n \"\"\"\n osm_ids_done = set()\n classification_done = set()\n deduped = SearchResults()\n for result in results:\n if result.source_table == SourceTable.PLACEX:\n classification = (result.osm_object[0] if result.osm_object else None,\n result.category,\n result.display_name,\n result.rank_address)\n if result.osm_object not in osm_ids_done \\\n and classification not in classification_done:\n deduped.append(result)\n osm_ids_done.add(result.osm_object)\n classification_done.add(classification)\n else:\n deduped.append(result)\n if len(deduped) >= max_results:\n break\n\n return deduped\n\n\ndef _deg(axis: str) -> str:\n return f\"(?P<{axis}_deg>\\\\d+\\\\.\\\\d+)°?\"\n\n\ndef _deg_min(axis: str) -> str:\n return f\"(?P<{axis}_deg>\\\\d+)[°\\\\s]+(?P<{axis}_min>[\\\\d.]+)[′']*\"\n\n\ndef _deg_min_sec(axis: str) -> str:\n return f\"(?P<{axis}_deg>\\\\d+)[°\\\\s]+(?P<{axis}_min>\\\\d+)[′'\\\\s]+(?P<{axis}_sec>[\\\\d.]+)[\\\"″]*\"\n\n\nCOORD_REGEX = [re.compile(r'(?:(?P.*?)\\s+)??' + r + r'(?:\\s+(?P.*))?') for r in (\n r\"(?P[NS])\\s*\" + _deg('lat') + r\"[\\s,]+\" + r\"(?P[EW])\\s*\" + _deg('lon'),\n _deg('lat') + r\"\\s*(?P[NS])[\\s,]+\" + _deg('lon') + r\"\\s*(?P[EW])\",\n r\"(?P[NS])\\s*\" + _deg_min('lat') + r\"[\\s,]+\" + r\"(?P[EW])\\s*\" + _deg_min('lon'),\n _deg_min('lat') + r\"\\s*(?P[NS])[\\s,]+\" + _deg_min('lon') + r\"\\s*(?P[EW])\",\n r\"(?P[NS])\\s*\" + _deg_min_sec('lat') + r\"[\\s,]+\" + r\"(?P[EW])\\s*\" + _deg_min_sec('lon'),\n _deg_min_sec('lat') + r\"\\s*(?P[NS])[\\s,]+\" + _deg_min_sec('lon') + r\"\\s*(?P[EW])\",\n r\"\\[?(?P[+-]?\\d+\\.\\d+)[\\s,]+(?P[+-]?\\d+\\.\\d+)\\]?\"\n)]\n\n\ndef extract_coords_from_query(query: str) -> Tuple[str, Optional[float], Optional[float]]:\n \"\"\" Look for something that is formatted like a coordinate at the\n beginning or end of the query. If found, extract the coordinate and\n return the remaining query (or the empty string if the query\n consisted of nothing but a coordinate).\n\n Only the first match will be returned.\n \"\"\"\n for regex in COORD_REGEX:\n match = regex.fullmatch(query)\n if match is None:\n continue\n groups = match.groupdict()\n if not groups['pre'] or not groups['post']:\n x = float(groups['lon_deg']) \\\n + float(groups.get('lon_min', 0.0)) / 60.0 \\\n + float(groups.get('lon_sec', 0.0)) / 3600.0\n if groups.get('ew') == 'W':\n x = -x\n y = float(groups['lat_deg']) \\\n + float(groups.get('lat_min', 0.0)) / 60.0 \\\n + float(groups.get('lat_sec', 0.0)) / 3600.0\n if groups.get('ns') == 'S':\n y = -y\n return groups['pre'] or groups['post'] or '', x, y\n\n return query, None, None\n\n\nCATEGORY_REGEX = re.compile(r'(?P.*?)\\[(?P[a-zA-Z_]+)=(?P[a-zA-Z_]+)\\](?P.*)')\n\n\ndef extract_category_from_query(query: str) -> Tuple[str, Optional[str], Optional[str]]:\n \"\"\" Extract a hidden category specification of the form '[key=value]' from\n the query. If found, extract key and value and\n return the remaining query (or the empty string if the query\n consisted of nothing but a category).\n\n Only the first match will be returned.\n \"\"\"\n match = CATEGORY_REGEX.search(query)\n if match is not None:\n return (match.group('pre').strip() + ' ' + match.group('post').strip()).strip(), \\\n match.group('cls'), match.group('typ')\n\n return query, None, None\n"
},
{
"path": "src/nominatim_api/v1/server_glue.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nGeneric part of the server implementation of the v1 API.\nCombine with the scaffolding provided for the various Python ASGI frameworks.\n\"\"\"\nfrom typing import Optional, Any, Type, Dict, cast, Sequence, Tuple\nfrom functools import reduce\nimport dataclasses\nfrom urllib.parse import urlencode\nimport asyncio\n\nimport sqlalchemy as sa\n\nfrom ..errors import UsageError\nfrom .. import logging as loglib\nfrom ..core import NominatimAPIAsync\nfrom .format import RawDataList\nfrom ..types import DataLayer, GeometryFormat, PlaceRef, PlaceID, OsmID, Point\nfrom ..status import StatusResult\nfrom ..results import DetailedResult, ReverseResults, SearchResult, SearchResults\nfrom ..localization import Locales\nfrom . import helpers\nfrom ..server import content_types as ct\nfrom ..server.asgi_adaptor import ASGIAdaptor, EndpointFunc\nfrom ..sql.async_core_library import PGCORE_ERROR\n\n\ndef build_response(adaptor: ASGIAdaptor, output: str, status: int = 200,\n num_results: int = 0) -> Any:\n \"\"\" Create a response from the given output. Wraps a JSONP function\n around the response, if necessary.\n \"\"\"\n if adaptor.content_type == ct.CONTENT_JSON and status == 200:\n jsonp = adaptor.get('json_callback')\n if jsonp is not None:\n if any(not part.isidentifier() for part in jsonp.split('.')):\n adaptor.raise_error('Invalid json_callback value')\n output = f\"{jsonp}({output})\"\n adaptor.content_type = 'application/javascript; charset=utf-8'\n\n return adaptor.create_response(status, output, num_results)\n\n\ndef get_accepted_languages(adaptor: ASGIAdaptor) -> str:\n \"\"\" Return the accepted languages.\n \"\"\"\n return adaptor.get('accept-language')\\\n or adaptor.get_header('accept-language')\\\n or adaptor.config().DEFAULT_LANGUAGE\n\n\ndef setup_debugging(adaptor: ASGIAdaptor) -> bool:\n \"\"\" Set up collection of debug information if requested.\n\n Return True when debugging was requested.\n \"\"\"\n if adaptor.get_bool('debug', False):\n loglib.set_log_output('html')\n adaptor.content_type = ct.CONTENT_HTML\n return True\n\n return False\n\n\ndef get_layers(adaptor: ASGIAdaptor) -> Optional[DataLayer]:\n \"\"\" Return a parsed version of the layer parameter.\n \"\"\"\n param = adaptor.get('layer', None)\n if param is None:\n return None\n\n return cast(DataLayer,\n reduce(DataLayer.__or__,\n (getattr(DataLayer, s.upper()) for s in param.split(','))))\n\n\ndef parse_format(adaptor: ASGIAdaptor, result_type: Type[Any], default: str) -> str:\n \"\"\" Get and check the 'format' parameter and prepare the formatter.\n `result_type` is the type of result to be returned by the function\n and `default` the format value to assume when no parameter is present.\n \"\"\"\n fmt = adaptor.get('format', default=default)\n assert fmt is not None\n\n formatting = adaptor.formatting()\n\n if not formatting.supports_format(result_type, fmt):\n adaptor.raise_error(\"Parameter 'format' must be one of: \" +\n ', '.join(formatting.list_formats(result_type)))\n\n adaptor.content_type = formatting.get_content_type(fmt)\n return fmt\n\n\ndef parse_geometry_details(adaptor: ASGIAdaptor, fmt: str) -> Dict[str, Any]:\n \"\"\" Create details structure from the supplied geometry parameters.\n \"\"\"\n numgeoms = 0\n output = GeometryFormat.NONE\n if adaptor.get_bool('polygon_geojson', False):\n output |= GeometryFormat.GEOJSON\n numgeoms += 1\n if fmt not in ('geojson', 'geocodejson'):\n if adaptor.get_bool('polygon_text', False):\n output |= GeometryFormat.TEXT\n numgeoms += 1\n if adaptor.get_bool('polygon_kml', False):\n output |= GeometryFormat.KML\n numgeoms += 1\n if adaptor.get_bool('polygon_svg', False):\n output |= GeometryFormat.SVG\n numgeoms += 1\n\n if numgeoms > adaptor.config().get_int('POLYGON_OUTPUT_MAX_TYPES'):\n adaptor.raise_error('Too many polygon output options selected.')\n\n return {'address_details': True,\n 'geometry_simplification': adaptor.get_float('polygon_threshold', 0.0),\n 'geometry_output': output\n }\n\n\ndef has_search_name(conn: sa.engine.Connection) -> bool:\n \"\"\" Check if the search_name table exists in the database.\n \"\"\"\n return sa.inspect(conn).has_table('search_name')\n\n\nasync def status_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /status endpoint. See API docs for details.\n \"\"\"\n result = await api.status()\n\n fmt = parse_format(params, StatusResult, 'text')\n\n if fmt == 'text' and result.status:\n status_code = 500\n else:\n status_code = 200\n\n return build_response(params, params.formatting().format_result(result, fmt, {}),\n status=status_code)\n\n\nasync def details_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /details endpoint. See API docs for details.\n \"\"\"\n fmt = parse_format(params, DetailedResult, 'json')\n place_id = params.get_int('place_id', 0)\n place: PlaceRef\n if place_id:\n place = PlaceID(place_id)\n else:\n osmtype = params.get('osmtype')\n if osmtype is None:\n params.raise_error(\"Missing ID parameter 'place_id' or 'osmtype'.\")\n place = OsmID(osmtype, params.get_int('osmid'), params.get('class'))\n\n debug = setup_debugging(params)\n\n result = await api.details(place,\n address_details=params.get_bool('addressdetails', False),\n entrances=params.get_bool('entrances', False),\n linked_places=params.get_bool('linkedplaces', True),\n parented_places=params.get_bool('hierarchy', False),\n keywords=params.get_bool('keywords', False),\n geometry_output=(GeometryFormat.GEOJSON\n if params.get_bool('polygon_geojson', False)\n else GeometryFormat.NONE),\n query_stats=params.query_stats()\n )\n\n if debug:\n return build_response(params, loglib.get_and_disable())\n\n if result is None:\n params.raise_error('No place with that OSM ID found.', status=404)\n\n locales = Locales.from_accept_languages(get_accepted_languages(params),\n params.config().OUTPUT_NAMES)\n locales.localize_results([result])\n\n output = params.formatting().format_result(\n result, fmt,\n {'locales': locales,\n 'group_hierarchy': params.get_bool('group_hierarchy', False),\n 'icon_base_url': params.config().MAPICON_URL,\n 'entrances': params.get_bool('entrances', False),\n })\n\n return build_response(params, output, num_results=1)\n\n\nasync def reverse_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /reverse endpoint. See API docs for details.\n \"\"\"\n fmt = parse_format(params, ReverseResults, 'xml')\n debug = setup_debugging(params)\n coord = Point(params.get_float('lon'), params.get_float('lat'))\n\n details = parse_geometry_details(params, fmt)\n details['max_rank'] = helpers.zoom_to_rank(params.get_int('zoom', 18))\n details['layers'] = get_layers(params)\n details['query_stats'] = params.query_stats()\n details['entrances'] = params.get_bool('entrances', False)\n\n result = await api.reverse(coord, **details)\n\n if debug:\n return build_response(params, loglib.get_and_disable(), num_results=1 if result else 0)\n\n if fmt == 'xml':\n queryparts = {'lat': str(coord.lat), 'lon': str(coord.lon), 'format': 'xml'}\n zoom = params.get('zoom', None)\n if zoom:\n queryparts['zoom'] = zoom\n query = urlencode(queryparts)\n else:\n query = ''\n\n if result:\n Locales.from_accept_languages(get_accepted_languages(params),\n params.config().OUTPUT_NAMES).localize_results([result])\n\n fmt_options = {'query': query,\n 'extratags': params.get_bool('extratags', False),\n 'namedetails': params.get_bool('namedetails', False),\n 'entrances': params.get_bool('entrances', False),\n 'addressdetails': params.get_bool('addressdetails', True)}\n\n output = params.formatting().format_result(ReverseResults([result] if result else []),\n fmt, fmt_options)\n\n return build_response(params, output, num_results=1 if result else 0)\n\n\nasync def lookup_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /lookup endpoint. See API docs for details.\n \"\"\"\n fmt = parse_format(params, SearchResults, 'xml')\n debug = setup_debugging(params)\n details = parse_geometry_details(params, fmt)\n details['query_stats'] = params.query_stats()\n details['entrances'] = params.get_bool('entrances', False)\n\n places = []\n for oid in (params.get('osm_ids') or '').split(','):\n oid = oid.strip()\n if len(oid) > 1 and oid[0] in 'RNWrnw' and oid[1:].isdigit():\n places.append(OsmID(oid[0].upper(), int(oid[1:])))\n\n if len(places) > params.config().get_int('LOOKUP_MAX_COUNT'):\n params.raise_error('Too many object IDs.')\n\n if places:\n results = await api.lookup(places, **details)\n else:\n results = SearchResults()\n\n if debug:\n return build_response(params, loglib.get_and_disable(), num_results=len(results))\n\n Locales.from_accept_languages(get_accepted_languages(params),\n params.config().OUTPUT_NAMES).localize_results(results)\n\n fmt_options = {'extratags': params.get_bool('extratags', False),\n 'namedetails': params.get_bool('namedetails', False),\n 'entrances': params.get_bool('entrances', False),\n 'addressdetails': params.get_bool('addressdetails', True)}\n\n output = params.formatting().format_result(results, fmt, fmt_options)\n\n return build_response(params, output, num_results=len(results))\n\n\nasync def _unstructured_search(query: str, api: NominatimAPIAsync,\n details: Dict[str, Any]) -> SearchResults:\n if not query:\n return SearchResults()\n\n # Extract special format for coordinates from query.\n query, x, y = helpers.extract_coords_from_query(query)\n if x is not None:\n assert y is not None\n details['near'] = Point(x, y)\n details['near_radius'] = 0.1\n\n # If no query is left, revert to reverse search.\n if x is not None and not query:\n result = await api.reverse(details['near'], **details)\n if not result:\n return SearchResults()\n\n return SearchResults(\n [SearchResult(**{f.name: getattr(result, f.name)\n for f in dataclasses.fields(SearchResult)\n if hasattr(result, f.name)})])\n\n query, cls, typ = helpers.extract_category_from_query(query)\n if cls is not None:\n assert typ is not None\n return await api.search_category([(cls, typ)], near_query=query, **details)\n\n return await api.search(query, **details)\n\n\nasync def search_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /search endpoint. See API docs for details.\n \"\"\"\n fmt = parse_format(params, SearchResults, 'jsonv2')\n debug = setup_debugging(params)\n details = parse_geometry_details(params, fmt)\n\n details['query_stats'] = params.query_stats()\n details['countries'] = params.get('countrycodes', None)\n details['entrances'] = params.get_bool('entrances', False)\n details['excluded'] = params.get('exclude_place_ids', None)\n details['viewbox'] = params.get('viewbox', None) or params.get('viewboxlbrt', None)\n details['bounded_viewbox'] = params.get_bool('bounded', False)\n details['dedupe'] = params.get_bool('dedupe', True)\n\n max_results = max(1, min(50, params.get_int('limit', 10)))\n details['max_results'] = (max_results + min(10, max_results)\n if details['dedupe'] else max_results)\n\n details['min_rank'], details['max_rank'] = \\\n helpers.feature_type_to_rank(params.get('featureType', ''))\n if params.get('featureType', None) is not None:\n details['layers'] = DataLayer.ADDRESS\n else:\n details['layers'] = get_layers(params)\n details['locales'] = Locales.from_accept_languages(get_accepted_languages(params),\n params.config().OUTPUT_NAMES)\n\n # unstructured query parameters\n query = params.get('q', None)\n # structured query parameters\n queryparts = {}\n for key in ('amenity', 'street', 'city', 'county', 'state', 'postalcode', 'country'):\n details[key] = params.get(key, None)\n if details[key]:\n queryparts[key] = details[key]\n\n try:\n if query is not None:\n if queryparts:\n params.raise_error(\"Structured query parameters\"\n \"(amenity, street, city, county, state, postalcode, country)\"\n \" cannot be used together with 'q' parameter.\")\n queryparts['q'] = query\n results = await _unstructured_search(query, api, details)\n else:\n query = ', '.join(queryparts.values())\n\n results = await api.search_address(**details)\n except UsageError as err:\n params.raise_error(str(err))\n\n details['locales'].localize_results(results)\n\n if details['dedupe'] and len(results) > 1:\n results = helpers.deduplicate_results(results, max_results)\n\n if debug:\n return build_response(params, loglib.get_and_disable(), num_results=len(results))\n\n if fmt == 'xml':\n helpers.extend_query_parts(queryparts, details,\n params.get('featureType', ''),\n params.get_bool('namedetails', False),\n params.get_bool('extratags', False),\n (f\"{r.osm_object[0]}{r.osm_object[1]}\"\n if r.osm_object\n else str(r.place_id)\n for r in results\n if r.osm_object or r.place_id))\n queryparts['format'] = fmt\n\n moreurl = params.base_uri() + '/search?' + urlencode(queryparts)\n else:\n moreurl = ''\n\n fmt_options = {'query': query, 'more_url': moreurl,\n 'exclude_place_ids': queryparts.get('exclude_place_ids'),\n 'viewbox': queryparts.get('viewbox'),\n 'extratags': params.get_bool('extratags', False),\n 'namedetails': params.get_bool('namedetails', False),\n 'entrances': params.get_bool('entrances', False),\n 'addressdetails': params.get_bool('addressdetails', False)}\n\n output = params.formatting().format_result(results, fmt, fmt_options)\n\n return build_response(params, output, num_results=len(results))\n\n\nasync def deletable_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /deletable endpoint.\n This is a special endpoint that shows polygons that have been\n deleted or are broken in the OSM data but are kept in the\n Nominatim database to minimize disruption.\n \"\"\"\n fmt = parse_format(params, RawDataList, 'json')\n\n results = RawDataList()\n async with api.begin() as conn:\n for osm_type in ('N', 'W', 'R'):\n sql = sa.text(\"\"\" SELECT p.place_id, country_code,\n name->'name' as name, i.*\n FROM placex p, import_polygon_delete i\n WHERE i.osm_type = :osm_type\n AND p.osm_id = i.osm_id AND p.osm_type = :osm_type\n AND p.class = i.class AND p.type = i.type\n \"\"\")\n results.extend(r._asdict() for r in await conn.execute(sql, {'osm_type': osm_type}))\n\n return build_response(params, params.formatting().format_result(results, fmt, {}))\n\n\nasync def polygons_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /polygons endpoint.\n This is a special endpoint that shows polygons that have changed\n their size but are kept in the Nominatim database with their\n old area to minimize disruption.\n \"\"\"\n fmt = parse_format(params, RawDataList, 'json')\n sql_params: Dict[str, Any] = {\n 'days': params.get_int('days', -1),\n 'cls': params.get('class')\n }\n reduced = params.get_bool('reduced', False)\n\n async with api.begin() as conn:\n sql = sa.select(sa.text(\"\"\"osm_type, osm_id, class, type,\n name->'name' as name,\n country_code, errormessage, updated\"\"\"))\\\n .select_from(sa.text('import_polygon_error'))\n if sql_params['days'] > 0:\n sql = sql.where(sa.text(\"updated > 'now'::timestamp - make_interval(days => :days)\"))\n if reduced:\n sql = sql.where(sa.text(\"errormessage like 'Area reduced%'\"))\n if sql_params['cls'] is not None:\n sql = sql.where(sa.text(\"class = :cls\"))\n\n sql = sql.order_by(sa.literal_column('updated').desc()).limit(1000)\n\n results = RawDataList(r._asdict() for r in await conn.execute(sql, sql_params))\n\n return build_response(params, params.formatting().format_result(results, fmt, {}))\n\n\nasync def search_unavailable_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n \"\"\" Server glue for /search endpoint in reverse-only mode.\n Returns 404 when search functionality is not available.\n \"\"\"\n params.raise_error('Search not available (reverse-only mode)', 404)\n\n\nclass LazySearchEndpoint:\n \"\"\"\n Lazy-loading search endpoint that replaces itself after first successful check.\n\n - Falcon: EndpointWrapper stores this instance in wrapper.func\n On first request, replace wrapper.func directly with real endpoint\n\n - Starlette: _wrap_endpoint wraps this instance in a callback\n store a delegate function and call it on subsequent requests\n \"\"\"\n def __init__(self, api: NominatimAPIAsync, real_endpoint: EndpointFunc):\n self.api = api\n self.real_endpoint = real_endpoint\n self._lock = asyncio.Lock()\n self._wrapper: Any = None # Store reference to Falcon's EndpointWrapper\n self._delegate: Optional[EndpointFunc] = None\n\n def set_wrapper(self, wrapper: Any) -> None:\n self._wrapper = wrapper\n\n async def __call__(self, api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:\n if self._delegate is None:\n async with self._lock:\n # Double-check after acquiring lock (thread safety)\n if self._delegate is None:\n try:\n async with api.begin() as conn:\n has_table = await conn.connection.run_sync(\n has_search_name)\n\n if has_table:\n # For Starlette\n self._delegate = self.real_endpoint\n # For Falcon\n if self._wrapper is not None:\n self._wrapper.func = self.real_endpoint\n else:\n self._delegate = search_unavailable_endpoint\n if self._wrapper is not None:\n self._wrapper.func = search_unavailable_endpoint\n\n except (PGCORE_ERROR, sa.exc.OperationalError, OSError):\n # No _delegate set, so retry on next request\n params.raise_error('Search temporarily unavailable', 503)\n\n return await self._delegate(api, params)\n\n\nasync def get_routes(api: NominatimAPIAsync) -> Sequence[Tuple[str, EndpointFunc]]:\n routes = [\n ('status', status_endpoint),\n ('details', details_endpoint),\n ('reverse', reverse_endpoint),\n ('lookup', lookup_endpoint),\n ('deletable', deletable_endpoint),\n ('polygons', polygons_endpoint),\n ]\n\n try:\n async with api.begin() as conn:\n if await conn.connection.run_sync(has_search_name):\n routes.append(('search', search_endpoint))\n else:\n routes.append(('search', search_unavailable_endpoint))\n except (PGCORE_ERROR, sa.exc.OperationalError, OSError):\n routes.append(('search', LazySearchEndpoint(api, search_endpoint)))\n\n return routes\n"
},
{
"path": "src/nominatim_api/version.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nVersion information for the Nominatim API.\n\"\"\"\n\nNOMINATIM_API_VERSION = '5.3.0'\n"
},
{
"path": "src/nominatim_db/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/cli.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nCommand-line interface to the Nominatim functions for import, update,\ndatabase administration and querying.\n\"\"\"\nfrom typing import Optional, List, Mapping\nimport importlib\nimport logging\nimport sys\nimport argparse\nimport asyncio\nfrom pathlib import Path\n\nfrom .config import Configuration\nfrom .errors import UsageError\nfrom . import clicmd\nfrom . import version\nfrom .clicmd.args import NominatimArgs, Subcommand\n\nLOG = logging.getLogger()\n\n\nclass CommandlineParser:\n \"\"\" Wraps some of the common functions for parsing the command line\n and setting up subcommands.\n \"\"\"\n def __init__(self, prog: str, description: Optional[str]):\n self.parser = argparse.ArgumentParser(\n prog=prog,\n description=description,\n formatter_class=argparse.RawDescriptionHelpFormatter)\n\n self.subs = self.parser.add_subparsers(title='available commands',\n dest='subcommand')\n\n # Global arguments that only work if no sub-command given\n self.parser.add_argument('--version', action='store_true',\n help='Print Nominatim version and exit')\n\n # Arguments added to every sub-command\n self.default_args = argparse.ArgumentParser(add_help=False)\n group = self.default_args.add_argument_group('Default arguments')\n group.add_argument('-h', '--help', action='help',\n help='Show this help message and exit')\n group.add_argument('-q', '--quiet', action='store_const', const=0,\n dest='verbose', default=1,\n help='Print only error messages')\n group.add_argument('-v', '--verbose', action='count', default=1,\n help='Increase verboseness of output')\n group.add_argument('--project-dir', metavar='DIR', default='.',\n help='Base directory of the Nominatim installation (default:.)')\n group.add_argument('-j', '--threads', metavar='NUM', type=int,\n help='Number of parallel threads to use')\n\n def nominatim_version_text(self) -> str:\n \"\"\" Program name and version number as string\n \"\"\"\n text = f'Nominatim version {version.NOMINATIM_VERSION!s}'\n if version.GIT_COMMIT_HASH is not None:\n text += f' ({version.GIT_COMMIT_HASH})'\n return text\n\n def add_subcommand(self, name: str, cmd: Subcommand) -> None:\n \"\"\" Add a subcommand to the parser. The subcommand must be a class\n with a function add_args() that adds the parameters for the\n subcommand and a run() function that executes the command.\n \"\"\"\n assert cmd.__doc__ is not None\n\n parser = self.subs.add_parser(name, parents=[self.default_args],\n help=cmd.__doc__.split('\\n', 1)[0],\n description=cmd.__doc__,\n formatter_class=argparse.RawDescriptionHelpFormatter,\n add_help=False)\n parser.set_defaults(command=cmd)\n cmd.add_args(parser)\n\n def run(self, cli_args: Optional[List[str]],\n environ: Optional[Mapping[str, str]]) -> int:\n \"\"\" Parse the command line arguments of the program and execute the\n appropriate subcommand.\n \"\"\"\n args = NominatimArgs()\n try:\n self.parser.parse_args(args=cli_args, namespace=args)\n except SystemExit:\n return 1\n\n if args.version:\n print(self.nominatim_version_text())\n return 0\n\n if args.subcommand is None:\n self.parser.print_help()\n return 1\n\n args.project_dir = Path(args.project_dir).resolve()\n\n if cli_args is None:\n logging.basicConfig(stream=sys.stderr,\n format='%(asctime)s: %(message)s',\n datefmt='%Y-%m-%d %H:%M:%S',\n level=max(4 - args.verbose, 1) * 10)\n\n args.config = Configuration(args.project_dir, environ=environ)\n\n log = logging.getLogger()\n log.warning('Using project directory: %s', str(args.project_dir))\n\n try:\n return args.command.run(args)\n except UsageError as exception:\n if log.isEnabledFor(logging.DEBUG):\n raise # use Python's exception printing\n log.fatal('FATAL: %s', exception)\n\n # If we get here, then execution has failed in some way.\n return 1\n\n\n# Subcommand classes\n#\n# Each class needs to implement two functions: add_args() adds the CLI parameters\n# for the subfunction, run() executes the subcommand.\n#\n# The class documentation doubles as the help text for the command. The\n# first line is also used in the summary when calling the program without\n# a subcommand.\n#\n# No need to document the functions each time.\nclass AdminServe:\n \"\"\"\\\n Start a simple web server for serving the API.\n\n This command starts a built-in webserver to serve the website\n from the current project directory. This webserver is only suitable\n for testing and development. Do not use it in production setups!\n\n There are two different webserver implementations for Python available:\n falcon (the default) and starlette. You need to make sure the\n appropriate Python packages as well as the uvicorn package are\n installed to use this function.\n\n By the default, the webserver can be accessed at: http://127.0.0.1:8088\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Server arguments')\n group.add_argument('--server', default='127.0.0.1:8088',\n help='The address the server will listen to.')\n group.add_argument('--engine', default='falcon',\n choices=('falcon', 'starlette'),\n help='Webserver framework to run. (default: falcon)')\n\n def run(self, args: NominatimArgs) -> int:\n asyncio.run(self.run_uvicorn(args))\n\n return 0\n\n async def run_uvicorn(self, args: NominatimArgs) -> None:\n import uvicorn\n\n server_info = args.server.split(':', 1)\n host = server_info[0]\n if len(server_info) > 1:\n if not server_info[1].isdigit():\n raise UsageError('Invalid format for --server parameter. Use :')\n port = int(server_info[1])\n else:\n port = 8088\n\n server_module = importlib.import_module(f'nominatim_api.server.{args.engine}.server')\n\n app = server_module.get_application(args.project_dir)\n\n config = uvicorn.Config(app, host=host, port=port)\n server = uvicorn.Server(config)\n await server.serve()\n\n\ndef get_set_parser() -> CommandlineParser:\n \"\"\"\\\n Initializes the parser and adds various subcommands for\n nominatim cli.\n \"\"\"\n parser = CommandlineParser('nominatim', nominatim.__doc__)\n\n parser.add_subcommand('import', clicmd.SetupAll())\n parser.add_subcommand('freeze', clicmd.SetupFreeze())\n parser.add_subcommand('replication', clicmd.UpdateReplication())\n\n parser.add_subcommand('special-phrases', clicmd.ImportSpecialPhrases())\n\n parser.add_subcommand('add-data', clicmd.UpdateAddData())\n parser.add_subcommand('index', clicmd.UpdateIndex())\n parser.add_subcommand('refresh', clicmd.UpdateRefresh())\n\n parser.add_subcommand('admin', clicmd.AdminFuncs())\n\n try:\n exportcmd = importlib.import_module('nominatim_db.clicmd.export')\n apicmd = importlib.import_module('nominatim_db.clicmd.api')\n convertcmd = importlib.import_module('nominatim_db.clicmd.convert')\n\n parser.add_subcommand('export', exportcmd.QueryExport())\n parser.add_subcommand('convert', convertcmd.ConvertDB())\n parser.add_subcommand('serve', AdminServe())\n\n parser.add_subcommand('search', apicmd.APISearch())\n parser.add_subcommand('reverse', apicmd.APIReverse())\n parser.add_subcommand('lookup', apicmd.APILookup())\n parser.add_subcommand('details', apicmd.APIDetails())\n parser.add_subcommand('status', apicmd.APIStatus())\n except ModuleNotFoundError as ex:\n if not ex.name or 'nominatim_api' not in ex.name:\n raise ex\n\n parser.parser.epilog = \\\n f'\\n\\nNominatim API package not found (was looking for module: {ex.name}).'\\\n '\\nThe following commands are not available:'\\\n '\\n export, convert, serve, search, reverse, lookup, details, status'\\\n \"\\n\\nRun 'pip install nominatim-api' to install the package.\"\n\n return parser\n\n\ndef nominatim(cli_args: Optional[List[str]] = None,\n environ: Optional[Mapping[str, str]] = None) -> int:\n \"\"\"\\\n Command-line tools for importing, updating, administrating and\n querying the Nominatim database.\n\n 'cli_args' is a list of parameters for the command to run. If not given,\n sys.args will be used.\n\n 'environ' is the dictionary of environment variables containing the\n Nominatim configuration. When None, the os.environ is inherited.\n \"\"\"\n return get_set_parser().run(cli_args=cli_args, environ=environ)\n"
},
{
"path": "src/nominatim_db/clicmd/__init__.py",
"content": "# SPDX-License-Identifier: GPL-2.0-only\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2023 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSubcommand definitions for the command-line tool.\n\"\"\"\n\nfrom .setup import SetupAll as SetupAll\nfrom .replication import UpdateReplication as UpdateReplication\nfrom .index import UpdateIndex as UpdateIndex\nfrom .refresh import UpdateRefresh as UpdateRefresh\nfrom .add_data import UpdateAddData as UpdateAddData\nfrom .admin import AdminFuncs as AdminFuncs\nfrom .freeze import SetupFreeze as SetupFreeze\nfrom .special_phrases import ImportSpecialPhrases as ImportSpecialPhrases\n"
},
{
"path": "src/nominatim_db/clicmd/add_data.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'add-data' subcommand.\n\"\"\"\nfrom typing import cast\nimport argparse\nimport logging\nimport asyncio\n\nimport psutil\n\nfrom .args import NominatimArgs\nfrom ..db.connection import connect\nfrom ..tools.freeze import is_frozen\n\n\nLOG = logging.getLogger()\n\n\nclass UpdateAddData:\n \"\"\"\\\n Add additional data from a file or an online source.\n\n This command allows to add or update the search data in the database.\n The data can come either from an OSM file or single OSM objects can\n directly be downloaded from the OSM API. This function only loads the\n data into the database. Afterwards it still needs to be integrated\n in the search index. Use the `nominatim index` command for that.\n\n The command can also be used to add external non-OSM data to the\n database. At the moment the only supported format is TIGER housenumber\n data. See the online documentation at\n https://nominatim.org/release-docs/latest/customize/Tiger/\n for more information.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group_name = parser.add_argument_group('Source')\n group1 = group_name.add_mutually_exclusive_group(required=True)\n group1.add_argument('--file', metavar='FILE',\n help='Import data from an OSM file or diff file')\n group1.add_argument('--diff', metavar='FILE',\n help='Import data from an OSM diff file (deprecated: use --file)')\n group1.add_argument('--node', metavar='ID', type=int,\n help='Import a single node from the API')\n group1.add_argument('--way', metavar='ID', type=int,\n help='Import a single way from the API')\n group1.add_argument('--relation', metavar='ID', type=int,\n help='Import a single relation from the API')\n group1.add_argument('--tiger-data', metavar='DIR',\n help='Add housenumbers from the US TIGER census database')\n group2 = parser.add_argument_group('Extra arguments')\n group2.add_argument('--use-main-api', action='store_true',\n help='Use OSM API instead of Overpass to download objects')\n group2.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,\n help='Size of cache to be used by osm2pgsql (in MB)')\n group2.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,\n help='Set timeout for file downloads')\n\n def run(self, args: NominatimArgs) -> int:\n from ..tools import add_osm_data\n\n if args.tiger_data:\n return asyncio.run(self._add_tiger_data(args))\n\n with connect(args.config.get_libpq_dsn()) as conn:\n if is_frozen(conn):\n print('Database is marked frozen. New data can\\'t be added.')\n return 1\n\n osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)\n if args.file or args.diff:\n return add_osm_data.add_data_from_file(args.config.get_libpq_dsn(),\n cast(str, args.file or args.diff),\n osm2pgsql_params)\n\n if args.node:\n return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),\n 'node', args.node,\n args.use_main_api,\n osm2pgsql_params)\n\n if args.way:\n return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),\n 'way', args.way,\n args.use_main_api,\n osm2pgsql_params)\n\n if args.relation:\n return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),\n 'relation', args.relation,\n args.use_main_api,\n osm2pgsql_params)\n\n return 0\n\n async def _add_tiger_data(self, args: NominatimArgs) -> int:\n from ..tokenizer import factory as tokenizer_factory\n from ..tools import tiger_data\n\n assert args.tiger_data\n\n tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)\n return await tiger_data.add_tiger_data(args.tiger_data,\n args.config,\n args.threads or psutil.cpu_count() or 1,\n tokenizer)\n"
},
{
"path": "src/nominatim_db/clicmd/admin.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'admin' subcommand.\n\"\"\"\nimport logging\nimport argparse\nimport random\n\nfrom ..errors import UsageError\nfrom ..db.connection import connect, table_exists\nfrom .args import NominatimArgs\n\n\nLOG = logging.getLogger()\n\n\nclass AdminFuncs:\n \"\"\"\\\n Analyse and maintain the database.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Admin tasks')\n objs = group.add_mutually_exclusive_group(required=True)\n objs.add_argument('--warm', action='store_true',\n help='Warm database caches for search and reverse queries')\n objs.add_argument('--check-database', action='store_true',\n help='Check that the database is complete and operational')\n objs.add_argument('--migrate', action='store_true',\n help='Migrate the database to a new software version')\n objs.add_argument('--analyse-indexing', action='store_true',\n help='Print performance analysis of the indexing process')\n objs.add_argument('--collect-os-info', action=\"store_true\",\n help=\"Generate a report about the host system information\")\n objs.add_argument('--clean-deleted', action='store', metavar='AGE',\n help='Clean up deleted relations')\n group = parser.add_argument_group('Arguments for cache warming')\n group.add_argument('--search-only', action='store_const', dest='target',\n const='search',\n help=\"Only pre-warm tables for search queries\")\n group.add_argument('--reverse-only', action='store_const', dest='target',\n const='reverse',\n help=\"Only pre-warm tables for reverse queries\")\n group = parser.add_argument_group('Arguments for index anaysis')\n mgroup = group.add_mutually_exclusive_group()\n mgroup.add_argument('--osm-id', type=str,\n help='Analyse indexing of the given OSM object')\n mgroup.add_argument('--place-id', type=int,\n help='Analyse indexing of the given Nominatim object')\n\n def run(self, args: NominatimArgs) -> int:\n if args.warm:\n return self._warm(args)\n\n if args.check_database:\n LOG.warning('Checking database')\n from ..tools import check_database\n return check_database.check_database(args.config)\n\n if args.analyse_indexing:\n LOG.warning('Analysing performance of indexing function')\n from ..tools import admin\n admin.analyse_indexing(args.config, osm_id=args.osm_id, place_id=args.place_id)\n return 0\n\n if args.migrate:\n LOG.warning('Checking for necessary database migrations')\n from ..tools import migration\n return migration.migrate(args.config, args)\n\n if args.collect_os_info:\n LOG.warning(\"Reporting System Information\")\n from ..tools import collect_os_info\n collect_os_info.report_system_information(args.config)\n return 0\n\n if args.clean_deleted:\n LOG.warning('Cleaning up deleted relations')\n from ..tools import admin\n admin.clean_deleted_relations(args.config, age=args.clean_deleted)\n return 0\n\n return 1\n\n def _warm(self, args: NominatimArgs) -> int:\n try:\n import nominatim_api as napi\n except ModuleNotFoundError as exp:\n raise UsageError(\"Warming requires nominatim API. \"\n \"Install with 'pip install nominatim-api'.\") from exp\n LOG.warning('Warming database caches')\n\n api = napi.NominatimAPI(args.project_dir)\n\n try:\n if args.target != 'search':\n for _ in range(1000):\n api.reverse((random.uniform(-90, 90), random.uniform(-180, 180)),\n address_details=True)\n\n if args.target != 'reverse':\n from ..tokenizer import factory as tokenizer_factory\n\n tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)\n with connect(args.config.get_libpq_dsn()) as conn:\n if table_exists(conn, 'search_name'):\n words = tokenizer.most_frequent_words(conn, 1000)\n else:\n words = []\n\n for word in words:\n api.search(word)\n finally:\n api.close()\n\n return 0\n"
},
{
"path": "src/nominatim_db/clicmd/api.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSubcommand definitions for API calls from the command line.\n\"\"\"\nfrom typing import Dict, Any, Optional, Type, Mapping\nimport argparse\nimport logging\nimport json\nimport sys\nimport pprint\nfrom functools import reduce\n\nimport nominatim_api as napi\nfrom nominatim_api.v1.helpers import zoom_to_rank, deduplicate_results\nfrom nominatim_api.server.content_types import CONTENT_JSON\nimport nominatim_api.logging as loglib\nfrom ..config import Configuration\nfrom ..errors import UsageError\nfrom .args import NominatimArgs\n\n\nLOG = logging.getLogger()\n\n\nSTRUCTURED_QUERY = (\n ('amenity', 'name and/or type of POI'),\n ('street', 'housenumber and street'),\n ('city', 'city, town or village'),\n ('county', 'county'),\n ('state', 'state'),\n ('country', 'country'),\n ('postalcode', 'postcode')\n)\n\n\nEXTRADATA_PARAMS = (\n ('addressdetails', 'Include a breakdown of the address into elements'),\n ('extratags', (\"Include additional information if available \"\n \"(e.g. wikipedia link, opening hours)\")),\n ('entrances', 'Include a list of tagged entrance nodes'),\n ('namedetails', 'Include a list of alternative names')\n)\n\n\ndef _add_list_format(parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Other options')\n group.add_argument('--list-formats', action='store_true',\n help='List supported output formats and exit.')\n\n\ndef _add_api_output_arguments(parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Output formatting')\n group.add_argument('--format', type=str, default='jsonv2',\n help='Format of result (use --list-format to see supported formats)')\n for name, desc in EXTRADATA_PARAMS:\n group.add_argument('--' + name, action='store_true', help=desc)\n\n group.add_argument('--lang', '--accept-language', metavar='LANGS',\n help='Preferred language order for presenting search results')\n group.add_argument('--polygon-output',\n choices=['geojson', 'kml', 'svg', 'text'],\n help='Output geometry of results as a GeoJSON, KML, SVG or WKT')\n group.add_argument('--polygon-threshold', type=float, default=0.0,\n metavar='TOLERANCE',\n help=(\"Simplify output geometry.\"\n \"Parameter is difference tolerance in degrees.\"))\n\n\ndef _get_geometry_output(args: NominatimArgs) -> napi.GeometryFormat:\n \"\"\" Get the requested geometry output format in a API-compatible\n format.\n \"\"\"\n if not args.polygon_output:\n return napi.GeometryFormat.NONE\n if args.polygon_output == 'geojson':\n return napi.GeometryFormat.GEOJSON\n if args.polygon_output == 'kml':\n return napi.GeometryFormat.KML\n if args.polygon_output == 'svg':\n return napi.GeometryFormat.SVG\n if args.polygon_output == 'text':\n return napi.GeometryFormat.TEXT\n\n try:\n return napi.GeometryFormat[args.polygon_output.upper()]\n except KeyError as exp:\n raise UsageError(f\"Unknown polygon output format '{args.polygon_output}'.\") from exp\n\n\ndef _get_locales(args: NominatimArgs, config: Configuration) -> napi.Locales:\n \"\"\" Get the locales from the language parameter.\n \"\"\"\n language = args.lang or config.DEFAULT_LANGUAGE\n output_names = config.OUTPUT_NAMES\n\n if language:\n return napi.Locales.from_accept_languages(language, output_names)\n\n return napi.Locales()\n\n\ndef _get_layers(args: NominatimArgs, default: Optional[napi.DataLayer]) -> Optional[napi.DataLayer]:\n \"\"\" Get the list of selected layers as a DataLayer enum.\n \"\"\"\n if not args.layers:\n return default\n\n return reduce(napi.DataLayer.__or__,\n (napi.DataLayer[s.upper()] for s in args.layers))\n\n\ndef _list_formats(formatter: napi.FormatDispatcher, rtype: Type[Any]) -> int:\n for fmt in formatter.list_formats(rtype):\n print(fmt)\n print('debug')\n print('raw')\n\n return 0\n\n\ndef _print_output(formatter: napi.FormatDispatcher, result: Any,\n fmt: str, options: Mapping[str, Any]) -> None:\n\n if fmt == 'raw':\n pprint.pprint(result)\n else:\n output = formatter.format_result(result, fmt, options)\n if formatter.get_content_type(fmt) == CONTENT_JSON:\n # reformat the result, so it is pretty-printed\n try:\n json.dump(json.loads(output), sys.stdout, indent=4, ensure_ascii=False)\n except json.decoder.JSONDecodeError as err:\n # Catch the error here, so that data can be debugged,\n # when people are developing custom result formatters.\n LOG.fatal(\"Parsing json failed: %s\\nUnformatted output:\\n%s\", err, output)\n else:\n sys.stdout.write(output)\n sys.stdout.write('\\n')\n\n\nclass APISearch:\n \"\"\"\\\n Execute a search query.\n\n This command works exactly the same as if calling the /search endpoint on\n the web API. See the online documentation for more details on the\n various parameters:\n https://nominatim.org/release-docs/latest/api/Search/\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Query arguments')\n group.add_argument('--query',\n help='Free-form query string')\n for name, desc in STRUCTURED_QUERY:\n group.add_argument('--' + name, help='Structured query: ' + desc)\n\n _add_api_output_arguments(parser)\n\n group = parser.add_argument_group('Result limitation')\n group.add_argument('--countrycodes', metavar='CC,..',\n help='Limit search results to one or more countries')\n group.add_argument('--exclude_place_ids', metavar='ID,..',\n help='List of search object to be excluded')\n group.add_argument('--limit', type=int, default=10,\n help='Limit the number of returned results')\n group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',\n help='Preferred area to find search results')\n group.add_argument('--bounded', action='store_true',\n help='Strictly restrict results to viewbox area')\n group.add_argument('--layer', metavar='LAYER',\n choices=[n.name.lower() for n in napi.DataLayer if n.name],\n action='append', required=False, dest='layers',\n help='Restrict results to one or more layers (may be repeated)')\n group.add_argument('--no-dedupe', action='store_false', dest='dedupe',\n help='Do not remove duplicates from the result list')\n _add_list_format(parser)\n\n def run(self, args: NominatimArgs) -> int:\n formatter = napi.load_format_dispatcher('v1', args.project_dir)\n\n if args.list_formats:\n return _list_formats(formatter, napi.SearchResults)\n\n if args.format in ('debug', 'raw'):\n loglib.set_log_output('text')\n elif not formatter.supports_format(napi.SearchResults, args.format):\n raise UsageError(f\"Unsupported format '{args.format}'. \"\n 'Use --list-formats to see supported formats.')\n\n layers = _get_layers(args, None)\n\n try:\n with napi.NominatimAPI(args.project_dir) as api:\n params: Dict[str, Any] = {'max_results': args.limit + min(args.limit, 10),\n 'address_details': True, # needed for display name\n 'geometry_output': _get_geometry_output(args),\n 'geometry_simplification': args.polygon_threshold,\n 'countries': args.countrycodes,\n 'excluded': args.exclude_place_ids,\n 'viewbox': args.viewbox,\n 'bounded_viewbox': args.bounded,\n 'layers': layers,\n 'entrances': args.entrances,\n }\n\n if args.query:\n results = api.search(args.query, **params)\n else:\n results = api.search_address(amenity=args.amenity,\n street=args.street,\n city=args.city,\n county=args.county,\n state=args.state,\n postalcode=args.postalcode,\n country=args.country,\n **params)\n except napi.UsageError as ex:\n raise UsageError(ex) from ex\n\n locales = _get_locales(args, api.config)\n locales.localize_results(results)\n\n if args.dedupe and len(results) > 1:\n results = deduplicate_results(results, args.limit)\n\n if args.format == 'debug':\n print(loglib.get_and_disable())\n return 0\n\n _print_output(formatter, results, args.format,\n {'extratags': args.extratags,\n 'namedetails': args.namedetails,\n 'entrances': args.entrances,\n 'addressdetails': args.addressdetails})\n return 0\n\n\nclass APIReverse:\n \"\"\"\\\n Execute API reverse query.\n\n This command works exactly the same as if calling the /reverse endpoint on\n the web API. See the online documentation for more details on the\n various parameters:\n https://nominatim.org/release-docs/latest/api/Reverse/\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Query arguments')\n group.add_argument('--lat', type=float,\n help='Latitude of coordinate to look up (in WGS84)')\n group.add_argument('--lon', type=float,\n help='Longitude of coordinate to look up (in WGS84)')\n group.add_argument('--zoom', type=int,\n help='Level of detail required for the address')\n group.add_argument('--layer', metavar='LAYER',\n choices=[n.name.lower() for n in napi.DataLayer if n.name],\n action='append', required=False, dest='layers',\n help='Restrict results to one or more layers (may be repeated)')\n\n _add_api_output_arguments(parser)\n _add_list_format(parser)\n\n def run(self, args: NominatimArgs) -> int:\n formatter = napi.load_format_dispatcher('v1', args.project_dir)\n\n if args.list_formats:\n return _list_formats(formatter, napi.ReverseResults)\n\n if args.format in ('debug', 'raw'):\n loglib.set_log_output('text')\n elif not formatter.supports_format(napi.ReverseResults, args.format):\n raise UsageError(f\"Unsupported format '{args.format}'. \"\n 'Use --list-formats to see supported formats.')\n\n if args.lat is None or args.lon is None:\n raise UsageError(\"lat' and 'lon' parameters are required.\")\n\n layers = _get_layers(args, napi.DataLayer.ADDRESS | napi.DataLayer.POI)\n\n try:\n with napi.NominatimAPI(args.project_dir) as api:\n result = api.reverse(napi.Point(args.lon, args.lat),\n max_rank=zoom_to_rank(args.zoom or 18),\n layers=layers,\n address_details=True, # needed for display name\n geometry_output=_get_geometry_output(args),\n geometry_simplification=args.polygon_threshold)\n except napi.UsageError as ex:\n raise UsageError(ex) from ex\n\n if result is not None:\n locales = _get_locales(args, api.config)\n locales.localize_results([result])\n\n if args.format == 'debug':\n print(loglib.get_and_disable())\n return 0\n\n if result:\n _print_output(formatter, napi.ReverseResults([result]), args.format,\n {'extratags': args.extratags,\n 'namedetails': args.namedetails,\n 'entrances': args.entrances,\n 'addressdetails': args.addressdetails})\n\n return 0\n\n LOG.error(\"Unable to geocode.\")\n return 42\n\n\nclass APILookup:\n \"\"\"\\\n Execute API lookup query.\n\n This command works exactly the same as if calling the /lookup endpoint on\n the web API. See the online documentation for more details on the\n various parameters:\n https://nominatim.org/release-docs/latest/api/Lookup/\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Query arguments')\n group.add_argument('--id', metavar='OSMID',\n action='append', dest='ids',\n help='OSM id to lookup in format (may be repeated)')\n\n _add_api_output_arguments(parser)\n _add_list_format(parser)\n\n def run(self, args: NominatimArgs) -> int:\n formatter = napi.load_format_dispatcher('v1', args.project_dir)\n\n if args.list_formats:\n return _list_formats(formatter, napi.ReverseResults)\n\n if args.format in ('debug', 'raw'):\n loglib.set_log_output('text')\n elif not formatter.supports_format(napi.ReverseResults, args.format):\n raise UsageError(f\"Unsupported format '{args.format}'. \"\n 'Use --list-formats to see supported formats.')\n\n if args.ids is None:\n raise UsageError(\"'id' parameter required.\")\n\n places = [napi.OsmID(o[0], int(o[1:])) for o in args.ids]\n\n try:\n with napi.NominatimAPI(args.project_dir) as api:\n results = api.lookup(places,\n address_details=True, # needed for display name\n geometry_output=_get_geometry_output(args),\n geometry_simplification=args.polygon_threshold or 0.0)\n except napi.UsageError as ex:\n raise UsageError(ex) from ex\n\n locales = _get_locales(args, api.config)\n locales.localize_results(results)\n\n if args.format == 'debug':\n print(loglib.get_and_disable())\n return 0\n\n _print_output(formatter, results, args.format,\n {'extratags': args.extratags,\n 'namedetails': args.namedetails,\n 'entrances': args.entrances,\n 'addressdetails': args.addressdetails})\n return 0\n\n\nclass APIDetails:\n \"\"\"\\\n Execute API details query.\n\n This command works exactly the same as if calling the /details endpoint on\n the web API. See the online documentation for more details on the\n various parameters:\n https://nominatim.org/release-docs/latest/api/Details/\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Query arguments')\n group.add_argument('--node', '-n', type=int,\n help=\"Look up the OSM node with the given ID.\")\n group.add_argument('--way', '-w', type=int,\n help=\"Look up the OSM way with the given ID.\")\n group.add_argument('--relation', '-r', type=int,\n help=\"Look up the OSM relation with the given ID.\")\n group.add_argument('--place_id', '-p', type=int,\n help='Database internal identifier of the OSM object to look up')\n group.add_argument('--class', dest='object_class',\n help=(\"Class type to disambiguated multiple entries \"\n \"of the same object.\"))\n\n group = parser.add_argument_group('Output arguments')\n group.add_argument('--format', type=str, default='json',\n help='Format of result (use --list-formats to see supported formats)')\n group.add_argument('--addressdetails', action='store_true',\n help='Include a breakdown of the address into elements')\n group.add_argument('--keywords', action='store_true',\n help='Include a list of name keywords and address keywords')\n group.add_argument('--linkedplaces', action='store_true',\n help='Include a details of places that are linked with this one')\n group.add_argument('--entrances', action='store_true',\n help='Include a list of tagged entrance nodes')\n group.add_argument('--hierarchy', action='store_true',\n help='Include details of places lower in the address hierarchy')\n group.add_argument('--group_hierarchy', action='store_true',\n help='Group the places by type')\n group.add_argument('--polygon_geojson', action='store_true',\n help='Include geometry of result')\n group.add_argument('--lang', '--accept-language', metavar='LANGS',\n help='Preferred language order for presenting search results')\n _add_list_format(parser)\n\n def run(self, args: NominatimArgs) -> int:\n formatter = napi.load_format_dispatcher('v1', args.project_dir)\n\n if args.list_formats:\n return _list_formats(formatter, napi.DetailedResult)\n\n if args.format in ('debug', 'raw'):\n loglib.set_log_output('text')\n elif not formatter.supports_format(napi.DetailedResult, args.format):\n raise UsageError(f\"Unsupported format '{args.format}'. \"\n 'Use --list-formats to see supported formats.')\n\n place: napi.PlaceRef\n if args.node:\n place = napi.OsmID('N', args.node, args.object_class)\n elif args.way:\n place = napi.OsmID('W', args.way, args.object_class)\n elif args.relation:\n place = napi.OsmID('R', args.relation, args.object_class)\n elif args.place_id is not None:\n place = napi.PlaceID(args.place_id)\n else:\n raise UsageError('One of the arguments --node/-n --way/-w '\n '--relation/-r --place_id/-p is required/')\n\n try:\n with napi.NominatimAPI(args.project_dir) as api:\n result = api.details(place,\n address_details=args.addressdetails,\n entrances=args.entrances,\n linked_places=args.linkedplaces,\n parented_places=args.hierarchy,\n keywords=args.keywords,\n geometry_output=(napi.GeometryFormat.GEOJSON\n if args.polygon_geojson\n else napi.GeometryFormat.NONE))\n except napi.UsageError as ex:\n raise UsageError(ex) from ex\n\n if result is not None:\n locales = _get_locales(args, api.config)\n locales.localize_results([result])\n\n if args.format == 'debug':\n print(loglib.get_and_disable())\n return 0\n\n if result:\n _print_output(formatter, result, args.format or 'json',\n {'group_hierarchy': args.group_hierarchy})\n return 0\n\n LOG.error(\"Object not found in database.\")\n return 42\n\n\nclass APIStatus:\n \"\"\"\n Execute API status query.\n\n This command works exactly the same as if calling the /status endpoint on\n the web API. See the online documentation for more details on the\n various parameters:\n https://nominatim.org/release-docs/latest/api/Status/\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('API parameters')\n group.add_argument('--format', type=str, default='text',\n help='Format of result (use --list-formats to see supported formats)')\n _add_list_format(parser)\n\n def run(self, args: NominatimArgs) -> int:\n formatter = napi.load_format_dispatcher('v1', args.project_dir)\n\n if args.list_formats:\n return _list_formats(formatter, napi.StatusResult)\n\n if args.format in ('debug', 'raw'):\n loglib.set_log_output('text')\n elif not formatter.supports_format(napi.StatusResult, args.format):\n raise UsageError(f\"Unsupported format '{args.format}'. \"\n 'Use --list-formats to see supported formats.')\n\n try:\n with napi.NominatimAPI(args.project_dir) as api:\n status = api.status()\n except napi.UsageError as ex:\n raise UsageError(ex) from ex\n\n if args.format == 'debug':\n print(loglib.get_and_disable())\n return 0\n\n _print_output(formatter, status, args.format, {})\n\n return 0\n"
},
{
"path": "src/nominatim_db/clicmd/args.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nProvides custom functions over command-line arguments.\n\"\"\"\nfrom typing import Optional, List, Dict, Any, Sequence, Tuple\nimport argparse\nimport logging\nfrom pathlib import Path\n\nfrom ..errors import UsageError\nfrom ..config import Configuration\nfrom ..typing import Protocol\n\n\nLOG = logging.getLogger()\n\n\nclass Subcommand(Protocol):\n \"\"\"\n Interface to be implemented by classes implementing a CLI subcommand.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n \"\"\"\n Fill the given parser for the subcommand with the appropriate\n parameters.\n \"\"\"\n\n def run(self, args: 'NominatimArgs') -> int:\n \"\"\"\n Run the subcommand with the given parsed arguments.\n \"\"\"\n\n\nclass NominatimArgs:\n \"\"\" Customized namespace class for the nominatim command line tool\n to receive the command-line arguments.\n \"\"\"\n # Basic environment set by root program.\n config: Configuration\n project_dir: Path\n\n # Global switches\n version: bool\n subcommand: Optional[str]\n command: Subcommand\n\n # Shared parameters\n osm2pgsql_cache: Optional[int]\n socket_timeout: int\n\n # Arguments added to all subcommands.\n verbose: int\n threads: Optional[int]\n\n # Arguments to 'add-data'\n file: Optional[str]\n diff: Optional[str]\n node: Optional[int]\n way: Optional[int]\n relation: Optional[int]\n tiger_data: Optional[str]\n use_main_api: bool\n\n # Arguments to 'admin'\n warm: bool\n check_database: bool\n migrate: bool\n collect_os_info: bool\n clean_deleted: str\n analyse_indexing: bool\n target: Optional[str]\n osm_id: Optional[str]\n place_id: Optional[int]\n\n # Arguments to 'import'\n osm_file: List[str]\n continue_at: Optional[str]\n reverse_only: bool\n no_partitions: bool\n no_updates: bool\n offline: bool\n ignore_errors: bool\n index_noanalyse: bool\n prepare_database: bool\n\n # Arguments to 'index'\n boundaries_only: bool\n no_boundaries: bool\n minrank: int\n maxrank: int\n\n # Arguments to 'export'\n output_type: str\n output_format: str\n output_all_postcodes: bool\n language: Optional[str]\n restrict_to_country: Optional[str]\n\n # Arguments to 'convert'\n output: Path\n\n # Arguments to 'refresh'\n postcodes: bool\n word_tokens: bool\n word_counts: bool\n address_levels: bool\n functions: bool\n wiki_data: bool\n secondary_importance: bool\n importance: bool\n website: bool\n diffs: bool\n enable_debug_statements: bool\n data_object: Sequence[Tuple[str, int]]\n data_area: Sequence[Tuple[str, int]]\n ro_access: bool\n postcode_force_reimport: bool\n\n # Arguments to 'replication'\n init: bool\n update_functions: bool\n check_for_updates: bool\n once: bool\n catch_up: bool\n do_index: bool\n\n # Arguments to 'serve'\n server: str\n engine: str\n\n # Arguments to 'special-phrases\n import_from_wiki: bool\n import_from_csv: Optional[str]\n no_replace: bool\n min: int\n\n # Arguments to all query functions\n format: str\n list_formats: bool\n addressdetails: bool\n entrances: bool\n extratags: bool\n namedetails: bool\n lang: Optional[str]\n polygon_output: Optional[str]\n polygon_threshold: Optional[float]\n\n # Arguments to 'search'\n query: Optional[str]\n amenity: Optional[str]\n street: Optional[str]\n city: Optional[str]\n county: Optional[str]\n state: Optional[str]\n country: Optional[str]\n postalcode: Optional[str]\n countrycodes: Optional[str]\n exclude_place_ids: Optional[str]\n limit: int\n viewbox: Optional[str]\n bounded: bool\n dedupe: bool\n\n # Arguments to 'reverse'\n lat: float\n lon: float\n zoom: Optional[int]\n layers: Optional[Sequence[str]]\n\n # Arguments to 'lookup'\n ids: Sequence[str]\n\n # Arguments to 'details'\n object_class: Optional[str]\n linkedplaces: bool\n hierarchy: bool\n keywords: bool\n polygon_geojson: bool\n group_hierarchy: bool\n\n def osm2pgsql_options(self, default_cache: int,\n default_threads: int) -> Dict[str, Any]:\n \"\"\" Return the standard osm2pgsql options that can be derived\n from the command line arguments. The resulting dict can be\n further customized and then used in `run_osm2pgsql()`.\n \"\"\"\n return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY,\n osm2pgsql_cache=self.osm2pgsql_cache or default_cache,\n osm2pgsql_style=self.config.get_import_style_file(),\n osm2pgsql_style_path=self.config.lib_dir.lua,\n threads=self.threads or default_threads,\n dsn=self.config.get_libpq_dsn(),\n flatnode_file=str(self.config.get_path('FLATNODE_FILE') or ''),\n tablespaces=dict(slim_data=self.config.TABLESPACE_OSM_DATA,\n slim_index=self.config.TABLESPACE_OSM_INDEX,\n main_data=self.config.TABLESPACE_PLACE_DATA,\n main_index=self.config.TABLESPACE_PLACE_INDEX\n )\n )\n\n def get_osm_file_list(self) -> Optional[List[Path]]:\n \"\"\" Return the --osm-file argument as a list of Paths or None\n if no argument was given. The function also checks if the files\n exist and raises a UsageError if one cannot be found.\n \"\"\"\n if not self.osm_file:\n return None\n\n files = [Path(f) for f in self.osm_file]\n for fname in files:\n if not fname.is_file():\n LOG.fatal(\"OSM file '%s' does not exist.\", fname)\n raise UsageError('Cannot access file.')\n\n return files\n"
},
{
"path": "src/nominatim_db/clicmd/convert.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'convert' subcommand.\n\"\"\"\nfrom typing import Set, Any, Union, Optional, Sequence\nimport argparse\nimport asyncio\nfrom pathlib import Path\n\nfrom ..errors import UsageError\nfrom .args import NominatimArgs\n\n\nclass WithAction(argparse.Action):\n \"\"\" Special action that saves a list of flags, given on the command-line\n as `--with-foo` or `--without-foo`.\n \"\"\"\n def __init__(self, option_strings: Sequence[str], dest: Any,\n default: bool = True, **kwargs: Any) -> None:\n if 'nargs' in kwargs:\n raise ValueError(\"nargs not allowed.\")\n if option_strings is None:\n raise ValueError(\"Positional parameter not allowed.\")\n\n self.dest_set = kwargs.pop('dest_set')\n full_option_strings = []\n for opt in option_strings:\n if not opt.startswith('--'):\n raise ValueError(\"short-form options not allowed\")\n if default:\n self.dest_set.add(opt[2:])\n full_option_strings.append(f\"--with-{opt[2:]}\")\n full_option_strings.append(f\"--without-{opt[2:]}\")\n\n super().__init__(full_option_strings, argparse.SUPPRESS, nargs=0, **kwargs)\n\n def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace,\n values: Union[str, Sequence[Any], None],\n option_string: Optional[str] = None) -> None:\n assert option_string\n if option_string.startswith('--with-'):\n self.dest_set.add(option_string[7:])\n if option_string.startswith('--without-'):\n self.dest_set.discard(option_string[10:])\n\n\nclass ConvertDB:\n \"\"\" Convert an existing database into a different format. (EXPERIMENTAL)\n\n Dump a read-only version of the database in a different format.\n At the moment only a SQLite database suitable for reverse lookup\n can be created.\n \"\"\"\n\n def __init__(self) -> None:\n self.options: Set[str] = set()\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n parser.add_argument('--format', default='sqlite',\n choices=('sqlite', ),\n help='Format of the output database (must be sqlite currently)')\n parser.add_argument('--output', '-o', required=True, type=Path,\n help='File to write the database to.')\n group = parser.add_argument_group('Switches to define database layout'\n '(currently no effect)')\n group.add_argument('--reverse', action=WithAction, dest_set=self.options, default=True,\n help='Enable/disable support for reverse and lookup API'\n ' (default: enabled)')\n group.add_argument('--search', action=WithAction, dest_set=self.options, default=True,\n help='Enable/disable support for search API (default: disabled)')\n group.add_argument('--details', action=WithAction, dest_set=self.options, default=True,\n help='Enable/disable support for details API (default: enabled)')\n\n def run(self, args: NominatimArgs) -> int:\n if args.output.exists():\n raise UsageError(f\"File '{args.output}' already exists. Refusing to overwrite.\")\n\n if args.format == 'sqlite':\n from ..tools import convert_sqlite\n\n asyncio.run(convert_sqlite.convert(args.project_dir, args.output, self.options))\n return 0\n\n return 1\n"
},
{
"path": "src/nominatim_db/clicmd/export.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'export' subcommand.\n\"\"\"\nfrom typing import Optional, List, cast\nimport logging\nimport argparse\nimport asyncio\nimport csv\nimport sys\n\nimport nominatim_api as napi\nfrom nominatim_api.results import create_from_placex_row, ReverseResult, add_result_details\nfrom nominatim_api.types import LookupDetails\n\nimport sqlalchemy as sa\n\nfrom ..errors import UsageError\nfrom .args import NominatimArgs\n\n\nLOG = logging.getLogger()\n\n\nRANK_RANGE_MAP = {\n 'country': (4, 4),\n 'state': (5, 9),\n 'county': (10, 12),\n 'city': (13, 16),\n 'suburb': (17, 21),\n 'street': (26, 26),\n 'path': (27, 27)\n}\n\n\nRANK_TO_OUTPUT_MAP = {\n 4: 'country',\n 5: 'state', 6: 'state', 7: 'state', 8: 'state', 9: 'state',\n 10: 'county', 11: 'county', 12: 'county',\n 13: 'city', 14: 'city', 15: 'city', 16: 'city',\n 17: 'suburb', 18: 'suburb', 19: 'suburb', 20: 'suburb', 21: 'suburb',\n 26: 'street', 27: 'path'}\n\n\nclass QueryExport:\n \"\"\"\\\n Export places as CSV file from the database.\n\n\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Output arguments')\n group.add_argument('--output-type', default='street',\n choices=('country', 'state', 'county',\n 'city', 'suburb', 'street', 'path'),\n help='Type of places to output (default: street)')\n group.add_argument('--output-format',\n default='street;suburb;city;county;state;country',\n help=(\"Semicolon-separated list of address types \"\n \"(see --output-type). Additionally accepts:\"\n \"placeid,postcode\"))\n group.add_argument('--language',\n help=(\"Preferred language for output \"\n \"(use local name, if omitted)\"))\n group = parser.add_argument_group('Filter arguments')\n group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',\n help='Export only objects within country')\n group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,\n dest='node',\n help='Export only children of this OSM node')\n group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,\n dest='way',\n help='Export only children of this OSM way')\n group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,\n dest='relation',\n help='Export only children of this OSM relation')\n\n def run(self, args: NominatimArgs) -> int:\n return asyncio.run(export(args))\n\n\nasync def export(args: NominatimArgs) -> int:\n \"\"\" The actual export as a asynchronous function.\n \"\"\"\n\n api = napi.NominatimAPIAsync(args.project_dir)\n\n try:\n output_range = RANK_RANGE_MAP[args.output_type]\n\n writer = init_csv_writer(args.output_format)\n\n async with api.begin() as conn, api.begin() as detail_conn:\n t = conn.t.placex\n\n sql = sa.select(t.c.place_id, t.c.parent_place_id,\n t.c.osm_type, t.c.osm_id, t.c.name,\n t.c.class_, t.c.type, t.c.admin_level,\n t.c.address, t.c.extratags,\n t.c.housenumber, t.c.postcode, t.c.country_code,\n t.c.importance, t.c.wikipedia, t.c.indexed_date,\n t.c.rank_address, t.c.rank_search,\n t.c.centroid)\\\n .where(t.c.linked_place_id == None)\\\n .where(t.c.rank_address.between(*output_range))\n\n parent_place_id = await get_parent_id(conn, args.node, args.way, args.relation)\n if parent_place_id:\n taddr = conn.t.addressline\n\n sql = sql.join(taddr, taddr.c.place_id == t.c.place_id)\\\n .where(taddr.c.address_place_id == parent_place_id)\\\n .where(taddr.c.isaddress)\n\n if args.restrict_to_country:\n sql = sql.where(t.c.country_code == args.restrict_to_country.lower())\n\n results = []\n for row in await conn.execute(sql):\n result = create_from_placex_row(row, ReverseResult)\n if result is not None:\n results.append(result)\n\n if len(results) == 1000:\n await dump_results(detail_conn, results, writer, args.language)\n results = []\n\n if results:\n await dump_results(detail_conn, results, writer, args.language)\n finally:\n await api.close()\n\n return 0\n\n\ndef init_csv_writer(output_format: str) -> 'csv.DictWriter[str]':\n fields = output_format.split(';')\n writer = csv.DictWriter(sys.stdout, fieldnames=fields, extrasaction='ignore')\n writer.writeheader()\n\n return writer\n\n\nasync def dump_results(conn: napi.SearchConnection,\n results: List[ReverseResult],\n writer: 'csv.DictWriter[str]',\n lang: Optional[str]) -> None:\n await add_result_details(conn, results,\n LookupDetails(address_details=True))\n\n locale = napi.Locales([lang] if lang else None, conn.config.OUTPUT_NAMES)\n locale.localize_results(results)\n\n for result in results:\n data = {'placeid': result.place_id,\n 'postcode': result.postcode}\n\n for line in (result.address_rows or []):\n if line.isaddress and line.local_name:\n if line.category[1] == 'postcode':\n data['postcode'] = line.local_name\n elif line.rank_address in RANK_TO_OUTPUT_MAP:\n data[RANK_TO_OUTPUT_MAP[line.rank_address]] = line.local_name\n\n writer.writerow(data)\n\n\nasync def get_parent_id(conn: napi.SearchConnection, node_id: Optional[int],\n way_id: Optional[int],\n relation_id: Optional[int]) -> Optional[int]:\n \"\"\" Get the place ID for the given OSM object.\n \"\"\"\n if node_id is not None:\n osm_type, osm_id = 'N', node_id\n elif way_id is not None:\n osm_type, osm_id = 'W', way_id\n elif relation_id is not None:\n osm_type, osm_id = 'R', relation_id\n else:\n return None\n\n t = conn.t.placex\n sql = sa.select(t.c.place_id).limit(1)\\\n .where(t.c.osm_type == osm_type)\\\n .where(t.c.osm_id == osm_id)\\\n .where(t.c.rank_address > 0)\\\n .order_by(t.c.rank_address)\n\n for result in await conn.execute(sql):\n return cast(int, result[0])\n\n raise UsageError(f'Cannot find a place {osm_type}{osm_id}.')\n"
},
{
"path": "src/nominatim_db/clicmd/freeze.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'freeze' subcommand.\n\"\"\"\nimport argparse\n\nfrom ..db.connection import connect\nfrom .args import NominatimArgs\n\n\nclass SetupFreeze:\n \"\"\"\\\n Make database read-only.\n\n About half of data in the Nominatim database is kept only to be able to\n keep the data up-to-date with new changes made in OpenStreetMap. This\n command drops all this data and only keeps the part needed for geocoding\n itself.\n\n This command has the same effect as the `--no-updates` option for imports.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n pass # No options\n\n def run(self, args: NominatimArgs) -> int:\n from ..tools import freeze\n\n with connect(args.config.get_libpq_dsn()) as conn:\n freeze.drop_update_tables(conn)\n freeze.drop_flatnode_file(args.config.get_path('FLATNODE_FILE'))\n\n return 0\n"
},
{
"path": "src/nominatim_db/clicmd/index.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'index' subcommand.\n\"\"\"\nimport argparse\nimport asyncio\n\nimport psutil\n\nfrom ..db import status\nfrom ..db.connection import connect\nfrom .args import NominatimArgs\n\n\nclass UpdateIndex:\n \"\"\"\\\n Reindex all new and modified data.\n\n Indexing is the process of computing the address and search terms for\n the places in the database. Every time data is added or changed, indexing\n needs to be run. Imports and replication updates automatically take care\n of indexing. For other cases, this function allows to run indexing manually.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Filter arguments')\n group.add_argument('--boundaries-only', action='store_true',\n help=\"\"\"Index only administrative boundaries.\"\"\")\n group.add_argument('--no-boundaries', action='store_true',\n help=\"\"\"Index everything except administrative boundaries.\"\"\")\n group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,\n help='Minimum/starting rank')\n group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,\n help='Maximum/finishing rank')\n\n def run(self, args: NominatimArgs) -> int:\n asyncio.run(self._do_index(args))\n\n if not args.no_boundaries and not args.boundaries_only \\\n and args.minrank == 0 and args.maxrank == 30:\n with connect(args.config.get_libpq_dsn()) as conn:\n status.set_indexed(conn, True)\n\n return 0\n\n async def _do_index(self, args: NominatimArgs) -> None:\n from ..tokenizer import factory as tokenizer_factory\n\n tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)\n from ..indexer.indexer import Indexer\n\n indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,\n args.threads or psutil.cpu_count() or 1)\n\n has_pending = True # run at least once\n while has_pending:\n if not args.no_boundaries:\n await indexer.index_boundaries(args.minrank, args.maxrank)\n if not args.boundaries_only:\n await indexer.index_by_rank(args.minrank, args.maxrank)\n await indexer.index_postcodes()\n has_pending = indexer.has_pending(args.minrank, args.maxrank)\n"
},
{
"path": "src/nominatim_db/clicmd/refresh.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of 'refresh' subcommand.\n\"\"\"\nfrom typing import Tuple, Optional\nimport argparse\nimport logging\nfrom pathlib import Path\nimport asyncio\n\nfrom ..config import Configuration\nfrom ..db.connection import connect, table_exists\nfrom ..tokenizer.base import AbstractTokenizer\nfrom .args import NominatimArgs\n\n\nLOG = logging.getLogger()\n\n\ndef _parse_osm_object(obj: str) -> Tuple[str, int]:\n \"\"\" Parse the given argument into a tuple of OSM type and ID.\n Raises an ArgumentError if the format is not recognized.\n \"\"\"\n if len(obj) < 2 or obj[0].lower() not in 'nrw' or not obj[1:].isdigit():\n raise argparse.ArgumentTypeError(\"Cannot parse OSM ID. Expect format: [N|W|R].\")\n\n return (obj[0].upper(), int(obj[1:]))\n\n\nclass UpdateRefresh:\n \"\"\"\\\n Recompute auxiliary data used by the indexing process.\n\n This sub-commands updates various static data and functions in the database.\n It usually needs to be run after changing various aspects of the\n configuration. The configuration documentation will mention the exact\n command to use in such case.\n\n Warning: the 'update' command must not be run in parallel with other update\n commands like 'replication' or 'add-data'.\n \"\"\"\n def __init__(self) -> None:\n self.tokenizer: Optional[AbstractTokenizer] = None\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Data arguments')\n group.add_argument('--postcodes', action='store_true',\n help='Update postcode centroid table')\n group.add_argument('--word-tokens', action='store_true',\n help='Clean up search terms')\n group.add_argument('--word-counts', action='store_true',\n help='Compute frequency of full-word search terms')\n group.add_argument('--address-levels', action='store_true',\n help='Reimport address level configuration')\n group.add_argument('--functions', action='store_true',\n help='Update the PL/pgSQL functions in the database')\n group.add_argument('--wiki-data', action='store_true',\n help='Update Wikipedia/data importance numbers')\n group.add_argument('--secondary-importance', action='store_true',\n help='Update secondary importance raster data')\n group.add_argument('--importance', action='store_true',\n help='Recompute place importances (expensive!)')\n group.add_argument('--ro-access', action='store_true',\n help='Grant read-only access to web user for all tables')\n group.add_argument('--website', action='store_true',\n help='DEPRECATED. This function has no function anymore'\n ' and will be removed in a future version.')\n group.add_argument('--data-object', action='append',\n type=_parse_osm_object, metavar='OBJECT',\n help='Mark the given OSM object as requiring an update'\n ' (format: [NWR])')\n group.add_argument('--data-area', action='append',\n type=_parse_osm_object, metavar='OBJECT',\n help='Mark the area around the given OSM object as requiring an update'\n ' (format: [NWR])')\n\n group = parser.add_argument_group('Arguments for function refresh')\n group.add_argument('--no-diff-updates', action='store_false', dest='diffs',\n help='Do not enable code for propagating updates')\n group.add_argument('--enable-debug-statements', action='store_true',\n help='Enable debug warning statements in functions')\n group = parser.add_argument_group('Arguments for postcode refresh')\n group.add_argument('--force-reimport', action='store_true',\n dest='postcode_force_reimport',\n help='Recompute the postcodes from scratch instead of updating')\n\n def run(self, args: NominatimArgs) -> int:\n from ..tools import refresh, postcodes\n from ..indexer.indexer import Indexer\n\n need_function_refresh = args.functions\n\n if args.postcodes:\n if postcodes.can_compute(args.config.get_libpq_dsn()):\n LOG.warning(\"Update postcodes centroid\")\n tokenizer = self._get_tokenizer(args.config)\n postcodes.update_postcodes(args.config.get_libpq_dsn(),\n args.project_dir, tokenizer,\n force_reimport=args.postcode_force_reimport)\n indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,\n args.threads or 1)\n asyncio.run(indexer.index_postcodes())\n else:\n LOG.error(\"The place table doesn't exist. \"\n \"Postcode updates on a frozen database is not possible.\")\n\n if args.word_tokens:\n LOG.warning('Updating word tokens')\n tokenizer = self._get_tokenizer(args.config)\n tokenizer.update_word_tokens()\n\n if args.word_counts:\n LOG.warning('Recompute word statistics')\n self._get_tokenizer(args.config).update_statistics(args.config,\n threads=args.threads or 1)\n\n if args.address_levels:\n LOG.warning('Updating address levels')\n with connect(args.config.get_libpq_dsn()) as conn:\n refresh.load_address_levels_from_config(conn, args.config)\n\n # Attention: must come BEFORE functions\n if args.secondary_importance:\n with connect(args.config.get_libpq_dsn()) as conn:\n # If the table did not exist before, then the importance code\n # needs to be enabled.\n if not table_exists(conn, 'secondary_importance'):\n args.functions = True\n\n LOG.warning('Import secondary importance raster data from %s', args.project_dir)\n if refresh.import_secondary_importance(args.config.get_libpq_dsn(),\n args.project_dir) > 0:\n LOG.fatal('FATAL: Cannot update secondary importance raster data')\n return 1\n need_function_refresh = True\n\n if args.wiki_data:\n data_path = Path(args.config.WIKIPEDIA_DATA_PATH\n or args.project_dir)\n LOG.warning('Import wikipedia article importance from %s', data_path)\n if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),\n data_path) > 0:\n LOG.fatal('FATAL: Wikipedia importance file not found in %s', data_path)\n return 1\n need_function_refresh = True\n\n if need_function_refresh:\n LOG.warning('Create functions')\n with connect(args.config.get_libpq_dsn()) as conn:\n refresh.create_functions(conn, args.config,\n args.diffs, args.enable_debug_statements)\n self._get_tokenizer(args.config).update_sql_functions(args.config)\n\n # Attention: importance MUST come after wiki data import and after functions.\n if args.importance:\n LOG.warning('Update importance values for database')\n with connect(args.config.get_libpq_dsn()) as conn:\n refresh.recompute_importance(conn)\n\n if args.website:\n LOG.error('WARNING: Website setup is no longer required. '\n 'This function will be removed in future version of Nominatim.')\n\n if args.ro_access:\n from ..tools import admin\n LOG.warning('Grant read-only access to web user')\n admin.grant_ro_access(args.config.get_libpq_dsn(), args.config)\n\n if args.data_object or args.data_area:\n with connect(args.config.get_libpq_dsn()) as conn:\n for obj in args.data_object or []:\n refresh.invalidate_osm_object(*obj, conn, recursive=False)\n for obj in args.data_area or []:\n refresh.invalidate_osm_object(*obj, conn, recursive=True)\n conn.commit()\n\n return 0\n\n def _get_tokenizer(self, config: Configuration) -> AbstractTokenizer:\n if self.tokenizer is None:\n from ..tokenizer import factory as tokenizer_factory\n\n self.tokenizer = tokenizer_factory.get_tokenizer_for_db(config)\n\n return self.tokenizer\n"
},
{
"path": "src/nominatim_db/clicmd/replication.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'replication' sub-command.\n\"\"\"\nfrom typing import Optional\nimport argparse\nimport datetime as dt\nimport logging\nimport socket\nimport time\nimport asyncio\n\nfrom ..db import status\nfrom ..db.connection import connect\nfrom ..errors import UsageError\nfrom .args import NominatimArgs\n\nLOG = logging.getLogger()\n\n\nclass UpdateReplication:\n \"\"\"\\\n Update the database using an online replication service.\n\n An OSM replication service is an online service that provides regular\n updates (OSM diff files) for the planet or update they provide. The OSMF\n provides the primary replication service for the full planet at\n https://planet.osm.org/replication/ but there are other providers of\n extracts of OSM data who provide such a service as well.\n\n This sub-command allows to set up such a replication service and download\n and import updates at regular intervals. You need to call '--init' once to\n set up the process or whenever you change the replication configuration\n parameters. Without any arguments, the sub-command will go into a loop and\n continuously apply updates as they become available. Giving `--once` just\n downloads and imports the next batch of updates.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Arguments for initialisation')\n group.add_argument('--init', action='store_true',\n help='Initialise the update process')\n group.add_argument('--no-update-functions', dest='update_functions',\n action='store_false',\n help=\"Do not update the trigger function to \"\n \"support differential updates (EXPERT)\")\n group = parser.add_argument_group('Arguments for updates')\n group.add_argument('--check-for-updates', action='store_true',\n help='Check if new updates are available and exit')\n group.add_argument('--once', action='store_true',\n help=\"Download and apply updates only once. When \"\n \"not set, updates are continuously applied\")\n group.add_argument('--catch-up', action='store_true',\n help=\"Download and apply updates until no new \"\n \"data is available on the server\")\n group.add_argument('--no-index', action='store_false', dest='do_index',\n help=(\"Do not index the new data. Only usable \"\n \"together with --once\"))\n group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,\n help='Size of cache to be used by osm2pgsql (in MB)')\n group = parser.add_argument_group('Download parameters')\n group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,\n help='Set timeout for file downloads')\n\n def _init_replication(self, args: NominatimArgs) -> int:\n from ..tools import replication, refresh\n\n LOG.warning(\"Initialising replication updates\")\n with connect(args.config.get_libpq_dsn()) as conn:\n replication.init_replication(conn, base_url=args.config.REPLICATION_URL,\n socket_timeout=args.socket_timeout)\n if args.update_functions:\n LOG.warning(\"Create functions\")\n refresh.create_functions(conn, args.config, True, False)\n return 0\n\n def _check_for_updates(self, args: NominatimArgs) -> int:\n from ..tools import replication\n\n with connect(args.config.get_libpq_dsn()) as conn:\n return replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL,\n socket_timeout=args.socket_timeout)\n\n def _report_update(self, batchdate: dt.datetime,\n start_import: dt.datetime,\n start_index: Optional[dt.datetime]) -> None:\n def round_time(delta: dt.timedelta) -> dt.timedelta:\n return dt.timedelta(seconds=int(delta.total_seconds()))\n\n end = dt.datetime.now(dt.timezone.utc)\n LOG.warning(\"Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.\",\n round_time((start_index or end) - start_import),\n f\"Indexing: {round_time(end - start_index)} \" if start_index else '',\n round_time(end - start_import),\n round_time(end - batchdate))\n\n def _compute_update_interval(self, args: NominatimArgs) -> int:\n if args.catch_up:\n return 0\n\n update_interval = args.config.get_int('REPLICATION_UPDATE_INTERVAL')\n # Sanity check to not overwhelm the Geofabrik servers.\n if 'download.geofabrik.de' in args.config.REPLICATION_URL\\\n and update_interval < 86400:\n LOG.fatal(\"Update interval too low for download.geofabrik.de.\\n\"\n \"Please check install documentation \"\n \"(https://nominatim.org/release-docs/latest/admin/Update/#\"\n \"setting-up-the-update-process).\")\n raise UsageError(\"Invalid replication update interval setting.\")\n\n return update_interval\n\n async def _update(self, args: NominatimArgs) -> None:\n from ..tools import replication\n from ..indexer.indexer import Indexer\n from ..tokenizer import factory as tokenizer_factory\n\n update_interval = self._compute_update_interval(args)\n\n params = args.osm2pgsql_options(default_cache=2000, default_threads=1)\n params.update(base_url=args.config.REPLICATION_URL,\n update_interval=update_interval,\n import_file=args.project_dir / 'osmosischange.osc',\n max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),\n indexed_only=not args.once)\n\n if not args.once:\n if not args.do_index:\n LOG.fatal(\"Indexing cannot be disabled when running updates continuously.\")\n raise UsageError(\"Bad argument '--no-index'.\")\n recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')\n\n tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)\n indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1)\n\n dsn = args.config.get_libpq_dsn()\n\n while True:\n start = dt.datetime.now(dt.timezone.utc)\n state = replication.update(dsn, params, socket_timeout=args.socket_timeout)\n\n with connect(dsn) as conn:\n if state is not replication.UpdateState.NO_CHANGES:\n status.log_status(conn, start, 'import')\n batchdate, _, _ = status.get_status(conn)\n conn.commit()\n\n if state is not replication.UpdateState.NO_CHANGES and args.do_index:\n index_start = dt.datetime.now(dt.timezone.utc)\n await indexer.index_full(analyse=False)\n\n with connect(dsn) as conn:\n status.set_indexed(conn, True)\n status.log_status(conn, index_start, 'index')\n conn.commit()\n else:\n index_start = None\n\n if state is replication.UpdateState.NO_CHANGES and \\\n args.catch_up or update_interval > 40*60:\n await indexer.index_full(analyse=False)\n\n if LOG.isEnabledFor(logging.WARNING):\n assert batchdate is not None\n self._report_update(batchdate, start, index_start)\n\n if args.once or (args.catch_up and state is replication.UpdateState.NO_CHANGES):\n break\n\n if state is replication.UpdateState.NO_CHANGES:\n LOG.warning(\"No new changes. Sleeping for %d sec.\", recheck_interval)\n time.sleep(recheck_interval)\n\n def run(self, args: NominatimArgs) -> int:\n socket.setdefaulttimeout(args.socket_timeout)\n\n if args.init:\n return self._init_replication(args)\n\n if args.check_for_updates:\n return self._check_for_updates(args)\n\n asyncio.run(self._update(args))\n return 0\n"
},
{
"path": "src/nominatim_db/clicmd/setup.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nImplementation of the 'import' subcommand.\n\"\"\"\nfrom typing import Optional\nimport argparse\nimport logging\nfrom pathlib import Path\nimport asyncio\n\nimport psutil\n\nfrom ..errors import UsageError\nfrom ..config import Configuration\nfrom ..db.connection import connect\nfrom ..db import status, properties\nfrom ..tokenizer.base import AbstractTokenizer\nfrom ..version import NOMINATIM_VERSION\nfrom .args import NominatimArgs\n\nimport time\n\nLOG = logging.getLogger()\n\n\nclass SetupAll:\n \"\"\"\\\n Create a new Nominatim database from an OSM file.\n\n This sub-command sets up a new Nominatim database from scratch starting\n with creating a new database in Postgresql. The user running this command\n needs superuser rights on the database.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group1 = parser.add_argument_group('Required arguments')\n group1.add_argument('--osm-file', metavar='FILE', action='append',\n help='OSM file to be imported'\n ' (repeat for importing multiple files)',\n default=None)\n group1.add_argument('--continue', dest='continue_at',\n choices=['import-from-file', 'load-data', 'indexing', 'db-postprocess'],\n help='Continue an import that was interrupted',\n default=None)\n group2 = parser.add_argument_group('Optional arguments')\n group2.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,\n help='Size of cache to be used by osm2pgsql (in MB)')\n group2.add_argument('--reverse-only', action='store_true',\n help='Do not create tables and indexes for searching')\n group2.add_argument('--no-partitions', action='store_true',\n help=\"Do not partition search indices \"\n \"(speeds up import of single country extracts)\")\n group2.add_argument('--no-updates', action='store_true',\n help=\"Do not keep tables that are only needed for \"\n \"updating the database later\")\n group2.add_argument('--offline', action='store_true',\n help=\"Do not attempt to load any additional data from the internet\")\n group3 = parser.add_argument_group('Expert options')\n group3.add_argument('--ignore-errors', action='store_true',\n help='Continue import even when errors in SQL are present')\n group3.add_argument('--index-noanalyse', action='store_true',\n help='Do not perform analyse operations during index (expert only)')\n group3.add_argument('--prepare-database', action='store_true',\n help='Create the database but do not import any data')\n\n def run(self, args: NominatimArgs) -> int:\n if args.osm_file is None and args.continue_at is None and not args.prepare_database:\n raise UsageError(\"No input files (use --osm-file).\")\n\n if args.osm_file is not None and args.continue_at not in ('import-from-file', None):\n raise UsageError(f\"Cannot use --continue {args.continue_at} and --osm-file together.\")\n\n if args.continue_at is not None and args.prepare_database:\n raise UsageError(\n \"Cannot use --continue and --prepare-database together.\"\n )\n\n return asyncio.run(self.async_run(args))\n\n async def async_run(self, args: NominatimArgs) -> int:\n from ..data import country_info\n from ..tools import database_import, postcodes, freeze\n from ..indexer.indexer import Indexer\n\n start_time = time.time()\n\n num_threads = args.threads or psutil.cpu_count() or 1\n country_info.setup_country_config(args.config)\n\n if args.prepare_database or args.continue_at is None:\n LOG.warning('Creating database')\n database_import.setup_database_skeleton(args.config.get_libpq_dsn(),\n rouser=args.config.DATABASE_WEBUSER)\n if args.prepare_database:\n return 0\n\n if args.continue_at in (None, 'import-from-file'):\n self._base_import(args)\n\n if args.continue_at in ('import-from-file', 'load-data', None):\n LOG.warning('Initialise tables')\n with connect(args.config.get_libpq_dsn()) as conn:\n database_import.truncate_data_tables(conn)\n\n LOG.warning('Load data into placex table')\n await database_import.load_data(args.config.get_libpq_dsn(), num_threads)\n\n LOG.warning(\"Setting up tokenizer\")\n tokenizer = self._get_tokenizer(args.continue_at, args.config)\n\n if args.continue_at in ('import-from-file', 'load-data', None):\n LOG.warning('Calculate postcodes')\n postcodes.update_postcodes(args.config.get_libpq_dsn(),\n args.project_dir, tokenizer)\n\n if args.continue_at in ('import-from-file', 'load-data', 'indexing', None):\n LOG.warning('Indexing places')\n indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, num_threads)\n await indexer.index_full(analyse=not args.index_noanalyse)\n\n LOG.warning('Post-process tables')\n with connect(args.config.get_libpq_dsn()) as conn:\n conn.autocommit = True\n await database_import.create_search_indices(conn, args.config,\n drop=args.no_updates,\n threads=num_threads)\n LOG.warning('Create search index for default country names.')\n conn.autocommit = False\n country_info.create_country_names(conn, tokenizer,\n args.config.get_str_list('LANGUAGES'))\n if args.no_updates:\n conn.autocommit = True\n freeze.drop_update_tables(conn)\n tokenizer.finalize_import(args.config)\n\n LOG.warning('Recompute word counts')\n tokenizer.update_statistics(args.config, threads=num_threads)\n\n end_time = time.time()\n elapsed = end_time - start_time\n LOG.warning(f'Import completed successfully in {elapsed:.2f} seconds.')\n\n self._finalize_database(args.config.get_libpq_dsn(), args.offline)\n\n return 0\n\n def _base_import(self, args: NominatimArgs) -> None:\n from ..tools import database_import, refresh\n from ..data import country_info\n\n files = args.get_osm_file_list()\n if not files:\n raise UsageError(\"No input files (use --osm-file).\")\n\n if args.continue_at in ('import-from-file', None):\n # Check if the correct plugins are installed\n database_import.check_existing_database_plugins(args.config.get_libpq_dsn())\n LOG.warning('Setting up country tables')\n country_info.setup_country_tables(args.config.get_libpq_dsn(),\n args.config.lib_dir.data,\n args.no_partitions)\n\n LOG.warning('Importing OSM data file')\n database_import.import_osm_data(files,\n args.osm2pgsql_options(0, 1),\n drop=args.no_updates,\n ignore_errors=args.ignore_errors)\n\n LOG.warning('Importing wikipedia importance data')\n data_path = Path(args.config.WIKIPEDIA_DATA_PATH or args.project_dir)\n if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),\n data_path) > 0:\n LOG.error('Wikipedia importance dump file not found. '\n 'Calculating importance values of locations will not '\n 'use Wikipedia importance data.')\n\n LOG.warning('Importing secondary importance raster data')\n if refresh.import_secondary_importance(args.config.get_libpq_dsn(),\n args.project_dir) != 0:\n LOG.error('Secondary importance file not imported. '\n 'Falling back to default ranking.')\n\n self._setup_tables(args.config, args.reverse_only)\n\n def _setup_tables(self, config: Configuration, reverse_only: bool) -> None:\n \"\"\" Set up the basic database layout: tables, indexes and functions.\n \"\"\"\n from ..tools import database_import, refresh\n\n with connect(config.get_libpq_dsn()) as conn:\n conn.autocommit = True\n LOG.warning('Create functions (1st pass)')\n refresh.create_functions(conn, config, False, False)\n LOG.warning('Create tables')\n database_import.create_tables(conn, config, reverse_only=reverse_only)\n refresh.load_address_levels_from_config(conn, config)\n LOG.warning('Create functions (2nd pass)')\n refresh.create_functions(conn, config, False, False)\n LOG.warning('Create table triggers')\n database_import.create_table_triggers(conn, config)\n LOG.warning('Create partition tables')\n database_import.create_partition_tables(conn, config)\n LOG.warning('Create functions (3rd pass)')\n refresh.create_functions(conn, config, False, False)\n\n def _get_tokenizer(self, continue_at: Optional[str],\n config: Configuration) -> AbstractTokenizer:\n \"\"\" Set up a new tokenizer or load an already initialised one.\n \"\"\"\n from ..tokenizer import factory as tokenizer_factory\n\n if continue_at in ('import-from-file', 'load-data', None):\n # (re)initialise the tokenizer data\n return tokenizer_factory.create_tokenizer(config)\n\n # just load the tokenizer\n return tokenizer_factory.get_tokenizer_for_db(config)\n\n def _finalize_database(self, dsn: str, offline: bool) -> None:\n \"\"\" Determine the database date and set the status accordingly.\n \"\"\"\n with connect(dsn) as conn:\n properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))\n\n try:\n dbdate = status.compute_database_date(conn, offline)\n status.set_status(conn, dbdate)\n LOG.info('Database is at %s.', dbdate)\n except Exception as exc:\n LOG.error('Cannot determine date of database: %s', exc)\n"
},
{
"path": "src/nominatim_db/clicmd/special_phrases.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\n Implementation of the 'special-phrases' command.\n\"\"\"\nimport argparse\nimport logging\nfrom pathlib import Path\n\nfrom ..errors import UsageError\nfrom ..db.connection import connect\nfrom ..tools.special_phrases.sp_importer import SPImporter, SpecialPhraseLoader\nfrom ..tools.special_phrases.sp_wiki_loader import SPWikiLoader\nfrom ..tools.special_phrases.sp_csv_loader import SPCsvLoader\nfrom .args import NominatimArgs\n\n\nLOG = logging.getLogger()\n\n\nclass ImportSpecialPhrases:\n \"\"\"\\\n Import special phrases.\n\n Special phrases are search terms that narrow down the type of object\n that should be searched. For example, you might want to search for\n 'Hotels in Barcelona'. The OSM wiki has a selection of special phrases\n in many languages, which can be imported with this command.\n\n You can also provide your own phrases in a CSV file. The file needs to have\n the following five columns:\n * phrase - the term expected for searching\n * class - the OSM tag key of the object type\n * type - the OSM tag value of the object type\n * operator - the kind of search to be done (one of: in, near, name, -)\n * plural - whether the term is a plural or not (Y/N)\n\n An example file can be found in the Nominatim sources at\n 'test/testdb/full_en_phrases_test.csv'.\n\n The import can be further configured to ignore specific key/value pairs.\n This is particularly useful when importing phrases from the wiki. The\n default configuration excludes some very common tags like building=yes.\n The configuration can be customized by putting a file `phrase-settings.json`\n with custom rules into the project directory or by using the `--config`\n option to point to another configuration file.\n \"\"\"\n\n def add_args(self, parser: argparse.ArgumentParser) -> None:\n group = parser.add_argument_group('Input arguments')\n group.add_argument('--import-from-wiki', action='store_true',\n help='Import special phrases from the OSM wiki to the database')\n group.add_argument('--import-from-csv', metavar='FILE',\n help='Import special phrases from a CSV file')\n group.add_argument('--no-replace', action='store_true',\n help='Keep the old phrases and only add the new ones')\n group.add_argument('--min', type=int, default=0,\n help='Restrict special phrases by minimum occurance')\n\n def run(self, args: NominatimArgs) -> int:\n\n if args.import_from_wiki:\n self.start_import(args, SPWikiLoader(args.config))\n\n if args.import_from_csv:\n if not Path(args.import_from_csv).is_file():\n LOG.fatal(\"CSV file '%s' does not exist.\", args.import_from_csv)\n raise UsageError('Cannot access file.')\n\n self.start_import(args, SPCsvLoader(args.import_from_csv))\n\n return 0\n\n def start_import(self, args: NominatimArgs, loader: SpecialPhraseLoader) -> None:\n \"\"\"\n Create the SPImporter object containing the right\n sp loader and then start the import of special phrases.\n \"\"\"\n from ..tokenizer import factory as tokenizer_factory\n\n tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)\n should_replace = not args.no_replace\n min = args.min\n\n with connect(args.config.get_libpq_dsn()) as db_connection:\n SPImporter(\n args.config, db_connection, loader\n ).import_phrases(tokenizer, should_replace, min)\n"
},
{
"path": "src/nominatim_db/config.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nNominatim configuration accessor.\n\"\"\"\nfrom typing import Union, Dict, Any, List, Mapping, Optional\nimport importlib.util\nimport logging\nimport os\nimport sys\nimport re\nfrom pathlib import Path\nimport json\nimport yaml\n\nfrom dotenv import dotenv_values\n\nfrom psycopg.conninfo import conninfo_to_dict\n\nfrom .typing import StrPath\nfrom .errors import UsageError\nfrom . import paths\n\nLOG = logging.getLogger()\nCONFIG_CACHE: Dict[str, Any] = {}\n\n\ndef flatten_config_list(content: Any, section: str = '') -> List[Any]:\n \"\"\" Flatten YAML configuration lists that contain include sections\n which are lists themselves.\n \"\"\"\n if not content:\n return []\n\n if not isinstance(content, list):\n raise UsageError(f\"List expected in section '{section}'.\")\n\n output = []\n for ele in content:\n if isinstance(ele, list):\n output.extend(flatten_config_list(ele, section))\n else:\n output.append(ele)\n\n return output\n\n\nclass Configuration:\n \"\"\" This class wraps access to the configuration settings\n for the Nominatim instance in use.\n\n All Nominatim configuration options are prefixed with 'NOMINATIM_' to\n avoid conflicts with other environment variables. All settings can\n be accessed as properties of the class under the same name as the\n setting but with the `NOMINATIM_` prefix removed. In addition, there\n are accessor functions that convert the setting values to types\n other than string.\n \"\"\"\n\n def __init__(self, project_dir: Optional[Union[Path, str]],\n environ: Optional[Mapping[str, str]] = None) -> None:\n self.environ = os.environ if environ is None else environ\n self.config_dir = paths.CONFIG_DIR\n self._config = dotenv_values(str(self.config_dir / 'env.defaults'))\n if project_dir is not None:\n self.project_dir: Optional[Path] = Path(project_dir).resolve()\n if (self.project_dir / '.env').is_file():\n self._config.update(dotenv_values(str(self.project_dir / '.env')))\n else:\n self.project_dir = None\n\n class _LibDirs:\n sql = paths.SQLLIB_DIR\n lua = paths.LUALIB_DIR\n data = paths.DATA_DIR\n\n self.lib_dir = _LibDirs()\n self._private_plugins: Dict[str, object] = {}\n\n if re.fullmatch(r'[\\w-]+', self.DATABASE_WEBUSER) is None:\n raise UsageError(\"Misconfigured DATABASE_WEBUSER. \"\n \"Only alphnumberic characters, - and _ are allowed.\")\n\n def set_libdirs(self, **kwargs: StrPath) -> None:\n \"\"\" Set paths to library functions and data.\n \"\"\"\n for key, value in kwargs.items():\n setattr(self.lib_dir, key, None if value is None else Path(value))\n\n def __getattr__(self, name: str) -> str:\n name = 'NOMINATIM_' + name\n\n if name in self.environ:\n return self.environ[name]\n\n return self._config[name] or ''\n\n def get_bool(self, name: str) -> bool:\n \"\"\" Return the given configuration parameter as a boolean.\n\n Parameters:\n name: Name of the configuration parameter with the NOMINATIM_\n prefix removed.\n\n Returns:\n `True` for values of '1', 'yes' and 'true', `False` otherwise.\n \"\"\"\n return getattr(self, name).lower() in ('1', 'yes', 'true')\n\n def get_int(self, name: str) -> int:\n \"\"\" Return the given configuration parameter as an int.\n\n Parameters:\n name: Name of the configuration parameter with the NOMINATIM_\n prefix removed.\n\n Returns:\n The configuration value converted to int.\n\n Raises:\n ValueError: when the value is not a number.\n \"\"\"\n try:\n return int(getattr(self, name))\n except ValueError as exp:\n LOG.fatal(\"Invalid setting NOMINATIM_%s. Needs to be a number.\", name)\n raise UsageError(\"Configuration error.\") from exp\n\n def get_str_list(self, name: str) -> Optional[List[str]]:\n \"\"\" Return the given configuration parameter as a list of strings.\n The values are assumed to be given as a comma-sparated list and\n will be stripped before returning them.\n\n Parameters:\n name: Name of the configuration parameter with the NOMINATIM_\n prefix removed.\n\n Returns:\n (List[str]): The comma-split parameter as a list. The\n elements are stripped of leading and final spaces before\n being returned.\n (None): The configuration parameter was unset or empty.\n \"\"\"\n raw = getattr(self, name)\n\n return [v.strip() for v in raw.split(',')] if raw else None\n\n def get_path(self, name: str) -> Optional[Path]:\n \"\"\" Return the given configuration parameter as a Path.\n\n Parameters:\n name: Name of the configuration parameter with the NOMINATIM_\n prefix removed.\n\n Returns:\n (Path): A Path object of the parameter value.\n If a relative path is configured, then the function converts this\n into an absolute path with the project directory as root path.\n (None): The configuration parameter was unset or empty.\n \"\"\"\n value = getattr(self, name)\n if not value:\n return None\n\n cfgpath = Path(value)\n\n if not cfgpath.is_absolute():\n assert self.project_dir is not None\n cfgpath = self.project_dir / cfgpath\n\n return cfgpath.resolve()\n\n def get_libpq_dsn(self) -> str:\n \"\"\" Get configured database DSN converted into the key/value format\n understood by libpq and psycopg.\n \"\"\"\n dsn = self.DATABASE_DSN\n\n def quote_param(param: str) -> str:\n key, val = param.split('=')\n val = val.replace('\\\\', '\\\\\\\\').replace(\"'\", \"\\\\'\")\n if ' ' in val:\n val = \"'\" + val + \"'\"\n return key + '=' + val\n\n if dsn.startswith('pgsql:'):\n # Old PHP DSN format. Convert before returning.\n return ' '.join([quote_param(p) for p in dsn[6:].split(';')])\n\n return dsn\n\n def get_database_params(self) -> Mapping[str, Union[str, int, None]]:\n \"\"\" Get the configured parameters for the database connection\n as a mapping.\n \"\"\"\n dsn = self.DATABASE_DSN\n\n if dsn.startswith('pgsql:'):\n return dict((p.split('=', 1) for p in dsn[6:].split(';')))\n\n return conninfo_to_dict(dsn) # type: ignore\n\n def get_import_style_file(self) -> Path:\n \"\"\" Return the import style file as a path object. Translates the\n name of the standard styles automatically into a file in the\n config style.\n \"\"\"\n style = getattr(self, 'IMPORT_STYLE')\n\n if style in ('admin', 'street', 'address', 'full', 'extratags'):\n return self.lib_dir.lua / f'import-{style}.lua'\n\n return self.find_config_file('', 'IMPORT_STYLE')\n\n def get_os_env(self) -> Dict[str, str]:\n \"\"\" Return a copy of the OS environment with the Nominatim configuration\n merged in.\n \"\"\"\n env = {k: v for k, v in self._config.items() if v is not None}\n env.update(self.environ)\n\n return env\n\n def load_sub_configuration(self, filename: StrPath,\n config: Optional[str] = None) -> Any:\n \"\"\" Load additional configuration from a file. `filename` is the name\n of the configuration file. The file is first searched in the\n project directory and then in the global settings directory.\n\n If `config` is set, then the name of the configuration file can\n be additionally given through a .env configuration option. When\n the option is set, then the file will be exclusively loaded as set:\n if the name is an absolute path, the file name is taken as is,\n if the name is relative, it is taken to be relative to the\n project directory.\n\n The format of the file is determined from the filename suffix.\n Currently only files with extension '.yaml' are supported.\n\n YAML files support a special '!include' construct. When the\n directive is given, the value is taken to be a filename, the file\n is loaded using this function and added at the position in the\n configuration tree.\n \"\"\"\n configfile = self.find_config_file(filename, config)\n\n if str(configfile) in CONFIG_CACHE:\n return CONFIG_CACHE[str(configfile)]\n\n if configfile.suffix in ('.yaml', '.yml'):\n result = self._load_from_yaml(configfile)\n elif configfile.suffix == '.json':\n with configfile.open('r', encoding='utf-8') as cfg:\n result = json.load(cfg)\n else:\n raise UsageError(f\"Config file '{configfile}' has unknown format.\")\n\n CONFIG_CACHE[str(configfile)] = result\n return result\n\n def load_plugin_module(self, module_name: str, internal_path: str) -> Any:\n \"\"\" Load a Python module as a plugin.\n\n The module_name may have three variants:\n\n * A name without any '.' is assumed to be an internal module\n and will be searched relative to `internal_path`.\n * If the name ends in `.py`, module_name is assumed to be a\n file name relative to the project directory.\n * Any other name is assumed to be an absolute module name.\n\n In either of the variants the module name must start with a letter.\n \"\"\"\n if not module_name or not module_name[0].isidentifier():\n raise UsageError(f'Invalid module name {module_name}')\n\n if '.' not in module_name:\n module_name = module_name.replace('-', '_')\n full_module = f'{internal_path}.{module_name}'\n return sys.modules.get(full_module) or importlib.import_module(full_module)\n\n if module_name.endswith('.py'):\n if self.project_dir is None or not (self.project_dir / module_name).exists():\n raise UsageError(f\"Cannot find module '{module_name}' in project directory.\")\n\n if module_name in self._private_plugins:\n return self._private_plugins[module_name]\n\n file_path = str(self.project_dir / module_name)\n spec = importlib.util.spec_from_file_location(module_name, file_path)\n if spec:\n module = importlib.util.module_from_spec(spec)\n # Do not add to global modules because there is no standard\n # module name that Python can resolve.\n self._private_plugins[module_name] = module\n assert spec.loader is not None\n spec.loader.exec_module(module)\n\n return module\n\n return sys.modules.get(module_name) or importlib.import_module(module_name)\n\n def find_config_file(self, filename: StrPath,\n config: Optional[str] = None) -> Path:\n \"\"\" Resolve the location of a configuration file given a filename and\n an optional configuration option with the file name.\n Raises a UsageError when the file cannot be found or is not\n a regular file.\n \"\"\"\n if config is not None:\n cfg_value = getattr(self, config)\n if cfg_value:\n cfg_filename = Path(cfg_value)\n\n if cfg_filename.is_absolute():\n cfg_filename = cfg_filename.resolve()\n\n if not cfg_filename.is_file():\n LOG.fatal(\"Cannot find config file '%s'.\", cfg_filename)\n raise UsageError(\"Config file not found.\")\n\n return cfg_filename\n\n filename = cfg_filename\n\n search_paths = [self.project_dir, self.config_dir]\n for path in search_paths:\n if path is not None and (path / filename).is_file():\n return path / filename\n\n LOG.fatal(\"Configuration file '%s' not found.\\nDirectories searched: %s\",\n filename, search_paths)\n raise UsageError(\"Config file not found.\")\n\n def _load_from_yaml(self, cfgfile: Path) -> Any:\n \"\"\" Load a YAML configuration file. This installs a special handler that\n allows to include other YAML files using the '!include' operator.\n \"\"\"\n yaml.add_constructor('!include', self._yaml_include_representer,\n Loader=yaml.SafeLoader)\n return yaml.safe_load(cfgfile.read_text(encoding='utf-8'))\n\n def _yaml_include_representer(self, loader: Any, node: yaml.Node) -> Any:\n \"\"\" Handler for the '!include' operator in YAML files.\n\n When the filename is relative, then the file is first searched in the\n project directory and then in the global settings directory.\n \"\"\"\n fname = loader.construct_scalar(node)\n\n if Path(fname).is_absolute():\n configfile = Path(fname)\n else:\n configfile = self.find_config_file(loader.construct_scalar(node))\n\n if configfile.suffix != '.yaml':\n LOG.fatal(\"Format error while reading '%s': only YAML format supported.\",\n configfile)\n raise UsageError(\"Cannot handle config file format.\")\n\n return yaml.safe_load(configfile.read_text(encoding='utf-8'))\n"
},
{
"path": "src/nominatim_db/data/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/data/country_info.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for importing and managing static country information.\n\"\"\"\nfrom typing import Dict, Any, Iterable, Tuple, Optional, Container, overload\nfrom pathlib import Path\n\nfrom ..db import utils as db_utils\nfrom ..db.connection import connect, Connection, register_hstore\nfrom ..errors import UsageError\nfrom ..config import Configuration\nfrom ..tokenizer.base import AbstractTokenizer\n\n\ndef _flatten_name_list(names: Any) -> Dict[str, str]:\n if names is None:\n return {}\n\n if not isinstance(names, dict):\n raise UsageError(\"Expected key-value list for names in country_settings.py\")\n\n flat = {}\n for prefix, remain in names.items():\n if isinstance(remain, str):\n flat[prefix] = remain\n elif not isinstance(remain, dict):\n raise UsageError(\"Entries in names must be key-value lists.\")\n else:\n for suffix, name in remain.items():\n if suffix == 'default':\n flat[prefix] = name\n else:\n flat[f'{prefix}:{suffix}'] = name\n\n return flat\n\n\nclass _CountryInfo:\n \"\"\" Caches country-specific properties from the configuration file.\n \"\"\"\n\n def __init__(self) -> None:\n self._info: Dict[str, Dict[str, Any]] = {}\n\n def load(self, config: Configuration) -> None:\n \"\"\" Load the country properties from the configuration files,\n if they are not loaded yet.\n \"\"\"\n if not self._info:\n self._info = config.load_sub_configuration('country_settings.yaml')\n for prop in self._info.values():\n # Convert languages into a list for simpler handling.\n if 'languages' not in prop:\n prop['languages'] = []\n elif not isinstance(prop['languages'], list):\n prop['languages'] = [x.strip()\n for x in prop['languages'].split(',')]\n prop['names'] = _flatten_name_list(prop.get('names'))\n\n def items(self) -> Iterable[Tuple[str, Dict[str, Any]]]:\n \"\"\" Return tuples of (country_code, property dict) as iterable.\n \"\"\"\n return self._info.items()\n\n def get(self, country_code: str) -> Dict[str, Any]:\n \"\"\" Get country information for the country with the given country code.\n \"\"\"\n return self._info.get(country_code, {})\n\n\n_COUNTRY_INFO = _CountryInfo()\n\n\ndef setup_country_config(config: Configuration) -> None:\n \"\"\" Load country properties from the configuration file.\n Needs to be called before using any other functions in this\n file.\n \"\"\"\n _COUNTRY_INFO.load(config)\n\n\n@overload\ndef iterate() -> Iterable[Tuple[str, Dict[str, Any]]]:\n ...\n\n\n@overload\ndef iterate(prop: str) -> Iterable[Tuple[str, Any]]:\n ...\n\n\ndef iterate(prop: Optional[str] = None) -> Iterable[Tuple[str, Dict[str, Any]]]:\n \"\"\" Iterate over country code and properties.\n\n When `prop` is None, all countries are returned with their complete\n set of properties.\n\n If `prop` is given, then only countries are returned where the\n given property is set. The second item of the tuple contains only\n the content of the given property.\n \"\"\"\n if prop is None:\n return _COUNTRY_INFO.items()\n\n return ((c, p[prop]) for c, p in _COUNTRY_INFO.items() if prop in p)\n\n\ndef setup_country_tables(dsn: str, sql_dir: Path, ignore_partitions: bool = False) -> None:\n \"\"\" Create and populate the tables with basic static data that provides\n the background for geocoding. Data is assumed to not yet exist.\n \"\"\"\n db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')\n\n params = []\n for ccode, props in _COUNTRY_INFO.items():\n if ccode is not None and props is not None:\n if ignore_partitions:\n partition = 0\n else:\n partition = props.get('partition', 0)\n lang = props['languages'][0] if len(\n props['languages']) == 1 else None\n\n params.append((ccode, props['names'], lang, partition))\n with connect(dsn) as conn:\n register_hstore(conn)\n with conn.cursor() as cur:\n cur.execute(\n \"\"\" CREATE TABLE public.country_name (\n country_code character varying(2),\n name public.hstore,\n derived_name public.hstore,\n country_default_language_code text,\n partition integer\n ); \"\"\")\n cur.executemany(\n \"\"\" INSERT INTO public.country_name\n (country_code, name, country_default_language_code, partition)\n VALUES (%s, %s, %s, %s)\n \"\"\", params)\n conn.commit()\n\n\ndef create_country_names(conn: Connection, tokenizer: AbstractTokenizer,\n languages: Optional[Container[str]] = None) -> None:\n \"\"\" Add default country names to search index. `languages` is a comma-\n separated list of language codes as used in OSM. If `languages` is not\n empty then only name translations for the given languages are added\n to the index.\n \"\"\"\n def _include_key(key: str) -> bool:\n return ':' not in key or not languages or \\\n key[key.index(':') + 1:] in languages\n\n register_hstore(conn)\n with conn.cursor() as cur:\n cur.execute(\"\"\"SELECT country_code, name FROM country_name\n WHERE country_code is not null\"\"\")\n\n with tokenizer.name_analyzer() as analyzer:\n for code, name in cur:\n names = {'countrycode': code}\n\n # country names (only in languages as provided)\n if name:\n names.update({k: v for k, v in name.items() if _include_key(k)})\n\n analyzer.add_country_names(code, names)\n\n conn.commit()\n"
},
{
"path": "src/nominatim_db/data/place_info.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nWrapper around place information the indexer gets from the database and hands to\nthe tokenizer.\n\"\"\"\nfrom typing import Optional, Mapping, Any, Tuple, cast\n\n\nclass PlaceInfo:\n \"\"\" This data class contains all information the tokenizer can access\n about a place.\n \"\"\"\n\n def __init__(self, info: Mapping[str, Any]) -> None:\n self._info = info\n\n @property\n def name(self) -> Optional[Mapping[str, str]]:\n \"\"\" A dictionary with the names of the place. Keys and values represent\n the full key and value of the corresponding OSM tag. Which tags\n are saved as names is determined by the import style.\n The property may be None if the place has no names.\n \"\"\"\n return self._info.get('name')\n\n @property\n def address(self) -> Optional[Mapping[str, str]]:\n \"\"\" A dictionary with the address elements of the place. They key\n usually corresponds to the suffix part of the key of an OSM\n 'addr:*' or 'isin:*' tag. There are also some special keys like\n `country` or `country_code` which merge OSM keys that contain\n the same information. See [Import Styles][1] for details.\n\n The property may be None if the place has no address information.\n\n [1]: ../customize/Import-Styles.md\n \"\"\"\n return self._info.get('address')\n\n @property\n def country_code(self) -> Optional[str]:\n \"\"\" The country code of the country the place is in. Guaranteed\n to be a two-letter lower-case string. If the place is not inside\n any country, the property is set to None.\n \"\"\"\n return self._info.get('country_code')\n\n @property\n def rank_address(self) -> int:\n \"\"\" The [rank address][1] before any rank correction is applied.\n\n [1]: ../customize/Ranking.md#address-rank\n \"\"\"\n return cast(int, self._info.get('rank_address', 0))\n\n @property\n def centroid(self) -> Optional[Tuple[float, float]]:\n \"\"\" A center point of the place in WGS84. May be None when the\n geometry of the place is unknown.\n \"\"\"\n x, y = self._info.get('centroid_x'), self._info.get('centroid_y')\n return None if x is None or y is None else (x, y)\n\n def is_a(self, key: str, value: str) -> bool:\n \"\"\" Set to True when the place's primary tag corresponds to the given\n key and value.\n \"\"\"\n return self._info.get('class') == key and self._info.get('type') == value\n\n def is_country(self) -> bool:\n \"\"\" Set to True when the place is a valid country boundary.\n \"\"\"\n return self.rank_address == 4 \\\n and self.is_a('boundary', 'administrative') \\\n and self.country_code is not None\n"
},
{
"path": "src/nominatim_db/data/place_name.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nData class for a single name of a place.\n\"\"\"\nfrom typing import Optional, Dict, Mapping\n\n\nclass PlaceName:\n \"\"\" Each name and address part of a place is encapsulated in an object of\n this class. It saves not only the name proper but also describes the\n kind of name with two properties:\n\n * `kind` describes the name of the OSM key used without any suffixes\n (i.e. the part after the colon removed)\n * `suffix` contains the suffix of the OSM tag, if any. The suffix\n is the part of the key after the first colon.\n\n In addition to that, a name may have arbitrary additional attributes.\n How attributes are used, depends on the sanitizers and token analysers.\n The exception is the 'analyzer' attribute. This attribute determines\n which token analysis module will be used to finalize the treatment of\n names.\n \"\"\"\n\n def __init__(self, name: str, kind: str, suffix: Optional[str]):\n self.name = name\n self.kind = kind\n self.suffix = suffix\n self.attr: Dict[str, str] = {}\n\n def __repr__(self) -> str:\n return f\"PlaceName(name={self.name!r},kind={self.kind!r},suffix={self.suffix!r})\"\n\n def clone(self, name: Optional[str] = None,\n kind: Optional[str] = None,\n suffix: Optional[str] = None,\n attr: Optional[Mapping[str, str]] = None) -> 'PlaceName':\n \"\"\" Create a deep copy of the place name, optionally with the\n given parameters replaced. In the attribute list only the given\n keys are updated. The list is not replaced completely.\n In particular, the function cannot to be used to remove an\n attribute from a place name.\n \"\"\"\n newobj = PlaceName(name or self.name,\n kind or self.kind,\n suffix or self.suffix)\n\n newobj.attr.update(self.attr)\n if attr:\n newobj.attr.update(attr)\n\n return newobj\n\n def set_attr(self, key: str, value: str) -> None:\n \"\"\" Add the given property to the name. If the property was already\n set, then the value is overwritten.\n \"\"\"\n self.attr[key] = value\n\n def get_attr(self, key: str, default: Optional[str] = None) -> Optional[str]:\n \"\"\" Return the given property or the value of 'default' if it\n is not set.\n \"\"\"\n return self.attr.get(key, default)\n\n def has_attr(self, key: str) -> bool:\n \"\"\" Check if the given attribute is set.\n \"\"\"\n return key in self.attr\n"
},
{
"path": "src/nominatim_db/data/postcode_format.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for formatting postcodes according to their country-specific\nformat.\n\"\"\"\nfrom typing import Any, Mapping, Optional, Set, Match\nimport re\n\nfrom ..errors import UsageError\nfrom . import country_info\n\n\nclass CountryPostcodeMatcher:\n \"\"\" Matches and formats a postcode according to a format definition\n of the given country.\n \"\"\"\n def __init__(self, country_code: str, config: Mapping[str, Any]) -> None:\n if 'pattern' not in config:\n raise UsageError(\"Field 'pattern' required for 'postcode' \"\n f\"for country '{country_code}'\")\n\n pc_pattern = config['pattern'].replace('d', '[0-9]').replace('l', '[A-Z]')\n\n self.norm_pattern = re.compile(f'\\\\s*(?:{country_code.upper()}[ -]?)?({pc_pattern})\\\\s*')\n self.pattern = re.compile(pc_pattern)\n\n # We want to exclude 0000, 00-000, 000 00 etc\n self.zero_pattern = re.compile(r'^[0\\- ]+$')\n\n self.output = config.get('output', r'\\g<0>')\n\n def match(self, postcode: str) -> Optional[Match[str]]:\n \"\"\" Match the given postcode against the postcode pattern for this\n matcher. Returns a `re.Match` object if the match was successful\n and None otherwise.\n \"\"\"\n # Upper-case, strip spaces and leading country code.\n normalized = self.norm_pattern.fullmatch(postcode.upper())\n\n if normalized:\n match = self.pattern.fullmatch(normalized.group(1))\n if match and self.zero_pattern.match(match.string):\n return None\n return match\n\n return None\n\n def normalize(self, match: Match[str]) -> str:\n \"\"\" Return the default format of the postcode for the given match.\n `match` must be a `re.Match` object previously returned by\n `match()`\n \"\"\"\n return match.expand(self.output)\n\n\nclass PostcodeFormatter:\n \"\"\" Container for different postcode formats of the world and\n access functions.\n \"\"\"\n def __init__(self) -> None:\n # Objects without a country code can't have a postcode per definition.\n self.country_without_postcode: Set[Optional[str]] = {None}\n self.country_matcher = {}\n self.default_matcher = CountryPostcodeMatcher('', {'pattern': '.*'})\n self.postcode_extent: dict[Optional[str], int] = {}\n\n for ccode, prop in country_info.iterate('postcode'):\n if prop is False:\n self.country_without_postcode.add(ccode)\n elif isinstance(prop, dict):\n self.country_matcher[ccode] = CountryPostcodeMatcher(ccode, prop)\n if 'extent' in prop:\n self.postcode_extent[ccode] = int(prop['extent'])\n else:\n raise UsageError(f\"Invalid entry 'postcode' for country '{ccode}'\")\n\n def set_default_pattern(self, pattern: str) -> None:\n \"\"\" Set the postcode match pattern to use, when a country does not\n have a specific pattern.\n \"\"\"\n self.default_matcher = CountryPostcodeMatcher('', {'pattern': pattern})\n\n def get_matcher(self, country_code: Optional[str]) -> Optional[CountryPostcodeMatcher]:\n \"\"\" Return the CountryPostcodeMatcher for the given country.\n Returns None if the country doesn't have a postcode and the\n default matcher if there is no specific matcher configured for\n the country.\n \"\"\"\n if country_code in self.country_without_postcode:\n return None\n\n assert country_code is not None\n\n return self.country_matcher.get(country_code, self.default_matcher)\n\n def match(self, country_code: Optional[str], postcode: str) -> Optional[Match[str]]:\n \"\"\" Match the given postcode against the postcode pattern for this\n matcher. Returns a `re.Match` object if the country has a pattern\n and the match was successful or None if the match failed.\n \"\"\"\n if country_code in self.country_without_postcode:\n return None\n\n assert country_code is not None\n\n return self.country_matcher.get(country_code, self.default_matcher).match(postcode)\n\n def normalize(self, country_code: str, match: Match[str]) -> str:\n \"\"\" Return the default format of the postcode for the given match.\n `match` must be a `re.Match` object previously returned by\n `match()`\n \"\"\"\n return self.country_matcher.get(country_code, self.default_matcher).normalize(match)\n\n def get_postcode_extent(self, country_code: Optional[str]) -> int:\n \"\"\" Return the extent (in m) to use for the given country. If no\n specific extent is set, then the default of 5km will be returned.\n \"\"\"\n return self.postcode_extent.get(country_code, 5000)\n"
},
{
"path": "src/nominatim_db/db/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/db/connection.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSpecialised connection and cursor functions.\n\"\"\"\nfrom typing import Optional, Any, Dict, Tuple\nimport logging\nimport os\n\nimport psycopg\nimport psycopg.types.hstore\nfrom psycopg import sql as pysql\n\nfrom ..typing import SysEnv\nfrom ..errors import UsageError\n\nLOG = logging.getLogger()\n\nCursor = psycopg.Cursor[Any]\nConnection = psycopg.Connection[Any]\n\n\ndef execute_scalar(conn: Connection, sql: psycopg.abc.Query, args: Any = None) -> Any:\n \"\"\" Execute query that returns a single value. The value is returned.\n If the query yields more than one row, a ValueError is raised.\n \"\"\"\n with conn.cursor(row_factory=psycopg.rows.tuple_row) as cur:\n cur.execute(sql, args)\n\n if cur.rowcount != 1:\n raise RuntimeError(\"Query did not return a single row.\")\n\n result = cur.fetchone()\n\n assert result is not None\n return result[0]\n\n\ndef table_exists(conn: Connection, table: str) -> bool:\n \"\"\" Check that a table with the given name exists in the database.\n \"\"\"\n num = execute_scalar(\n conn,\n \"\"\"SELECT count(*) FROM pg_tables\n WHERE tablename = %s and schemaname = 'public'\"\"\", (table, ))\n return num == 1 if isinstance(num, int) else False\n\n\ndef table_has_column(conn: Connection, table: str, column: str) -> bool:\n \"\"\" Check if the table 'table' exists and has a column with name 'column'.\n \"\"\"\n has_column = execute_scalar(conn,\n \"\"\"SELECT count(*) FROM information_schema.columns\n WHERE table_name = %s and column_name = %s\"\"\",\n (table, column))\n return has_column > 0 if isinstance(has_column, int) else False\n\n\ndef index_exists(conn: Connection, index: str, table: Optional[str] = None) -> bool:\n \"\"\" Check that an index with the given name exists in the database.\n If table is not None then the index must relate to the given\n table.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"\"\"SELECT tablename FROM pg_indexes\n WHERE indexname = %s and schemaname = 'public'\"\"\", (index, ))\n if cur.rowcount == 0:\n return False\n\n if table is not None:\n row = cur.fetchone()\n if row is None or not isinstance(row[0], str):\n return False\n return row[0] == table\n\n return True\n\n\ndef drop_tables(conn: Connection, *names: str,\n if_exists: bool = True, cascade: bool = False) -> None:\n \"\"\" Drop one or more tables with the given names.\n Set `if_exists` to False if a non-existent table should raise\n an exception instead of just being ignored. `cascade` will cause\n depended objects to be dropped as well.\n The caller needs to take care of committing the change.\n \"\"\"\n sql = pysql.SQL('DROP TABLE%s{}%s' % (\n ' IF EXISTS ' if if_exists else ' ',\n ' CASCADE' if cascade else ''))\n\n with conn.cursor() as cur:\n for name in names:\n cur.execute(sql.format(pysql.Identifier(name)))\n\n\ndef server_version_tuple(conn: Connection) -> Tuple[int, int]:\n \"\"\" Return the server version as a tuple of (major, minor).\n Converts correctly for pre-10 and post-10 PostgreSQL versions.\n \"\"\"\n version = conn.info.server_version\n major, minor = divmod(version, 10000)\n if major < 10:\n minor //= 100\n return major, minor\n\n\ndef postgis_version_tuple(conn: Connection) -> Tuple[int, int]:\n \"\"\" Return the postgis version installed in the database as a\n tuple of (major, minor). Assumes that the PostGIS extension\n has been installed already.\n \"\"\"\n version = execute_scalar(conn, 'SELECT postgis_lib_version()')\n\n version_parts = version.split('.')\n if len(version_parts) < 2:\n raise UsageError(f\"Error fetching Postgis version. Bad format: {version}\")\n\n return (int(version_parts[0]), int(version_parts[1]))\n\n\ndef register_hstore(conn: Connection) -> None:\n \"\"\" Register the hstore type with psycopg for the connection.\n \"\"\"\n info = psycopg.types.TypeInfo.fetch(conn, \"hstore\")\n if info is None:\n raise RuntimeError('Hstore extension is requested but not installed.')\n psycopg.types.hstore.register_hstore(info, conn)\n\n\ndef connect(dsn: str, **kwargs: Any) -> Connection:\n \"\"\" Open a connection to the database using the specialised connection\n factory. The returned object may be used in conjunction with 'with'.\n When used outside a context manager, use the `connection` attribute\n to get the connection.\n \"\"\"\n try:\n return psycopg.connect(dsn, row_factory=psycopg.rows.namedtuple_row, **kwargs)\n except psycopg.OperationalError as err:\n raise UsageError(f\"Cannot connect to database: {err}\") from err\n\n\n# Translation from PG connection string parameters to PG environment variables.\n# Derived from https://www.postgresql.org/docs/current/libpq-envars.html.\n_PG_CONNECTION_STRINGS = {\n 'host': 'PGHOST',\n 'hostaddr': 'PGHOSTADDR',\n 'port': 'PGPORT',\n 'dbname': 'PGDATABASE',\n 'user': 'PGUSER',\n 'password': 'PGPASSWORD',\n 'passfile': 'PGPASSFILE',\n 'channel_binding': 'PGCHANNELBINDING',\n 'service': 'PGSERVICE',\n 'options': 'PGOPTIONS',\n 'application_name': 'PGAPPNAME',\n 'sslmode': 'PGSSLMODE',\n 'requiressl': 'PGREQUIRESSL',\n 'sslcompression': 'PGSSLCOMPRESSION',\n 'sslcert': 'PGSSLCERT',\n 'sslkey': 'PGSSLKEY',\n 'sslrootcert': 'PGSSLROOTCERT',\n 'sslcrl': 'PGSSLCRL',\n 'requirepeer': 'PGREQUIREPEER',\n 'ssl_min_protocol_version': 'PGSSLMINPROTOCOLVERSION',\n 'ssl_max_protocol_version': 'PGSSLMAXPROTOCOLVERSION',\n 'gssencmode': 'PGGSSENCMODE',\n 'krbsrvname': 'PGKRBSRVNAME',\n 'gsslib': 'PGGSSLIB',\n 'connect_timeout': 'PGCONNECT_TIMEOUT',\n 'target_session_attrs': 'PGTARGETSESSIONATTRS',\n}\n\n\ndef get_pg_env(dsn: str,\n base_env: Optional[SysEnv] = None) -> Dict[str, str]:\n \"\"\" Return a copy of `base_env` with the environment variables for\n PostgreSQL set up from the given database connection string.\n If `base_env` is None, then the OS environment is used as a base\n environment.\n \"\"\"\n env = dict(base_env if base_env is not None else os.environ)\n\n for param, value in psycopg.conninfo.conninfo_to_dict(dsn).items():\n if param in _PG_CONNECTION_STRINGS:\n env[_PG_CONNECTION_STRINGS[param]] = str(value)\n else:\n LOG.error(\"Unknown connection parameter '%s' ignored.\", param)\n\n return env\n\n\nasync def run_async_query(dsn: str, query: psycopg.abc.Query) -> None:\n \"\"\" Open a connection to the database and run a single query\n asynchronously.\n \"\"\"\n async with await psycopg.AsyncConnection.connect(dsn) as aconn:\n await aconn.execute(query)\n"
},
{
"path": "src/nominatim_db/db/properties.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nQuery and access functions for the in-database property table.\n\"\"\"\nfrom typing import Optional, cast\n\nfrom .connection import Connection, table_exists\n\n\ndef set_property(conn: Connection, name: str, value: str) -> None:\n \"\"\" Add or replace the property with the given name.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute('SELECT value FROM nominatim_properties WHERE property = %s',\n (name, ))\n\n if cur.rowcount == 0:\n sql = 'INSERT INTO nominatim_properties (value, property) VALUES (%s, %s)'\n else:\n sql = 'UPDATE nominatim_properties SET value = %s WHERE property = %s'\n\n cur.execute(sql, (value, name))\n conn.commit()\n\n\ndef get_property(conn: Connection, name: str) -> Optional[str]:\n \"\"\" Return the current value of the given property or None if the property\n is not set.\n \"\"\"\n if not table_exists(conn, 'nominatim_properties'):\n return None\n\n with conn.cursor() as cur:\n cur.execute('SELECT value FROM nominatim_properties WHERE property = %s',\n (name, ))\n\n if cur.rowcount == 0:\n return None\n\n result = cur.fetchone()\n assert result is not None\n\n return cast(Optional[str], result[0])\n"
},
{
"path": "src/nominatim_db/db/query_pool.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nA connection pool that executes incoming queries in parallel.\n\"\"\"\nfrom typing import Any, Tuple, Optional\nimport asyncio\nimport logging\nimport time\n\nimport psycopg\n\nLOG = logging.getLogger()\n\nQueueItem = Optional[Tuple[psycopg.abc.Query, Any]]\n\n\nclass QueryPool:\n \"\"\" Pool to run SQL queries in parallel asynchronous execution.\n\n All queries are run in autocommit mode. If parallel execution leads\n to a deadlock, then the query is repeated.\n The results of the queries is discarded.\n \"\"\"\n def __init__(self, dsn: str, pool_size: int = 1, **conn_args: Any) -> None:\n self.is_cancelled = False\n self.wait_time = 0.0\n self.query_queue: 'asyncio.Queue[QueueItem]' = asyncio.Queue(maxsize=2 * pool_size)\n\n self.pool = [asyncio.create_task(self._worker_loop_cancellable(dsn, **conn_args))\n for _ in range(pool_size)]\n\n async def put_query(self, query: psycopg.abc.Query, params: Any) -> None:\n \"\"\" Schedule a query for execution.\n \"\"\"\n if self.is_cancelled:\n self.clear_queue()\n await self.finish()\n return\n\n tstart = time.time()\n await self.query_queue.put((query, params))\n self.wait_time += time.time() - tstart\n await asyncio.sleep(0)\n\n if self.is_cancelled:\n self.clear_queue()\n await self.finish()\n\n async def finish(self) -> None:\n \"\"\" Wait for all queries to finish and close the pool.\n \"\"\"\n for _ in self.pool:\n await self.query_queue.put(None)\n\n tstart = time.time()\n await asyncio.wait(self.pool)\n self.wait_time += time.time() - tstart\n\n for task in self.pool:\n excp = task.exception()\n if excp is not None:\n raise excp\n\n def clear_queue(self) -> None:\n \"\"\" Drop all items silently that might still be queued.\n \"\"\"\n try:\n while True:\n self.query_queue.get_nowait()\n except asyncio.QueueEmpty:\n pass # expected\n\n async def _worker_loop_cancellable(self, dsn: str, **conn_args: Any) -> None:\n try:\n await self._worker_loop(dsn, **conn_args)\n except Exception as e:\n # Make sure the exception is forwarded to the main function\n self.is_cancelled = True\n # clear the queue here to ensure that any put() that may be blocked returns\n self.clear_queue()\n raise e\n\n async def _worker_loop(self, dsn: str, **conn_args: Any) -> None:\n conn_args['autocommit'] = True\n aconn = await psycopg.AsyncConnection.connect(dsn, **conn_args)\n async with aconn:\n async with aconn.cursor() as cur:\n item = await self.query_queue.get()\n while item is not None:\n try:\n if item[1] is None:\n await cur.execute(item[0])\n else:\n await cur.execute(item[0], item[1])\n\n item = await self.query_queue.get()\n except psycopg.errors.DeadlockDetected:\n assert item is not None\n LOG.info(\"Deadlock detected (sql = %s, params = %s), retry.\",\n str(item[0]), str(item[1]))\n # item is still valid here, causing a retry\n\n async def __aenter__(self) -> 'QueryPool':\n return self\n\n async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:\n await self.finish()\n"
},
{
"path": "src/nominatim_db/db/sql_preprocessor.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nPreprocessing of SQL files.\n\"\"\"\nfrom typing import Set, Dict, Any, cast\nimport re\n\nimport jinja2\n\nfrom .connection import Connection\nfrom ..config import Configuration\nfrom ..db.query_pool import QueryPool\n\n\ndef _get_partitions(conn: Connection) -> Set[int]:\n \"\"\" Get the set of partitions currently in use.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute('SELECT DISTINCT partition FROM country_name')\n partitions = set([0])\n for row in cur:\n partitions.add(row[0])\n\n return partitions\n\n\ndef _get_tables(conn: Connection) -> Set[str]:\n \"\"\" Return the set of tables currently in use.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"SELECT tablename FROM pg_tables WHERE schemaname = 'public'\")\n\n # paranoia check: make sure we don't get table names that cause\n # an SQL injection later\n return {row[0] for row in list(cur) if re.fullmatch(r'\\w+', row[0])}\n\n\ndef _get_middle_db_format(conn: Connection, tables: Set[str]) -> str:\n \"\"\" Returns the version of the slim middle tables.\n \"\"\"\n if 'osm2pgsql_properties' not in tables:\n return '1'\n\n with conn.cursor() as cur:\n cur.execute(\"SELECT value FROM osm2pgsql_properties WHERE property = 'db_format'\")\n row = cur.fetchone()\n\n return cast(str, row[0]) if row is not None else '1'\n\n\ndef _setup_tablespace_sql(config: Configuration) -> Dict[str, str]:\n \"\"\" Returns a dict with tablespace expressions for the different tablespace\n kinds depending on whether a tablespace is configured or not.\n \"\"\"\n out = {}\n for subset in ('ADDRESS', 'SEARCH', 'AUX'):\n for kind in ('DATA', 'INDEX'):\n tspace = getattr(config, f'TABLESPACE_{subset}_{kind}')\n if tspace:\n tspace = f'TABLESPACE \"{tspace}\"'\n out[f'{subset.lower()}_{kind.lower()}'] = tspace\n\n return out\n\n\ndef _setup_postgresql_features(conn: Connection) -> Dict[str, Any]:\n \"\"\" Set up a dictionary with various optional Postgresql/Postgis features that\n depend on the database version.\n \"\"\"\n return {}\n\n\nclass SQLPreprocessor:\n \"\"\" A environment for preprocessing SQL files from the\n lib-sql directory.\n\n The preprocessor provides a number of default filters and variables.\n The variables may be overwritten when rendering an SQL file.\n\n The preprocessing is currently based on the jinja2 templating library\n and follows its syntax.\n \"\"\"\n\n def __init__(self, conn: Connection, config: Configuration) -> None:\n self.env = jinja2.Environment(autoescape=False,\n loader=jinja2.FileSystemLoader(str(config.lib_dir.sql)))\n\n db_info: Dict[str, Any] = {}\n db_info['partitions'] = _get_partitions(conn)\n db_info['tables'] = _get_tables(conn)\n db_info['reverse_only'] = 'search_name' not in db_info['tables']\n db_info['tablespace'] = _setup_tablespace_sql(config)\n db_info['middle_db_format'] = _get_middle_db_format(conn, db_info['tables'])\n\n self.env.globals['config'] = config\n self.env.globals['db'] = db_info\n self.env.globals['postgres'] = _setup_postgresql_features(conn)\n\n def run_string(self, conn: Connection, template: str, **kwargs: Any) -> None:\n \"\"\" Execute the given SQL template string on the connection.\n The keyword arguments may supply additional parameters\n for preprocessing.\n \"\"\"\n sql = self.env.from_string(template).render(**kwargs)\n\n with conn.cursor() as cur:\n cur.execute(sql)\n conn.commit()\n\n def run_sql_file(self, conn: Connection, name: str, **kwargs: Any) -> None:\n \"\"\" Execute the given SQL file on the connection. The keyword arguments\n may supply additional parameters for preprocessing.\n \"\"\"\n sql = self.env.get_template(name).render(**kwargs)\n\n with conn.cursor() as cur:\n cur.execute(sql)\n conn.commit()\n\n async def run_parallel_sql_file(self, dsn: str, name: str, num_threads: int = 1,\n **kwargs: Any) -> None:\n \"\"\" Execute the given SQL files using parallel asynchronous connections.\n The keyword arguments may supply additional parameters for\n preprocessing.\n\n After preprocessing the SQL code is cut at lines containing only\n '---'. Each chunk is sent to one of the `num_threads` workers.\n \"\"\"\n sql = self.env.get_template(name).render(**kwargs)\n\n parts = sql.split('\\n---\\n')\n\n async with QueryPool(dsn, num_threads) as pool:\n for part in parts:\n await pool.put_query(part, None)\n"
},
{
"path": "src/nominatim_db/db/status.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nAccess and helper functions for the status and status log table.\n\"\"\"\nfrom typing import Optional, Tuple\nimport datetime as dt\nimport logging\nimport re\n\nfrom .connection import Connection, table_exists, execute_scalar\nfrom ..utils.url_utils import get_url\nfrom ..errors import UsageError\n\nLOG = logging.getLogger()\nISODATE_FORMAT = '%Y-%m-%dT%H:%M:%S'\n\n\ndef compute_database_date(conn: Connection, offline: bool = False) -> dt.datetime:\n \"\"\" Determine the date of the database from the newest object in the\n data base.\n \"\"\"\n # If there is a date from osm2pgsql available, use that.\n if table_exists(conn, 'osm2pgsql_properties'):\n with conn.cursor() as cur:\n cur.execute(\"\"\" SELECT value FROM osm2pgsql_properties\n WHERE property = 'current_timestamp' \"\"\")\n row = cur.fetchone()\n if row is not None:\n return dt.datetime.strptime(row[0], \"%Y-%m-%dT%H:%M:%SZ\")\\\n .replace(tzinfo=dt.timezone.utc)\n\n if offline:\n raise UsageError(\"Cannot determine database date from data in offline mode.\")\n\n # Else, find the node with the highest ID in the database\n if table_exists(conn, 'place'):\n osmid = execute_scalar(conn, \"SELECT max(osm_id) FROM place WHERE osm_type='N'\")\n else:\n osmid = execute_scalar(conn, \"SELECT max(osm_id) FROM placex WHERE osm_type='N'\")\n\n if osmid is None:\n LOG.fatal(\"No data found in the database.\")\n raise UsageError(\"No data found in the database.\")\n\n LOG.info(\"Using node id %d for timestamp lookup\", osmid)\n # Get the node from the API to find the timestamp when it was created.\n node_url = f'https://www.openstreetmap.org/api/0.6/node/{osmid}/1'\n data = get_url(node_url)\n\n match = re.search(r'timestamp=\"((\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}))Z\"', data)\n\n if match is None:\n LOG.fatal(\"The node data downloaded from the API does not contain valid data.\\n\"\n \"URL used: %s\", node_url)\n raise UsageError(\"Bad API data.\")\n\n LOG.debug(\"Found timestamp %s\", match.group(1))\n\n return dt.datetime.strptime(match.group(1), ISODATE_FORMAT).replace(tzinfo=dt.timezone.utc)\n\n\ndef set_status(conn: Connection, date: Optional[dt.datetime],\n seq: Optional[int] = None, indexed: bool = True) -> None:\n \"\"\" Replace the current status with the given status. If date is `None`\n then only sequence and indexed will be updated as given. Otherwise\n the whole status is replaced.\n The change will be committed to the database.\n \"\"\"\n assert date is None or date.tzinfo == dt.timezone.utc\n with conn.cursor() as cur:\n if date is None:\n cur.execute(\"UPDATE import_status set sequence_id = %s, indexed = %s\",\n (seq, indexed))\n else:\n cur.execute(\"TRUNCATE TABLE import_status\")\n cur.execute(\"\"\"INSERT INTO import_status (lastimportdate, sequence_id, indexed)\n VALUES (%s, %s, %s)\"\"\", (date, seq, indexed))\n\n conn.commit()\n\n\ndef get_status(conn: Connection) -> Tuple[Optional[dt.datetime], Optional[int], Optional[bool]]:\n \"\"\" Return the current status as a triple of (date, sequence, indexed).\n If status has not been set up yet, a triple of None is returned.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"SELECT * FROM import_status LIMIT 1\")\n if cur.rowcount < 1:\n return None, None, None\n\n row = cur.fetchone()\n assert row\n return row.lastimportdate, row.sequence_id, row.indexed\n\n\ndef set_indexed(conn: Connection, state: bool) -> None:\n \"\"\" Set the indexed flag in the status table to the given state.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"UPDATE import_status SET indexed = %s\", (state, ))\n conn.commit()\n\n\ndef log_status(conn: Connection, start: dt.datetime,\n event: str, batchsize: Optional[int] = None) -> None:\n \"\"\" Write a new status line to the `import_osmosis_log` table.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"\"\"INSERT INTO import_osmosis_log\n (batchend, batchseq, batchsize, starttime, endtime, event)\n SELECT lastimportdate, sequence_id, %s, %s, now(), %s FROM import_status\"\"\",\n (batchsize, start, event))\n conn.commit()\n"
},
{
"path": "src/nominatim_db/db/utils.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelper functions for handling DB accesses.\n\"\"\"\nfrom typing import IO, Optional, Union\nimport subprocess\nimport logging\nimport gzip\nfrom pathlib import Path\n\nfrom .connection import get_pg_env\nfrom ..errors import UsageError\n\nLOG = logging.getLogger()\n\n\ndef _pipe_to_proc(proc: 'subprocess.Popen[bytes]',\n fdesc: Union[IO[bytes], gzip.GzipFile]) -> int:\n assert proc.stdin is not None\n chunk = fdesc.read(2048)\n while chunk and proc.poll() is None:\n try:\n proc.stdin.write(chunk)\n except BrokenPipeError as exc:\n raise UsageError(\"Failed to execute SQL file.\") from exc\n chunk = fdesc.read(2048)\n\n return len(chunk)\n\n\ndef execute_file(dsn: str, fname: Path,\n ignore_errors: bool = False,\n pre_code: Optional[str] = None,\n post_code: Optional[str] = None) -> None:\n \"\"\" Read an SQL file and run its contents against the given database\n using psql. Use `pre_code` and `post_code` to run extra commands\n before or after executing the file. The commands are run within the\n same session, so they may be used to wrap the file execution in a\n transaction.\n \"\"\"\n cmd = ['psql']\n if not ignore_errors:\n cmd.extend(('-v', 'ON_ERROR_STOP=1'))\n if not LOG.isEnabledFor(logging.INFO):\n cmd.append('--quiet')\n\n with subprocess.Popen(cmd, env=get_pg_env(dsn), stdin=subprocess.PIPE) as proc:\n assert proc.stdin is not None\n try:\n if not LOG.isEnabledFor(logging.INFO):\n proc.stdin.write('set client_min_messages to WARNING;'.encode('utf-8'))\n\n if pre_code:\n proc.stdin.write((pre_code + ';').encode('utf-8'))\n\n if fname.suffix == '.gz':\n with gzip.open(str(fname), 'rb') as fdesc:\n remain = _pipe_to_proc(proc, fdesc)\n else:\n with fname.open('rb') as fdesc:\n remain = _pipe_to_proc(proc, fdesc)\n\n if remain == 0 and post_code:\n proc.stdin.write((';' + post_code).encode('utf-8'))\n finally:\n proc.stdin.close()\n ret = proc.wait()\n\n if ret != 0 or remain > 0:\n raise UsageError(\"Failed to execute SQL file.\")\n"
},
{
"path": "src/nominatim_db/errors.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nCustom exception and error classes for Nominatim.\n\"\"\"\n\n\nclass UsageError(Exception):\n \"\"\" An error raised because of bad user input. This error will usually\n not cause a stack trace to be printed unless debugging is enabled.\n \"\"\"\n"
},
{
"path": "src/nominatim_db/indexer/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/indexer/indexer.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nMain work horse for indexing (computing addresses) the database.\n\"\"\"\nfrom typing import cast, List, Any, Optional\nimport logging\nimport time\n\nimport psycopg\n\nfrom ..db.connection import connect, execute_scalar\nfrom ..db.query_pool import QueryPool\nfrom ..tokenizer.base import AbstractTokenizer\nfrom .progress import ProgressLogger\nfrom . import runners\n\nLOG = logging.getLogger()\n\n\nclass Indexer:\n \"\"\" Main indexing routine.\n \"\"\"\n\n def __init__(self, dsn: str, tokenizer: AbstractTokenizer, num_threads: int):\n self.dsn = dsn\n self.tokenizer = tokenizer\n self.num_threads = num_threads\n\n def has_pending(self, minrank: int = 0, maxrank: int = 30) -> bool:\n \"\"\" Check if any data still needs indexing.\n This function must only be used after the import has finished.\n Otherwise it will be very expensive.\n \"\"\"\n with connect(self.dsn) as conn:\n with conn.cursor() as cur:\n cur.execute(\"\"\" SELECT 'a'\n FROM placex\n WHERE rank_address BETWEEN %s AND %s\n AND indexed_status > 0\n LIMIT 1\"\"\",\n (minrank, maxrank))\n return cur.rowcount > 0\n\n async def index_full(self, analyse: bool = True) -> None:\n \"\"\" Index the complete database. This will first index boundaries\n followed by all other objects. When `analyse` is True, then the\n database will be analysed at the appropriate places to\n ensure that database statistics are updated.\n \"\"\"\n with connect(self.dsn) as conn:\n conn.autocommit = True\n\n def _analyze() -> None:\n if analyse:\n with conn.cursor() as cur:\n cur.execute('ANALYZE')\n\n while True:\n if await self.index_by_rank(1, 4) > 0:\n _analyze()\n\n if await self.index_boundaries() > 100:\n _analyze()\n\n if await self.index_by_rank(5, 25) > 100:\n _analyze()\n\n if await self.index_by_rank(26, 30) > 1000:\n _analyze()\n\n # Special case: rank zero depends on the previously-indexed [1..30] ranks\n await self.index_by_rank(0, 0)\n\n if await self.index_postcodes() > 100:\n _analyze()\n\n if not self.has_pending():\n break\n\n async def index_boundaries(self, minrank: int = 0, maxrank: int = 30) -> int:\n \"\"\" Index only administrative boundaries within the given rank range.\n \"\"\"\n total = 0\n LOG.warning(\"Starting indexing boundaries using %s threads\",\n self.num_threads)\n\n minrank = max(minrank, 4)\n maxrank = min(maxrank, 25)\n\n # Precompute number of rows to process for all rows\n with connect(self.dsn) as conn:\n hstore_info = psycopg.types.TypeInfo.fetch(conn, \"hstore\")\n if hstore_info is None:\n raise RuntimeError('Hstore extension is requested but not installed.')\n psycopg.types.hstore.register_hstore(hstore_info)\n\n with conn.cursor() as cur:\n cur = conn.execute(\"\"\" SELECT rank_search, count(*)\n FROM placex\n WHERE rank_search between %s and %s\n AND class = 'boundary' and type = 'administrative'\n AND indexed_status > 0\n GROUP BY rank_search\"\"\",\n (minrank, maxrank))\n total_tuples = {row.rank_search: row.count for row in cur}\n\n with self.tokenizer.name_analyzer() as analyzer:\n for rank in range(minrank, maxrank + 1):\n total += await self._index(runners.BoundaryRunner(rank, analyzer),\n total_tuples=total_tuples.get(rank, 0))\n\n return total\n\n async def index_by_rank(self, minrank: int, maxrank: int) -> int:\n \"\"\" Index all entries of placex in the given rank range (inclusive)\n in order of their address rank.\n\n When rank 30 is requested then also interpolations and\n places with address rank 0 will be indexed.\n \"\"\"\n total = 0\n maxrank = min(maxrank, 30)\n LOG.warning(\"Starting indexing rank (%i to %i) using %i threads\",\n minrank, maxrank, self.num_threads)\n\n # Precompute number of rows to process for all rows\n with connect(self.dsn) as conn:\n hstore_info = psycopg.types.TypeInfo.fetch(conn, \"hstore\")\n if hstore_info is None:\n raise RuntimeError('Hstore extension is requested but not installed.')\n psycopg.types.hstore.register_hstore(hstore_info)\n\n with conn.cursor() as cur:\n cur = conn.execute(\"\"\" SELECT rank_address, count(*)\n FROM placex\n WHERE rank_address between %s and %s\n AND indexed_status > 0\n GROUP BY rank_address\"\"\",\n (minrank, maxrank))\n total_tuples = {row.rank_address: row.count for row in cur}\n\n with self.tokenizer.name_analyzer() as analyzer:\n for rank in range(max(1, minrank), maxrank + 1):\n if rank >= 30:\n batch = 20\n elif rank >= 26:\n batch = 5\n else:\n batch = 1\n total += await self._index(runners.RankRunner(rank, analyzer),\n batch=batch, total_tuples=total_tuples.get(rank, 0))\n\n # Special case: rank zero depends on ranks [1..30]\n if minrank == 0:\n total += await self._index(runners.RankRunner(0, analyzer))\n\n if maxrank == 30:\n total += await self._index(runners.InterpolationRunner(analyzer), batch=20)\n\n return total\n\n async def index_postcodes(self) -> int:\n \"\"\"Index the entries of the location_postcodes table.\n \"\"\"\n LOG.warning(\"Starting indexing postcodes using %s threads\", self.num_threads)\n\n return await self._index(runners.PostcodeRunner(), batch=20)\n\n def update_status_table(self) -> None:\n \"\"\" Update the status in the status table to 'indexed'.\n \"\"\"\n with connect(self.dsn) as conn:\n with conn.cursor() as cur:\n cur.execute('UPDATE import_status SET indexed = true')\n\n conn.commit()\n\n async def _index(self, runner: runners.Runner, batch: int = 1,\n total_tuples: Optional[int] = None) -> int:\n \"\"\" Index a single rank or table. `runner` describes the SQL to use\n for indexing. `batch` describes the number of objects that\n should be processed with a single SQL statement.\n\n `total_tuples` may contain the total number of rows to process.\n When not supplied, the value will be computed using the\n appropriate runner function.\n \"\"\"\n LOG.warning(\"Starting %s (using batch size %s)\", runner.name(), batch)\n\n if total_tuples is None:\n total_tuples = self._prepare_indexing(runner)\n\n progress = ProgressLogger(runner.name(), total_tuples)\n\n if total_tuples > 0:\n async with await psycopg.AsyncConnection.connect(\n self.dsn, row_factory=psycopg.rows.dict_row) as aconn, \\\n QueryPool(self.dsn, self.num_threads, autocommit=True) as pool:\n fetcher_time = 0.0\n tstart = time.time()\n async with aconn.cursor(name='places') as cur:\n query = runner.index_places_query(batch)\n params: List[Any] = []\n num_places = 0\n async for place in cur.stream(runner.sql_get_objects()):\n fetcher_time += time.time() - tstart\n\n params.extend(runner.index_places_params(place))\n num_places += 1\n\n if num_places >= batch:\n LOG.debug(\"Processing places: %s\", str(params))\n await pool.put_query(query, params)\n progress.add(num_places)\n params = []\n num_places = 0\n\n tstart = time.time()\n\n if num_places > 0:\n await pool.put_query(runner.index_places_query(num_places), params)\n\n LOG.info(\"Wait time: fetcher: %.2fs, pool: %.2fs\",\n fetcher_time, pool.wait_time)\n\n return progress.done()\n\n def _prepare_indexing(self, runner: runners.Runner) -> int:\n with connect(self.dsn) as conn:\n hstore_info = psycopg.types.TypeInfo.fetch(conn, \"hstore\")\n if hstore_info is None:\n raise RuntimeError('Hstore extension is requested but not installed.')\n psycopg.types.hstore.register_hstore(hstore_info)\n\n total_tuples = execute_scalar(conn, runner.sql_count_objects())\n LOG.debug(\"Total number of rows: %i\", total_tuples)\n return cast(int, total_tuples)\n"
},
{
"path": "src/nominatim_db/indexer/progress.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelpers for progress logging.\n\"\"\"\nimport logging\nfrom datetime import datetime\n\nLOG = logging.getLogger()\n\nINITIAL_PROGRESS = 10\n\n\nclass ProgressLogger:\n \"\"\" Tracks and prints progress for the indexing process.\n `name` is the name of the indexing step being tracked.\n `total` sets up the total number of items that need processing.\n `log_interval` denotes the interval in seconds at which progress\n should be reported.\n \"\"\"\n\n def __init__(self, name: str, total: int, log_interval: int = 1) -> None:\n self.name = name\n self.total_places = total\n self.done_places = 0\n self.rank_start_time = datetime.now()\n self.log_interval = log_interval\n self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1\n\n def add(self, num: int = 1) -> None:\n \"\"\" Mark `num` places as processed. Print a log message if the\n logging is at least info and the log interval has passed.\n \"\"\"\n self.done_places += num\n\n if self.done_places < self.next_info:\n return\n\n now = datetime.now()\n done_time = (now - self.rank_start_time).total_seconds()\n\n if done_time < 2:\n self.next_info = self.done_places + INITIAL_PROGRESS\n return\n\n places_per_sec = self.done_places / done_time\n eta = (self.total_places - self.done_places) / places_per_sec\n\n LOG.warning(\"Done %d in %.0f @ %.3f per second - %s ETA (seconds): %.2f\",\n self.done_places, done_time,\n places_per_sec, self.name, eta)\n\n self.next_info += int(places_per_sec) * self.log_interval\n\n def done(self) -> int:\n \"\"\" Print final statistics about the progress.\n \"\"\"\n rank_end_time = datetime.now()\n\n if rank_end_time == self.rank_start_time:\n diff_seconds = 0.0\n places_per_sec = float(self.done_places)\n else:\n diff_seconds = (rank_end_time - self.rank_start_time).total_seconds()\n places_per_sec = self.done_places / diff_seconds\n\n LOG.warning(\"Done %d/%d in %.0f @ %.3f per second - FINISHED %s\\n\",\n self.done_places, self.total_places, diff_seconds,\n places_per_sec, self.name)\n\n return self.done_places\n"
},
{
"path": "src/nominatim_db/indexer/runners.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nMix-ins that provide the actual commands for the indexer for various indexing\ntasks.\n\"\"\"\nfrom typing import Any, Sequence\n\nfrom psycopg import sql as pysql\nfrom psycopg.abc import Query\nfrom psycopg.rows import DictRow\nfrom psycopg.types.json import Json\n\nfrom ..typing import Protocol\nfrom ..data.place_info import PlaceInfo\nfrom ..tokenizer.base import AbstractAnalyzer\n\n\ndef _mk_valuelist(template: str, num: int) -> pysql.Composed:\n return pysql.SQL(',').join([pysql.SQL(template)] * num)\n\n\ndef _analyze_place(place: DictRow, analyzer: AbstractAnalyzer) -> Json:\n return Json(analyzer.process_place(PlaceInfo(place)))\n\n\nclass Runner(Protocol):\n def name(self) -> str: ...\n def sql_count_objects(self) -> Query: ...\n def sql_get_objects(self) -> Query: ...\n def index_places_query(self, batch_size: int) -> Query: ...\n def index_places_params(self, place: DictRow) -> Sequence[Any]: ...\n\n\nSELECT_SQL = pysql.SQL(\"\"\"SELECT place_id, extra.*\n FROM (SELECT * FROM placex {}) as px,\n LATERAL placex_indexing_prepare(px) as extra \"\"\")\nUPDATE_LINE = \"(%s, %s::hstore, %s::hstore, %s::int, %s::jsonb)\"\n\n\nclass AbstractPlacexRunner:\n \"\"\" Returns SQL commands for indexing of the placex table.\n \"\"\"\n\n def __init__(self, rank: int, analyzer: AbstractAnalyzer) -> None:\n self.rank = rank\n self.analyzer = analyzer\n\n def index_places_query(self, batch_size: int) -> Query:\n return pysql.SQL(\n \"\"\" UPDATE placex\n SET indexed_status = 0, address = v.addr, token_info = v.ti,\n name = v.name, linked_place_id = v.linked_place_id\n FROM (VALUES {}) as v(id, name, addr, linked_place_id, ti)\n WHERE place_id = v.id\n \"\"\").format(_mk_valuelist(UPDATE_LINE, batch_size))\n\n def index_places_params(self, place: DictRow) -> Sequence[Any]:\n return (place['place_id'],\n place['name'],\n place['address'],\n place['linked_place_id'],\n _analyze_place(place, self.analyzer))\n\n\nclass RankRunner(AbstractPlacexRunner):\n \"\"\" Returns SQL commands for indexing one rank within the placex table.\n \"\"\"\n\n def name(self) -> str:\n return f\"rank {self.rank}\"\n\n def sql_count_objects(self) -> pysql.Composed:\n return pysql.SQL(\"\"\"SELECT count(*) FROM placex\n WHERE rank_address = {} and indexed_status > 0\n \"\"\").format(pysql.Literal(self.rank))\n\n def sql_get_objects(self) -> pysql.Composed:\n return SELECT_SQL.format(pysql.SQL(\n \"\"\"WHERE placex.indexed_status > 0 and placex.rank_address = {}\n ORDER BY placex.geometry_sector\n \"\"\").format(pysql.Literal(self.rank)))\n\n\nclass BoundaryRunner(AbstractPlacexRunner):\n \"\"\" Returns SQL commands for indexing the administrative boundaries\n of a certain rank.\n \"\"\"\n\n def name(self) -> str:\n return f\"boundaries rank {self.rank}\"\n\n def sql_count_objects(self) -> Query:\n return pysql.SQL(\"\"\"SELECT count(*) FROM placex\n WHERE indexed_status > 0\n AND rank_search = {}\n AND class = 'boundary' and type = 'administrative'\n \"\"\").format(pysql.Literal(self.rank))\n\n def sql_get_objects(self) -> Query:\n return SELECT_SQL.format(pysql.SQL(\n \"\"\"WHERE placex.indexed_status > 0 and placex.rank_search = {}\n and placex.class = 'boundary' and placex.type = 'administrative'\n ORDER BY placex.partition, placex.admin_level\n \"\"\").format(pysql.Literal(self.rank)))\n\n\nclass InterpolationRunner:\n \"\"\" Returns SQL commands for indexing the address interpolation table\n location_property_osmline.\n \"\"\"\n\n def __init__(self, analyzer: AbstractAnalyzer) -> None:\n self.analyzer = analyzer\n\n def name(self) -> str:\n return \"interpolation lines (location_property_osmline)\"\n\n def sql_count_objects(self) -> Query:\n return \"\"\"SELECT count(*) FROM location_property_osmline\n WHERE indexed_status > 0\"\"\"\n\n def sql_get_objects(self) -> Query:\n return \"\"\"SELECT place_id, get_interpolation_address(address, osm_id) as address\n FROM location_property_osmline\n WHERE indexed_status > 0\n ORDER BY geometry_sector\"\"\"\n\n def index_places_query(self, batch_size: int) -> Query:\n return pysql.SQL(\"\"\"UPDATE location_property_osmline\n SET indexed_status = 0, address = v.addr, token_info = v.ti\n FROM (VALUES {}) as v(id, addr, ti)\n WHERE place_id = v.id\n \"\"\").format(_mk_valuelist(\"(%s, %s::hstore, %s::jsonb)\", batch_size))\n\n def index_places_params(self, place: DictRow) -> Sequence[Any]:\n return (place['place_id'], place['address'],\n _analyze_place(place, self.analyzer))\n\n\nclass PostcodeRunner(Runner):\n \"\"\" Provides the SQL commands for indexing the location_postcodes table.\n \"\"\"\n\n def name(self) -> str:\n return \"postcodes (location_postcodes)\"\n\n def sql_count_objects(self) -> Query:\n return 'SELECT count(*) FROM location_postcodes WHERE indexed_status > 0'\n\n def sql_get_objects(self) -> Query:\n return \"\"\"SELECT place_id FROM location_postcodes\n WHERE indexed_status > 0\n ORDER BY country_code, postcode\"\"\"\n\n def index_places_query(self, batch_size: int) -> Query:\n return pysql.SQL(\"\"\"UPDATE location_postcodes SET indexed_status = 0\n WHERE place_id IN ({})\"\"\")\\\n .format(pysql.SQL(',').join((pysql.Placeholder() for _ in range(batch_size))))\n\n def index_places_params(self, place: DictRow) -> Sequence[Any]:\n return (place['place_id'], )\n"
},
{
"path": "src/nominatim_db/paths.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nPath settings for extra data used by Nominatim.\n\"\"\"\nfrom pathlib import Path\n\nSQLLIB_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-sql').resolve()\nLUALIB_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-lua').resolve()\nDATA_DIR = (Path(__file__) / '..' / '..' / '..' / 'data').resolve()\nCONFIG_DIR = (Path(__file__) / '..' / '..' / '..' / 'settings').resolve()\n"
},
{
"path": "src/nominatim_db/tokenizer/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/tokenizer/base.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nAbstract class definitions for tokenizers. These base classes are here\nmainly for documentation purposes.\n\"\"\"\nfrom abc import ABC, abstractmethod\nfrom typing import List, Tuple, Dict, Any, Optional, Iterable\n\nfrom ..typing import Protocol\nfrom ..config import Configuration\nfrom ..db.connection import Connection\nfrom ..data.place_info import PlaceInfo\n\n\nclass AbstractAnalyzer(ABC):\n \"\"\" The analyzer provides the functions for analysing names and building\n the token database.\n\n Analyzers are instantiated on a per-thread base. Access to global data\n structures must be synchronised accordingly.\n \"\"\"\n\n def __enter__(self) -> 'AbstractAnalyzer':\n return self\n\n def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:\n self.close()\n\n @abstractmethod\n def close(self) -> None:\n \"\"\" Free all resources used by the analyzer.\n \"\"\"\n\n @abstractmethod\n def get_word_token_info(self, words: List[str]) -> List[Tuple[str, str, Optional[int]]]:\n \"\"\" Return token information for the given list of words.\n\n The function is used for testing and debugging only\n and does not need to be particularly efficient.\n\n Arguments:\n words: A list of words to look up the tokens for.\n If a word starts with # it is assumed to be a full name\n otherwise is a partial term.\n\n Returns:\n The function returns the list of all tuples that could be\n found for the given words. Each list entry is a tuple of\n (original word, word token, word id).\n \"\"\"\n\n @abstractmethod\n def normalize_postcode(self, postcode: str) -> str:\n \"\"\" Convert the postcode to its standardized form.\n\n This function must yield exactly the same result as the SQL function\n `token_normalized_postcode()`.\n\n Arguments:\n postcode: The postcode to be normalized.\n\n Returns:\n The given postcode after normalization.\n \"\"\"\n\n @abstractmethod\n def update_postcodes_from_db(self) -> None:\n \"\"\" Update the tokenizer's postcode tokens from the current content\n of the `location_postcodes` table.\n \"\"\"\n\n @abstractmethod\n def update_special_phrases(self,\n phrases: Iterable[Tuple[str, str, str, str]],\n should_replace: bool) -> None:\n \"\"\" Update the tokenizer's special phrase tokens from the given\n list of special phrases.\n\n Arguments:\n phrases: The new list of special phrases. Each entry is\n a tuple of (phrase, class, type, operator).\n should_replace: If true, replace the current list of phrases.\n When false, just add the given phrases to the\n ones that already exist.\n \"\"\"\n\n @abstractmethod\n def add_country_names(self, country_code: str, names: Dict[str, str]) -> None:\n \"\"\" Add the given names to the tokenizer's list of country tokens.\n\n Arguments:\n country_code: two-letter country code for the country the names\n refer to.\n names: Dictionary of name type to name.\n \"\"\"\n\n @abstractmethod\n def process_place(self, place: PlaceInfo) -> Any:\n \"\"\" Extract tokens for the given place and compute the\n information to be handed to the PL/pgSQL processor for building\n the search index.\n\n Arguments:\n place: Place information retrieved from the database.\n\n Returns:\n A JSON-serialisable structure that will be handed into\n the database via the `token_info` field.\n \"\"\"\n\n\nclass AbstractTokenizer(ABC):\n \"\"\" The tokenizer instance is the central instance of the tokenizer in\n the system. There will only be a single instance of the tokenizer\n active at any time.\n \"\"\"\n\n @abstractmethod\n def init_new_db(self, config: Configuration, init_db: bool = True) -> None:\n \"\"\" Set up a new tokenizer for the database.\n\n The function should copy all necessary data into the project\n directory or save it in the property table to make sure that\n the tokenizer remains stable over updates.\n\n Arguments:\n config: Read-only object with configuration options.\n\n init_db: When set to False, then initialisation of database\n tables should be skipped. This option is only required for\n migration purposes and can be safely ignored by custom\n tokenizers.\n \"\"\"\n\n @abstractmethod\n def init_from_project(self, config: Configuration) -> None:\n \"\"\" Initialise the tokenizer from an existing database setup.\n\n The function should load all previously saved configuration from\n the project directory and/or the property table.\n\n Arguments:\n config: Read-only object with configuration options.\n \"\"\"\n\n @abstractmethod\n def finalize_import(self, config: Configuration) -> None:\n \"\"\" This function is called at the very end of an import when all\n data has been imported and indexed. The tokenizer may create\n at this point any additional indexes and data structures needed\n during query time.\n\n Arguments:\n config: Read-only object with configuration options.\n \"\"\"\n\n @abstractmethod\n def update_sql_functions(self, config: Configuration) -> None:\n \"\"\" Update the SQL part of the tokenizer. This function is called\n automatically on migrations or may be called explicitly by the\n user through the `nominatim refresh --functions` command.\n\n The tokenizer must only update the code of the tokenizer. The\n data structures or data itself must not be changed by this function.\n\n Arguments:\n config: Read-only object with configuration options.\n \"\"\"\n\n @abstractmethod\n def check_database(self, config: Configuration) -> Optional[str]:\n \"\"\" Check that the database is set up correctly and ready for being\n queried.\n\n Arguments:\n config: Read-only object with configuration options.\n\n Returns:\n If an issue was found, return an error message with the\n description of the issue as well as hints for the user on\n how to resolve the issue. If everything is okay, return `None`.\n \"\"\"\n\n @abstractmethod\n def update_statistics(self, config: Configuration, threads: int = 1) -> None:\n \"\"\" Recompute any tokenizer statistics necessary for efficient lookup.\n This function is meant to be called from time to time by the user\n to improve performance. However, the tokenizer must not depend on\n it to be called in order to work.\n \"\"\"\n\n @abstractmethod\n def update_word_tokens(self) -> None:\n \"\"\" Do house-keeping on the tokenizers internal data structures.\n Remove unused word tokens, resort data etc.\n \"\"\"\n\n @abstractmethod\n def name_analyzer(self) -> AbstractAnalyzer:\n \"\"\" Create a new analyzer for tokenizing names and queries\n using this tokinzer. Analyzers are context managers and should\n be used accordingly:\n\n ```\n with tokenizer.name_analyzer() as analyzer:\n analyser.tokenize()\n ```\n\n When used outside the with construct, the caller must ensure to\n call the close() function before destructing the analyzer.\n \"\"\"\n\n @abstractmethod\n def most_frequent_words(self, conn: Connection, num: int) -> List[str]:\n \"\"\" Return a list of the most frequent full words in the database.\n\n Arguments:\n conn: Open connection to the database which may be used to\n retrieve the words.\n num: Maximum number of words to return.\n \"\"\"\n\n\nclass TokenizerModule(Protocol):\n \"\"\" Interface that must be exported by modules that implement their\n own tokenizer.\n \"\"\"\n\n def create(self, dsn: str) -> AbstractTokenizer:\n \"\"\" Factory for new tokenizers.\n \"\"\"\n"
},
{
"path": "src/nominatim_db/tokenizer/factory.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for creating a tokenizer or initialising the right one for an\nexisting database.\n\nA tokenizer is something that is bound to the lifetime of a database. It\ncan be chosen and configured before the initial import but then needs to\nbe used consistently when querying and updating the database.\n\nThis module provides the functions to create and configure a new tokenizer\nas well as instantiating the appropriate tokenizer for updating an existing\ndatabase.\n\"\"\"\nfrom typing import Optional\nimport logging\nimport importlib\nfrom pathlib import Path\n\nfrom ..errors import UsageError\nfrom ..db import properties\nfrom ..db.connection import connect\nfrom ..config import Configuration\nfrom ..tokenizer.base import AbstractTokenizer, TokenizerModule\n\nLOG = logging.getLogger()\n\n\ndef _import_tokenizer(name: str) -> TokenizerModule:\n \"\"\" Load the tokenizer.py module from project directory.\n \"\"\"\n src_file = Path(__file__).parent / (name + '_tokenizer.py')\n if not src_file.is_file():\n LOG.fatal(\"No tokenizer named '%s' available. \"\n \"Check the setting of NOMINATIM_TOKENIZER.\", name)\n raise UsageError('Tokenizer not found')\n\n return importlib.import_module('nominatim_db.tokenizer.' + name + '_tokenizer')\n\n\ndef create_tokenizer(config: Configuration, init_db: bool = True,\n module_name: Optional[str] = None) -> AbstractTokenizer:\n \"\"\" Create a new tokenizer as defined by the given configuration.\n\n The tokenizer data and code is copied into the 'tokenizer' directory\n of the project directory and the tokenizer loaded from its new location.\n \"\"\"\n if module_name is None:\n module_name = config.TOKENIZER\n\n # Import and initialize the tokenizer.\n tokenizer_module = _import_tokenizer(module_name)\n\n tokenizer = tokenizer_module.create(config.get_libpq_dsn())\n tokenizer.init_new_db(config, init_db=init_db)\n\n with connect(config.get_libpq_dsn()) as conn:\n properties.set_property(conn, 'tokenizer', module_name)\n\n return tokenizer\n\n\ndef get_tokenizer_for_db(config: Configuration) -> AbstractTokenizer:\n \"\"\" Instantiate a tokenizer for an existing database.\n\n The function looks up the appropriate tokenizer in the database\n and initialises it.\n \"\"\"\n with connect(config.get_libpq_dsn()) as conn:\n name = properties.get_property(conn, 'tokenizer')\n\n if name is None:\n LOG.fatal(\"Tokenizer was not set up properly. Database property missing.\")\n raise UsageError('Cannot initialize tokenizer.')\n\n tokenizer_module = _import_tokenizer(name)\n\n tokenizer = tokenizer_module.create(config.get_libpq_dsn())\n tokenizer.init_from_project(config)\n\n return tokenizer\n"
},
{
"path": "src/nominatim_db/tokenizer/icu_rule_loader.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelper class to create ICU rules from a configuration file.\n\"\"\"\nfrom typing import Mapping, Any, Dict, Optional\nimport io\nimport json\nimport logging\n\nfrom icu import Transliterator\n\nfrom ..config import flatten_config_list, Configuration\nfrom ..db.properties import set_property, get_property\nfrom ..db.connection import Connection\nfrom ..errors import UsageError\nfrom .place_sanitizer import PlaceSanitizer\nfrom .icu_token_analysis import ICUTokenAnalysis\nfrom .token_analysis.base import AnalysisModule, Analyzer\nfrom ..data import country_info\n\nLOG = logging.getLogger()\n\nDBCFG_IMPORT_NORM_RULES = \"tokenizer_import_normalisation\"\nDBCFG_IMPORT_TRANS_RULES = \"tokenizer_import_transliteration\"\nDBCFG_IMPORT_ANALYSIS_RULES = \"tokenizer_import_analysis_rules\"\n\n\ndef _get_section(rules: Mapping[str, Any], section: str) -> Any:\n \"\"\" Get the section named 'section' from the rules. If the section does\n not exist, raise a usage error with a meaningful message.\n \"\"\"\n if section not in rules:\n LOG.fatal(\"Section '%s' not found in tokenizer config.\", section)\n raise UsageError(\"Syntax error in tokenizer configuration file.\")\n\n return rules[section]\n\n\nclass ICURuleLoader:\n \"\"\" Compiler for ICU rules from a tokenizer configuration file.\n \"\"\"\n\n def __init__(self, config: Configuration) -> None:\n self.config = config\n rules = config.load_sub_configuration('icu_tokenizer.yaml',\n config='TOKENIZER_CONFIG')\n\n # Make sure country information is available to analyzers and sanitizers.\n country_info.setup_country_config(config)\n\n self.normalization_rules = self._cfg_to_icu_rules(rules, 'normalization')\n self.transliteration_rules = self._cfg_to_icu_rules(rules, 'transliteration')\n self.analysis_rules = _get_section(rules, 'token-analysis')\n self._setup_analysis()\n\n # Load optional sanitizer rule set.\n self.sanitizer_rules = rules.get('sanitizers', [])\n\n def load_config_from_db(self, conn: Connection) -> None:\n \"\"\" Get previously saved parts of the configuration from the\n database.\n \"\"\"\n rules = get_property(conn, DBCFG_IMPORT_NORM_RULES)\n if rules is not None:\n self.normalization_rules = rules\n\n rules = get_property(conn, DBCFG_IMPORT_TRANS_RULES)\n if rules is not None:\n self.transliteration_rules = rules\n\n rules = get_property(conn, DBCFG_IMPORT_ANALYSIS_RULES)\n if rules:\n self.analysis_rules = json.loads(rules)\n else:\n self.analysis_rules = []\n self._setup_analysis()\n\n def save_config_to_db(self, conn: Connection) -> None:\n \"\"\" Save the part of the configuration that cannot be changed into\n the database.\n \"\"\"\n set_property(conn, DBCFG_IMPORT_NORM_RULES, self.normalization_rules)\n set_property(conn, DBCFG_IMPORT_TRANS_RULES, self.transliteration_rules)\n set_property(conn, DBCFG_IMPORT_ANALYSIS_RULES, json.dumps(self.analysis_rules))\n\n def make_sanitizer(self) -> PlaceSanitizer:\n \"\"\" Create a place sanitizer from the configured rules.\n \"\"\"\n return PlaceSanitizer(self.sanitizer_rules, self.config)\n\n def make_token_analysis(self) -> ICUTokenAnalysis:\n \"\"\" Create a token analyser from the reviouly loaded rules.\n \"\"\"\n return ICUTokenAnalysis(self.normalization_rules,\n self.transliteration_rules, self.analysis)\n\n def get_search_rules(self) -> str:\n \"\"\" Return the ICU rules to be used during search.\n The rules combine normalization and transliteration.\n \"\"\"\n # First apply the normalization rules.\n rules = io.StringIO()\n rules.write(self.normalization_rules)\n\n # Then add transliteration.\n rules.write(self.transliteration_rules)\n return rules.getvalue()\n\n def get_normalization_rules(self) -> str:\n \"\"\" Return rules for normalisation of a term.\n \"\"\"\n return self.normalization_rules\n\n def get_transliteration_rules(self) -> str:\n \"\"\" Return the rules for converting a string into its asciii representation.\n \"\"\"\n return self.transliteration_rules\n\n def _setup_analysis(self) -> None:\n \"\"\" Process the rules used for creating the various token analyzers.\n \"\"\"\n self.analysis: Dict[Optional[str], TokenAnalyzerRule] = {}\n\n if not isinstance(self.analysis_rules, list):\n raise UsageError(\"Configuration section 'token-analysis' must be a list.\")\n\n norm = Transliterator.createFromRules(\"rule_loader_normalization\",\n self.normalization_rules)\n trans = Transliterator.createFromRules(\"rule_loader_transliteration\",\n self.transliteration_rules)\n\n for section in self.analysis_rules:\n name = section.get('id', None)\n if name in self.analysis:\n if name is None:\n LOG.fatal(\"ICU tokenizer configuration has two default token analyzers.\")\n else:\n LOG.fatal(\"ICU tokenizer configuration has two token \"\n \"analyzers with id '%s'.\", name)\n raise UsageError(\"Syntax error in ICU tokenizer config.\")\n self.analysis[name] = TokenAnalyzerRule(section, norm, trans,\n self.config)\n\n @staticmethod\n def _cfg_to_icu_rules(rules: Mapping[str, Any], section: str) -> str:\n \"\"\" Load an ICU ruleset from the given section. If the section is a\n simple string, it is interpreted as a file name and the rules are\n loaded verbatim from the given file. The filename is expected to be\n relative to the tokenizer rule file. If the section is a list then\n each line is assumed to be a rule. All rules are concatenated and returned.\n \"\"\"\n content = _get_section(rules, section)\n\n if content is None:\n return ''\n\n return ';'.join(flatten_config_list(content, section)) + ';'\n\n\nclass TokenAnalyzerRule:\n \"\"\" Factory for a single analysis module. The class saves the configuration\n and creates a new token analyzer on request.\n \"\"\"\n\n def __init__(self, rules: Mapping[str, Any],\n normalizer: Any, transliterator: Any,\n config: Configuration) -> None:\n analyzer_name = _get_section(rules, 'analyzer')\n if not analyzer_name or not isinstance(analyzer_name, str):\n raise UsageError(\"'analyzer' parameter needs to be simple string\")\n\n self._analysis_mod: AnalysisModule = \\\n config.load_plugin_module(analyzer_name, 'nominatim_db.tokenizer.token_analysis')\n\n self.config = self._analysis_mod.configure(rules, normalizer,\n transliterator)\n\n def create(self, normalizer: Any, transliterator: Any) -> Analyzer:\n \"\"\" Create a new analyser instance for the given rule.\n \"\"\"\n return self._analysis_mod.create(normalizer, transliterator, self.config)\n"
},
{
"path": "src/nominatim_db/tokenizer/icu_token_analysis.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nContainer class collecting all components required to transform an OSM name\ninto a Nominatim token.\n\"\"\"\nfrom typing import Mapping, Optional, TYPE_CHECKING\nfrom icu import Transliterator\n\nfrom .token_analysis.base import Analyzer\n\nif TYPE_CHECKING:\n from typing import Any # noqa\n from .icu_rule_loader import TokenAnalyzerRule\n\n\nclass ICUTokenAnalysis:\n \"\"\" Container class collecting the transliterators and token analysis\n modules for a single Analyser instance.\n \"\"\"\n\n def __init__(self, norm_rules: str, trans_rules: str,\n analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']):\n # additional break signs are not relevant during name analysis\n norm_rules += \";[[:Space:][-:]]+ > ' ';\"\n self.normalizer = Transliterator.createFromRules(\"icu_normalization\",\n norm_rules)\n trans_rules += \";[:Space:]+ > ' '\"\n self.to_ascii = Transliterator.createFromRules(\"icu_to_ascii\",\n trans_rules)\n self.search = Transliterator.createFromRules(\"icu_search\",\n norm_rules + trans_rules)\n\n self.analysis = {name: arules.create(self.normalizer, self.to_ascii)\n for name, arules in analysis_rules.items()}\n\n def get_analyzer(self, name: Optional[str]) -> Analyzer:\n \"\"\" Return the given named analyzer. If no analyzer with that\n name exists, return the default analyzer.\n \"\"\"\n return self.analysis.get(name) or self.analysis[None]\n"
},
{
"path": "src/nominatim_db/tokenizer/icu_tokenizer.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nTokenizer implementing normalisation as used before Nominatim 4 but using\nlibICU instead of the PostgreSQL module.\n\"\"\"\nfrom typing import Optional, Sequence, List, Tuple, Mapping, Any, cast, \\\n Dict, Set, Iterable\nimport itertools\nimport logging\n\nfrom psycopg.types.json import Jsonb\nfrom psycopg import sql as pysql\n\nfrom ..db.connection import connect, Connection, Cursor, \\\n drop_tables, table_exists, execute_scalar\nfrom ..config import Configuration\nfrom ..db.sql_preprocessor import SQLPreprocessor\nfrom ..data.place_info import PlaceInfo\nfrom ..data.place_name import PlaceName\nfrom .icu_rule_loader import ICURuleLoader\nfrom .place_sanitizer import PlaceSanitizer\nfrom .icu_token_analysis import ICUTokenAnalysis\nfrom .base import AbstractAnalyzer, AbstractTokenizer\n\nDBCFG_TERM_NORMALIZATION = \"tokenizer_term_normalization\"\n\nLOG = logging.getLogger()\n\nWORD_TYPES = (('country_names', 'C'),\n ('postcodes', 'P'),\n ('full_word', 'W'),\n ('housenumbers', 'H'))\n\n\ndef create(dsn: str) -> 'ICUTokenizer':\n \"\"\" Create a new instance of the tokenizer provided by this module.\n \"\"\"\n return ICUTokenizer(dsn)\n\n\nclass ICUTokenizer(AbstractTokenizer):\n \"\"\" This tokenizer uses libICU to convert names and queries to ASCII.\n Otherwise it uses the same algorithms and data structures as the\n normalization routines in Nominatim 3.\n \"\"\"\n\n def __init__(self, dsn: str) -> None:\n self.dsn = dsn\n self.loader: Optional[ICURuleLoader] = None\n\n def init_new_db(self, config: Configuration, init_db: bool = True) -> None:\n \"\"\" Set up a new tokenizer for the database.\n\n This copies all necessary data in the project directory to make\n sure the tokenizer remains stable even over updates.\n \"\"\"\n self.loader = ICURuleLoader(config)\n\n self._save_config()\n\n if init_db:\n self.update_sql_functions(config)\n self._setup_db_tables(config)\n self._create_base_indices(config, 'word')\n\n def init_from_project(self, config: Configuration) -> None:\n \"\"\" Initialise the tokenizer from the project directory.\n \"\"\"\n self.loader = ICURuleLoader(config)\n\n with connect(self.dsn) as conn:\n self.loader.load_config_from_db(conn)\n\n def finalize_import(self, config: Configuration) -> None:\n \"\"\" Do any required postprocessing to make the tokenizer data ready\n for use.\n \"\"\"\n self._create_lookup_indices(config, 'word')\n\n def update_sql_functions(self, config: Configuration) -> None:\n \"\"\" Reimport the SQL functions for this tokenizer.\n \"\"\"\n with connect(self.dsn) as conn:\n sqlp = SQLPreprocessor(conn, config)\n sqlp.run_sql_file(conn, 'tokenizer/icu_tokenizer.sql')\n\n def check_database(self, config: Configuration) -> None:\n \"\"\" Check that the tokenizer is set up correctly.\n \"\"\"\n # Will throw an error if there is an issue.\n self.init_from_project(config)\n\n def update_statistics(self, config: Configuration, threads: int = 2) -> None:\n \"\"\" Recompute frequencies for all name words.\n \"\"\"\n with connect(self.dsn) as conn:\n if not table_exists(conn, 'search_name'):\n return\n\n with conn.cursor() as cur:\n cur.execute('ANALYSE search_name')\n if threads > 1:\n cur.execute(pysql.SQL('SET max_parallel_workers_per_gather TO {}')\n .format(pysql.Literal(min(threads, 6),)))\n\n LOG.info('Computing word frequencies')\n drop_tables(conn, 'word_frequencies')\n cur.execute(\"\"\"\n CREATE TEMP TABLE word_frequencies AS\n WITH word_freq AS MATERIALIZED (\n SELECT unnest(name_vector) as id, count(*)\n FROM search_name GROUP BY id),\n addr_freq AS MATERIALIZED (\n SELECT unnest(nameaddress_vector) as id, count(*)\n FROM search_name GROUP BY id)\n SELECT coalesce(a.id, w.id) as id,\n (CASE WHEN w.count is null or w.count <= 1 THEN '{}'::JSONB\n ELSE jsonb_build_object('count', w.count) END\n ||\n CASE WHEN a.count is null or a.count <= 1 THEN '{}'::JSONB\n ELSE jsonb_build_object('addr_count', a.count) END) as info\n FROM word_freq w FULL JOIN addr_freq a ON a.id = w.id;\n \"\"\")\n cur.execute('CREATE UNIQUE INDEX ON word_frequencies(id) INCLUDE(info)')\n cur.execute('ANALYSE word_frequencies')\n LOG.info('Update word table with recomputed frequencies')\n drop_tables(conn, 'tmp_word')\n cur.execute(\"\"\"CREATE TABLE tmp_word AS\n SELECT word_id, word_token, type, word,\n coalesce(word.info, '{}'::jsonb)\n - 'count' - 'addr_count' ||\n coalesce(wf.info, '{}'::jsonb)\n as info\n FROM word LEFT JOIN word_frequencies wf\n ON word.word_id = wf.id\n \"\"\")\n drop_tables(conn, 'word_frequencies')\n\n with conn.cursor() as cur:\n cur.execute('SET max_parallel_workers_per_gather TO 0')\n\n sqlp = SQLPreprocessor(conn, config)\n sqlp.run_string(conn,\n 'GRANT SELECT ON tmp_word TO \"{{config.DATABASE_WEBUSER}}\"')\n conn.commit()\n self._create_base_indices(config, 'tmp_word')\n self._create_lookup_indices(config, 'tmp_word')\n self._move_temporary_word_table('tmp_word')\n\n def _cleanup_housenumbers(self) -> None:\n \"\"\" Remove unused house numbers.\n \"\"\"\n with connect(self.dsn) as conn:\n if not table_exists(conn, 'search_name'):\n return\n with conn.cursor(name=\"hnr_counter\") as cur:\n cur.execute(\"\"\"SELECT DISTINCT word_id, coalesce(info->>'lookup', word_token)\n FROM word\n WHERE type = 'H'\n AND NOT EXISTS(SELECT * FROM search_name\n WHERE ARRAY[word.word_id] && name_vector)\n AND (char_length(coalesce(word, word_token)) > 6\n OR coalesce(word, word_token) not similar to '\\\\d+')\n \"\"\")\n candidates = {token: wid for wid, token in cur}\n with conn.cursor(name=\"hnr_counter\") as cur:\n cur.execute(\"\"\"SELECT housenumber FROM placex\n WHERE housenumber is not null\n AND (char_length(housenumber) > 6\n OR housenumber not similar to '\\\\d+')\n \"\"\")\n for row in cur:\n for hnr in row[0].split(';'):\n candidates.pop(hnr, None)\n LOG.info(\"There are %s outdated housenumbers.\", len(candidates))\n LOG.debug(\"Outdated housenumbers: %s\", candidates.keys())\n if candidates:\n with conn.cursor() as cur:\n cur.execute(\"\"\"DELETE FROM word WHERE word_id = any(%s)\"\"\",\n (list(candidates.values()), ))\n conn.commit()\n\n def update_word_tokens(self) -> None:\n \"\"\" Remove unused tokens.\n \"\"\"\n LOG.warning(\"Cleaning up housenumber tokens.\")\n self._cleanup_housenumbers()\n LOG.warning(\"Tokenizer house-keeping done.\")\n\n def name_analyzer(self) -> 'ICUNameAnalyzer':\n \"\"\" Create a new analyzer for tokenizing names and queries\n using this tokinzer. Analyzers are context managers and should\n be used accordingly:\n\n ```\n with tokenizer.name_analyzer() as analyzer:\n analyser.tokenize()\n ```\n\n When used outside the with construct, the caller must ensure to\n call the close() function before destructing the analyzer.\n\n Analyzers are not thread-safe. You need to instantiate one per thread.\n \"\"\"\n assert self.loader is not None\n return ICUNameAnalyzer(self.dsn, self.loader.make_sanitizer(),\n self.loader.make_token_analysis())\n\n def most_frequent_words(self, conn: Connection, num: int) -> List[str]:\n \"\"\" Return a list of the `num` most frequent full words\n in the database.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"\"\"SELECT word, sum((info->>'count')::int) as count\n FROM word WHERE type = 'W'\n GROUP BY word\n ORDER BY count DESC LIMIT %s\"\"\", (num,))\n return list(s[0].split('@')[0] for s in cur)\n\n def _save_config(self) -> None:\n \"\"\" Save the configuration that needs to remain stable for the given\n database as database properties.\n \"\"\"\n assert self.loader is not None\n with connect(self.dsn) as conn:\n self.loader.save_config_to_db(conn)\n\n def _setup_db_tables(self, config: Configuration) -> None:\n \"\"\" Set up the word table and fill it with pre-computed word\n frequencies.\n \"\"\"\n with connect(self.dsn) as conn:\n drop_tables(conn, 'word')\n sqlp = SQLPreprocessor(conn, config)\n sqlp.run_string(conn, \"\"\"\n CREATE TABLE word (\n word_id INTEGER,\n word_token text NOT NULL,\n type text NOT NULL,\n word text,\n info jsonb\n ) {{db.tablespace.search_data}};\n GRANT SELECT ON word TO \"{{config.DATABASE_WEBUSER}}\";\n\n DROP SEQUENCE IF EXISTS seq_word;\n CREATE SEQUENCE seq_word start 1;\n GRANT SELECT ON seq_word to \"{{config.DATABASE_WEBUSER}}\";\n \"\"\")\n conn.commit()\n\n def _create_base_indices(self, config: Configuration, table_name: str) -> None:\n \"\"\" Set up the word table and fill it with pre-computed word\n frequencies.\n \"\"\"\n with connect(self.dsn) as conn:\n sqlp = SQLPreprocessor(conn, config)\n sqlp.run_string(conn,\n \"\"\"CREATE INDEX idx_{{table_name}}_word_token ON {{table_name}}\n USING BTREE (word_token) {{db.tablespace.search_index}}\"\"\",\n table_name=table_name)\n for name, ctype in WORD_TYPES:\n sqlp.run_string(conn,\n \"\"\"CREATE INDEX idx_{{table_name}}_{{idx_name}} ON {{table_name}}\n USING BTREE (word) {{db.tablespace.address_index}}\n WHERE type = '{{column_type}}'\n \"\"\",\n table_name=table_name, idx_name=name,\n column_type=ctype)\n conn.commit()\n\n def _create_lookup_indices(self, config: Configuration, table_name: str) -> None:\n \"\"\" Create additional indexes used when running the API.\n \"\"\"\n with connect(self.dsn) as conn:\n sqlp = SQLPreprocessor(conn, config)\n # Index required for details lookup.\n sqlp.run_string(\n conn,\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_{{table_name}}_word_id\n ON {{table_name}} USING BTREE (word_id) {{db.tablespace.search_index}}\n \"\"\",\n table_name=table_name)\n conn.commit()\n\n def _move_temporary_word_table(self, old: str) -> None:\n \"\"\" Rename all tables and indexes used by the tokenizer.\n \"\"\"\n with connect(self.dsn) as conn:\n drop_tables(conn, 'word')\n with conn.cursor() as cur:\n cur.execute(pysql.SQL(\"ALTER TABLE {} RENAME TO word\")\n .format(pysql.Identifier(old)))\n for idx in ['word_token', 'word_id'] + [n[0] for n in WORD_TYPES]:\n cur.execute(pysql.SQL(\"ALTER INDEX {} RENAME TO {}\")\n .format(pysql.Identifier(f\"idx_{old}_{idx}\"),\n pysql.Identifier(f\"idx_word_{idx}\")))\n conn.commit()\n\n\nclass ICUNameAnalyzer(AbstractAnalyzer):\n \"\"\" The ICU analyzer uses the ICU library for splitting names.\n\n Each instance opens a connection to the database to request the\n normalization.\n \"\"\"\n\n def __init__(self, dsn: str, sanitizer: PlaceSanitizer,\n token_analysis: ICUTokenAnalysis) -> None:\n self.conn: Optional[Connection] = connect(dsn)\n self.conn.autocommit = True\n self.sanitizer = sanitizer\n self.token_analysis = token_analysis\n\n self._cache = _TokenCache()\n\n def close(self) -> None:\n \"\"\" Free all resources used by the analyzer.\n \"\"\"\n if self.conn:\n self.conn.close()\n self.conn = None\n\n def _search_normalized(self, name: str) -> str:\n \"\"\" Return the search token transliteration of the given name.\n \"\"\"\n return cast(str, self.token_analysis.search.transliterate(name)).strip()\n\n def _normalized(self, name: str) -> str:\n \"\"\" Return the normalized version of the given name with all\n non-relevant information removed.\n \"\"\"\n return cast(str, self.token_analysis.normalizer.transliterate(name)).strip()\n\n def get_word_token_info(self, words: Sequence[str]) -> List[Tuple[str, str, Optional[int]]]:\n \"\"\" Return token information for the given list of words.\n If a word starts with # it is assumed to be a full name\n otherwise is a partial name.\n\n The function returns a list of tuples with\n (original word, word token, word id).\n\n The function is used for testing and debugging only\n and not necessarily efficient.\n \"\"\"\n assert self.conn is not None\n full_tokens = {}\n partial_tokens = {}\n for word in words:\n if word.startswith('#'):\n full_tokens[word] = self._search_normalized(word[1:])\n else:\n partial_tokens[word] = self._search_normalized(word)\n\n with self.conn.cursor() as cur:\n cur.execute(\"\"\"SELECT word_token, word_id\n FROM word WHERE word_token = ANY(%s) and type = 'W'\n \"\"\", (list(full_tokens.values()),))\n full_ids = {r[0]: cast(int, r[1]) for r in cur}\n cur.execute(\"\"\"SELECT word_token, word_id\n FROM word WHERE word_token = ANY(%s) and type = 'w'\"\"\",\n (list(partial_tokens.values()),))\n part_ids = {r[0]: cast(int, r[1]) for r in cur}\n\n return [(k, v, full_ids.get(v, None)) for k, v in full_tokens.items()] \\\n + [(k, v, part_ids.get(v, None)) for k, v in partial_tokens.items()]\n\n def normalize_postcode(self, postcode: str) -> str:\n \"\"\" Convert the postcode to a standardized form.\n\n This function must yield exactly the same result as the SQL function\n 'token_normalized_postcode()'.\n \"\"\"\n return postcode.strip().upper()\n\n def update_postcodes_from_db(self) -> None:\n \"\"\" Postcode update.\n\n Removes all postcodes from the word table because they are not\n needed. Postcodes are recognised by pattern.\n \"\"\"\n assert self.conn is not None\n\n with self.conn.cursor() as cur:\n cur.execute(\"DELETE FROM word WHERE type = 'P'\")\n\n def update_special_phrases(self, phrases: Iterable[Tuple[str, str, str, str]],\n should_replace: bool) -> None:\n \"\"\" Replace the search index for special phrases with the new phrases.\n If `should_replace` is True, then the previous set of will be\n completely replaced. Otherwise the phrases are added to the\n already existing ones.\n \"\"\"\n assert self.conn is not None\n norm_phrases = set(((self._normalized(p[0]), p[1], p[2], p[3])\n for p in phrases))\n\n with self.conn.cursor() as cur:\n # Get the old phrases.\n existing_phrases = set()\n cur.execute(\"SELECT word, info FROM word WHERE type = 'S'\")\n for word, info in cur:\n existing_phrases.add((word, info['class'], info['type'],\n info.get('op') or '-'))\n\n added = self._add_special_phrases(cur, norm_phrases, existing_phrases)\n if should_replace:\n deleted = self._remove_special_phrases(cur, norm_phrases,\n existing_phrases)\n else:\n deleted = 0\n\n LOG.info(\"Total phrases: %s. Added: %s. Deleted: %s\",\n len(norm_phrases), added, deleted)\n\n def _add_special_phrases(self, cursor: Cursor,\n new_phrases: Set[Tuple[str, str, str, str]],\n existing_phrases: Set[Tuple[str, str, str, str]]) -> int:\n \"\"\" Add all phrases to the database that are not yet there.\n \"\"\"\n to_add = new_phrases - existing_phrases\n\n added = 0\n with cursor.copy('COPY word(word_token, type, word, info) FROM STDIN') as copy:\n for word, cls, typ, oper in to_add:\n term = self._search_normalized(word)\n if term:\n copy.write_row((term, 'S', word,\n Jsonb({'class': cls, 'type': typ,\n 'op': oper if oper in ('in', 'near') else None})))\n added += 1\n\n return added\n\n def _remove_special_phrases(self, cursor: Cursor,\n new_phrases: Set[Tuple[str, str, str, str]],\n existing_phrases: Set[Tuple[str, str, str, str]]) -> int:\n \"\"\" Remove all phrases from the database that are no longer in the\n new phrase list.\n \"\"\"\n to_delete = existing_phrases - new_phrases\n\n if to_delete:\n cursor.executemany(\n \"\"\" DELETE FROM word\n WHERE type = 'S' and word = %s\n and info->>'class' = %s and info->>'type' = %s\n and %s = coalesce(info->>'op', '-')\n \"\"\", to_delete)\n\n return len(to_delete)\n\n def add_country_names(self, country_code: str, names: Mapping[str, str]) -> None:\n \"\"\" Add default names for the given country to the search index.\n \"\"\"\n # Make sure any name preprocessing for country names applies.\n info = PlaceInfo({'name': names, 'country_code': country_code,\n 'rank_address': 4, 'class': 'boundary',\n 'type': 'administrative'})\n self._add_country_full_names(country_code,\n self.sanitizer.process_names(info)[0],\n internal=True)\n\n def _add_country_full_names(self, country_code: str, names: Sequence[PlaceName],\n internal: bool = False) -> None:\n \"\"\" Add names for the given country from an already sanitized\n name list.\n \"\"\"\n assert self.conn is not None\n word_tokens = set()\n for name in names:\n norm_name = self._normalized(name.name)\n token_name = self._search_normalized(name.name)\n if norm_name and token_name:\n word_tokens.add((token_name, norm_name))\n\n with self.conn.cursor() as cur:\n # Get existing names\n cur.execute(\"\"\"SELECT word_token,\n word as lookup,\n coalesce(info ? 'internal', false) as is_internal\n FROM word\n WHERE type = 'C' and info->>'cc' = %s\"\"\",\n (country_code, ))\n # internal/external names\n existing_tokens: Dict[bool, Set[Tuple[str, str]]] = {True: set(), False: set()}\n for word in cur:\n existing_tokens[word[2]].add((word[0], word[1]))\n\n # Delete names that no longer exist.\n gone_tokens = existing_tokens[internal] - word_tokens\n if internal:\n gone_tokens.update(existing_tokens[False] & word_tokens)\n if gone_tokens:\n cur.execute(\"\"\"DELETE FROM word\n USING jsonb_array_elements(%s) as data\n WHERE type = 'C' and info->>'cc' = %s\n and word_token = data->>0 and word = data->>1\"\"\",\n (Jsonb(list(gone_tokens)), country_code))\n\n # Only add those names that are not yet in the list.\n new_tokens = word_tokens - existing_tokens[True]\n if not internal:\n new_tokens -= existing_tokens[False]\n if new_tokens:\n if internal:\n sql = \"\"\"INSERT INTO word (word_token, type, word, info)\n (SELECT data->>0, 'C', data->>1,\n jsonb_build_object('internal', 'yes', 'cc', %s::text)\n FROM jsonb_array_elements(%s) as data)\n \"\"\"\n else:\n sql = \"\"\"INSERT INTO word (word_token, type, word, info)\n (SELECT data->>0, 'C', data->>1,\n jsonb_build_object('cc', %s::text)\n FROM jsonb_array_elements(%s) as data)\n \"\"\"\n cur.execute(sql, (country_code, Jsonb(list(new_tokens))))\n\n def process_place(self, place: PlaceInfo) -> Mapping[str, Any]:\n \"\"\" Determine tokenizer information about the given place.\n\n Returns a JSON-serializable structure that will be handed into\n the database via the token_info field.\n \"\"\"\n token_info = _TokenInfo()\n\n names, address = self.sanitizer.process_names(place)\n\n if names:\n token_info.set_names(*self._compute_name_tokens(names))\n\n if place.is_country():\n assert place.country_code is not None\n self._add_country_full_names(place.country_code, names)\n\n if address:\n self._process_place_address(token_info, address)\n\n return token_info.to_dict()\n\n def _process_place_address(self, token_info: '_TokenInfo',\n address: Sequence[PlaceName]) -> None:\n for item in address:\n if item.kind == 'postcode':\n token_info.set_postcode(self._add_postcode(item))\n elif item.kind == 'housenumber':\n token_info.add_housenumber(*self._compute_housenumber_token(item))\n elif item.kind == 'street':\n token_info.add_street(self._retrieve_full_tokens(item.name))\n elif item.kind == 'place':\n if not item.suffix:\n token_info.add_place(itertools.chain(*self._compute_name_tokens([item])))\n elif (not item.kind.startswith('_') and not item.suffix and\n item.kind not in ('country', 'full', 'inclusion')):\n token_info.add_address_term(item.kind,\n itertools.chain(*self._compute_name_tokens([item])))\n\n def _compute_housenumber_token(self, hnr: PlaceName) -> Tuple[Optional[int], Optional[str]]:\n \"\"\" Normalize the housenumber and return the word token and the\n canonical form.\n \"\"\"\n assert self.conn is not None\n analyzer = self.token_analysis.analysis.get('@housenumber')\n result: Tuple[Optional[int], Optional[str]] = (None, None)\n\n if analyzer is None:\n # When no custom analyzer is set, simply normalize and transliterate\n norm_name = self._search_normalized(hnr.name)\n if norm_name:\n result = self._cache.housenumbers.get(norm_name, result)\n if result[0] is None:\n hid = execute_scalar(self.conn, \"SELECT getorcreate_hnr_id(%s)\", (norm_name, ))\n\n result = hid, norm_name\n self._cache.housenumbers[norm_name] = result\n else:\n # Otherwise use the analyzer to determine the canonical name.\n # Per convention we use the first variant as the 'lookup name', the\n # name that gets saved in the housenumber field of the place.\n word_id = analyzer.get_canonical_id(hnr)\n if word_id:\n result = self._cache.housenumbers.get(word_id, result)\n if result[0] is None:\n varout = analyzer.compute_variants(word_id)\n if isinstance(varout, tuple):\n variants = varout[0]\n else:\n variants = varout\n if variants:\n hid = execute_scalar(self.conn, \"SELECT create_analyzed_hnr_id(%s, %s)\",\n (word_id, variants))\n result = hid, variants[0]\n self._cache.housenumbers[word_id] = result\n\n return result\n\n def _retrieve_full_tokens(self, name: str) -> List[int]:\n \"\"\" Get the full name token for the given name, if it exists.\n The name is only retrieved for the standard analyser.\n \"\"\"\n assert self.conn is not None\n norm_name = self._search_normalized(name)\n\n # return cached if possible\n if norm_name in self._cache.fulls:\n return self._cache.fulls[norm_name]\n\n with self.conn.cursor() as cur:\n cur.execute(\"SELECT word_id FROM word WHERE word_token = %s and type = 'W'\",\n (norm_name, ))\n full = [row[0] for row in cur]\n\n self._cache.fulls[norm_name] = full\n\n return full\n\n def _compute_name_tokens(self, names: Sequence[PlaceName]) -> Tuple[Set[int], Set[int]]:\n \"\"\" Computes the full name and partial name tokens for the given\n dictionary of names.\n \"\"\"\n assert self.conn is not None\n full_tokens: Set[int] = set()\n partial_tokens: Set[int] = set()\n\n for name in names:\n analyzer_id = name.get_attr('analyzer')\n analyzer = self.token_analysis.get_analyzer(analyzer_id)\n word_id = analyzer.get_canonical_id(name)\n if analyzer_id is None:\n token_id = word_id\n else:\n token_id = f'{word_id}@{analyzer_id}'\n\n full, part = self._cache.names.get(token_id, (None, None))\n if full is None:\n varset = analyzer.compute_variants(word_id)\n if isinstance(varset, tuple):\n variants, lookups = varset\n else:\n variants, lookups = varset, None\n if not variants:\n continue\n\n with self.conn.cursor() as cur:\n cur.execute(\"SELECT * FROM getorcreate_full_word(%s, %s, %s)\",\n (token_id, variants, lookups))\n full, part = cast(Tuple[int, List[int]], cur.fetchone())\n\n self._cache.names[token_id] = (full, part)\n\n assert part is not None\n\n full_tokens.add(full)\n partial_tokens.update(part)\n\n return full_tokens, partial_tokens\n\n def _add_postcode(self, item: PlaceName) -> Optional[str]:\n \"\"\" Make sure the normalized postcode is present in the word table.\n \"\"\"\n assert self.conn is not None\n analyzer = self.token_analysis.analysis.get('@postcode')\n\n if analyzer is None:\n return item.name.strip().upper()\n else:\n return analyzer.get_canonical_id(item)\n\n\nclass _TokenInfo:\n \"\"\" Collect token information to be sent back to the database.\n \"\"\"\n def __init__(self) -> None:\n self.names: Optional[str] = None\n self.housenumbers: Set[str] = set()\n self.housenumber_tokens: Set[int] = set()\n self.street_tokens: Optional[Set[int]] = None\n self.place_tokens: Set[int] = set()\n self.address_tokens: Dict[str, str] = {}\n self.postcode: Optional[str] = None\n\n def _mk_array(self, tokens: Iterable[Any]) -> str:\n return f\"{{{','.join((str(s) for s in tokens))}}}\"\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\" Return the token information in database importable format.\n \"\"\"\n out: Dict[str, Any] = {}\n\n if self.names:\n out['names'] = self.names\n\n if self.housenumbers:\n out['hnr'] = ';'.join(self.housenumbers)\n out['hnr_tokens'] = self._mk_array(self.housenumber_tokens)\n\n if self.street_tokens is not None:\n out['street'] = self._mk_array(self.street_tokens)\n\n if self.place_tokens:\n out['place'] = self._mk_array(self.place_tokens)\n\n if self.address_tokens:\n out['addr'] = self.address_tokens\n\n if self.postcode:\n out['postcode'] = self.postcode\n\n return out\n\n def set_names(self, fulls: Iterable[int], partials: Iterable[int]) -> None:\n \"\"\" Adds token information for the normalised names.\n \"\"\"\n self.names = self._mk_array(itertools.chain(fulls, partials))\n\n def add_housenumber(self, token: Optional[int], hnr: Optional[str]) -> None:\n \"\"\" Extract housenumber information from a list of normalised\n housenumbers.\n \"\"\"\n if token:\n assert hnr is not None\n self.housenumbers.add(hnr)\n self.housenumber_tokens.add(token)\n\n def add_street(self, tokens: Iterable[int]) -> None:\n \"\"\" Add addr:street match terms.\n \"\"\"\n if self.street_tokens is None:\n self.street_tokens = set()\n self.street_tokens.update(tokens)\n\n def add_place(self, tokens: Iterable[int]) -> None:\n \"\"\" Add addr:place search and match terms.\n \"\"\"\n self.place_tokens.update(tokens)\n\n def add_address_term(self, key: str, partials: Iterable[int]) -> None:\n \"\"\" Add additional address terms.\n \"\"\"\n array = self._mk_array(partials)\n if len(array) > 2:\n self.address_tokens[key] = array\n\n def set_postcode(self, postcode: Optional[str]) -> None:\n \"\"\" Set the postcode to the given one.\n \"\"\"\n self.postcode = postcode\n\n\nclass _TokenCache:\n \"\"\" Cache for token information to avoid repeated database queries.\n\n This cache is not thread-safe and needs to be instantiated per\n analyzer.\n \"\"\"\n def __init__(self) -> None:\n self.names: Dict[str, Tuple[int, List[int]]] = {}\n self.partials: Dict[str, int] = {}\n self.fulls: Dict[str, List[int]] = {}\n self.housenumbers: Dict[str, Tuple[Optional[int], Optional[str]]] = {}\n"
},
{
"path": "src/nominatim_db/tokenizer/place_sanitizer.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHandler for cleaning name and address tags in place information before it\nis handed to the token analysis.\n\"\"\"\nfrom typing import Optional, List, Mapping, Sequence, Callable, Any, Tuple\n\nfrom ..errors import UsageError\nfrom ..config import Configuration\nfrom .sanitizers.config import SanitizerConfig\nfrom .sanitizers.base import SanitizerHandler, ProcessInfo\nfrom ..data.place_name import PlaceName\nfrom ..data.place_info import PlaceInfo\n\n\nclass PlaceSanitizer:\n \"\"\" Controller class which applies sanitizer functions on the place\n names and address before they are used by the token analysers.\n \"\"\"\n\n def __init__(self, rules: Optional[Sequence[Mapping[str, Any]]],\n config: Configuration) -> None:\n self.handlers: List[Callable[[ProcessInfo], None]] = []\n\n if rules:\n for func in rules:\n if 'step' not in func:\n raise UsageError(\"Sanitizer rule is missing the 'step' attribute.\")\n if not isinstance(func['step'], str):\n raise UsageError(\"'step' attribute must be a simple string.\")\n\n module: SanitizerHandler = \\\n config.load_plugin_module(func['step'], 'nominatim_db.tokenizer.sanitizers')\n\n self.handlers.append(module.create(SanitizerConfig(func)))\n\n def process_names(self, place: PlaceInfo) -> Tuple[List[PlaceName], List[PlaceName]]:\n \"\"\" Extract a sanitized list of names and address parts from the\n given place. The function returns a tuple\n (list of names, list of address names)\n \"\"\"\n obj = ProcessInfo(place)\n\n for func in self.handlers:\n func(obj)\n\n return obj.names, obj.address\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/base.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nCommon data types and protocols for sanitizers.\n\"\"\"\nfrom typing import Optional, List, Mapping, Callable\n\nfrom ...typing import Protocol, Final\nfrom ...data.place_info import PlaceInfo\nfrom ...data.place_name import PlaceName\nfrom .config import SanitizerConfig\n\n\nclass ProcessInfo:\n \"\"\" Container class for information handed into to handler functions.\n The 'names' and 'address' members are mutable. A handler must change\n them by either modifying the lists place or replacing the old content\n with a new list.\n \"\"\"\n\n def __init__(self, place: PlaceInfo):\n self.place: Final = place\n self.names = self._convert_name_dict(place.name)\n self.address = self._convert_name_dict(place.address)\n\n @staticmethod\n def _convert_name_dict(names: Optional[Mapping[str, str]]) -> List[PlaceName]:\n \"\"\" Convert a dictionary of names into a list of PlaceNames.\n The dictionary key is split into the primary part of the key\n and the suffix (the part after an optional colon).\n \"\"\"\n out = []\n\n if names:\n for key, value in names.items():\n parts = key.split(':', 1)\n out.append(PlaceName(value.strip(),\n parts[0].strip(),\n parts[1].strip() if len(parts) > 1 else None))\n\n return out\n\n\nclass SanitizerHandler(Protocol):\n \"\"\" Protocol for sanitizer modules.\n \"\"\"\n\n def create(self, config: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\"\n Create a function for sanitizing a place.\n\n Arguments:\n config: A dictionary with the additional configuration options\n specified in the tokenizer configuration\n\n Return:\n The result must be a callable that takes a place description\n and transforms name and address as required.\n \"\"\"\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/clean_housenumbers.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSanitizer that preprocesses address tags for house numbers. The sanitizer\nallows to\n\n* define which tags are to be considered house numbers (see 'filter-kind')\n* split house number lists into individual numbers (see 'delimiters')\n* expand interpolated house numbers\n\nArguments:\n delimiters: Define the set of characters to be used for\n splitting a list of house numbers into parts. (default: ',;')\n filter-kind: Define the address tags that are considered to be a\n house number. Either takes a single string or a list of strings,\n where each string is a regular expression. An address item\n is considered a house number if the 'kind' fully matches any\n of the given regular expressions. (default: 'housenumber')\n convert-to-name: Define house numbers that should be treated as a name\n instead of a house number. Either takes a single string\n or a list of strings, where each string is a regular\n expression that must match the full house number value.\n expand-interpolations: When true, expand house number ranges to separate numbers\n when an 'interpolation' is present. (default: true)\n\n\"\"\"\nfrom typing import Callable, Iterator, Iterable, Union\nimport re\n\nfrom ...data.place_name import PlaceName\nfrom .base import ProcessInfo\nfrom .config import SanitizerConfig\n\nRANGE_REGEX = re.compile(r'\\d+-\\d+')\n\n\nclass _HousenumberSanitizer:\n\n def __init__(self, config: SanitizerConfig) -> None:\n self.filter_kind = config.get_filter('filter-kind', ['housenumber'])\n self.split_regexp = config.get_delimiter()\n\n self.filter_name = config.get_filter('convert-to-name', 'FAIL_ALL')\n self.expand_interpolations = config.get_bool('expand-interpolations', True)\n\n def __call__(self, obj: ProcessInfo) -> None:\n if not obj.address:\n return\n\n itype: Union[int, str, None] = None\n if self.expand_interpolations:\n itype = next((i.name for i in obj.address if i.kind == 'interpolation'), None)\n if itype is not None:\n if itype == 'all':\n itype = 1\n elif len(itype) == 1 and itype.isdigit():\n itype = int(itype)\n elif itype not in ('odd', 'even'):\n itype = None\n\n new_address: list[PlaceName] = []\n for item in obj.address:\n if self.filter_kind(item.kind):\n if itype is not None and RANGE_REGEX.fullmatch(item.name):\n hnrs = self._expand_range(itype, item.name)\n if hnrs:\n new_address.extend(item.clone(kind='housenumber', name=str(hnr))\n for hnr in hnrs)\n continue\n\n if self.filter_name(item.name):\n obj.names.append(item.clone(kind='housenumber'))\n else:\n new_address.extend(item.clone(kind='housenumber', name=n)\n for n in self.sanitize(item.name))\n elif item.kind != 'interpolation':\n # Ignore interpolation, otherwise don't touch other address items.\n new_address.append(item)\n\n obj.address = new_address\n\n def sanitize(self, value: str) -> Iterator[str]:\n \"\"\" Extract housenumbers in a regularized format from an OSM value.\n\n The function works as a generator that yields all valid housenumbers\n that can be created from the value.\n \"\"\"\n for hnr in self.split_regexp.split(value):\n if hnr:\n yield from self._regularize(hnr)\n\n def _regularize(self, hnr: str) -> Iterator[str]:\n yield hnr\n\n def _expand_range(self, itype: Union[str, int], hnr: str) -> Iterable[int]:\n first, last = (int(i) for i in hnr.split('-'))\n\n if isinstance(itype, int):\n step = itype\n else:\n step = 2\n if (itype == 'even' and first % 2 == 1)\\\n or (itype == 'odd' and first % 2 == 0):\n first += 1\n\n if (last + 1 - first) / step < 10:\n return range(first, last + 1, step)\n\n return []\n\n\ndef create(config: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\" Create a housenumber processing function.\n \"\"\"\n\n return _HousenumberSanitizer(config)\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/clean_postcodes.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSanitizer that filters postcodes by their officially allowed pattern.\n\nArguments:\n convert-to-address: If set to 'yes' (the default), then postcodes that do\n not conform with their country-specific pattern are\n converted to an address component. That means that\n the postcode does not take part when computing the\n postcode centroids of a country but is still searchable.\n When set to 'no', non-conforming postcodes are not\n searchable either.\n default-pattern: Pattern to use, when there is none available for the\n country in question. Warning: will not be used for\n objects that have no country assigned. These are always\n assumed to have no postcode.\n\"\"\"\nfrom typing import Callable, Optional, Tuple\n\nfrom ...data.postcode_format import PostcodeFormatter\nfrom .base import ProcessInfo\nfrom .config import SanitizerConfig\n\n\nclass _PostcodeSanitizer:\n\n def __init__(self, config: SanitizerConfig) -> None:\n self.convert_to_address = config.get_bool('convert-to-address', True)\n self.matcher = PostcodeFormatter()\n\n default_pattern = config.get('default-pattern')\n if default_pattern is not None and isinstance(default_pattern, str):\n self.matcher.set_default_pattern(default_pattern)\n\n def __call__(self, obj: ProcessInfo) -> None:\n if not obj.address:\n return\n\n postcodes = ((i, o) for i, o in enumerate(obj.address) if o.kind == 'postcode')\n\n for pos, postcode in postcodes:\n formatted = self.scan(postcode.name, obj.place.country_code)\n\n if formatted is None:\n if self.convert_to_address:\n postcode.kind = 'unofficial_postcode'\n else:\n obj.address.pop(pos)\n else:\n postcode.name = formatted[0]\n postcode.set_attr('variant', formatted[1])\n\n def scan(self, postcode: str, country: Optional[str]) -> Optional[Tuple[str, str]]:\n \"\"\" Check the postcode for correct formatting and return the\n normalized version. Returns None if the postcode does not\n correspond to the official format of the given country.\n \"\"\"\n match = self.matcher.match(country, postcode)\n if match is None:\n return None\n\n assert country is not None\n\n return self.matcher.normalize(country, match), \\\n ' '.join(filter(lambda p: p is not None, match.groups()))\n\n\ndef create(config: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\" Create a function that filters postcodes by their officially allowed pattern.\n \"\"\"\n\n return _PostcodeSanitizer(config)\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/clean_tiger_tags.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSanitizer that preprocesses tags from the TIGER import.\n\nIt makes the following changes:\n\n* remove state reference from tiger:county\n\"\"\"\nfrom typing import Callable\nimport re\n\nfrom .base import ProcessInfo\nfrom .config import SanitizerConfig\n\nCOUNTY_MATCH = re.compile('(.*), [A-Z][A-Z]')\n\n\ndef _clean_tiger_county(obj: ProcessInfo) -> None:\n \"\"\" Remove the state reference from tiger:county tags.\n\n This transforms a name like 'Hamilton, AL' into 'Hamilton'.\n If no state reference is detected at the end, the name is left as is.\n \"\"\"\n if not obj.address:\n return\n\n for item in obj.address:\n if item.kind == 'tiger' and item.suffix == 'county':\n m = COUNTY_MATCH.fullmatch(item.name)\n if m:\n item.name = m[1]\n # Switch kind and suffix, the split left them reversed.\n item.kind = 'county'\n item.suffix = 'tiger'\n\n return\n\n\ndef create(_: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\" Create a function that preprocesses tags from the TIGER import.\n \"\"\"\n return _clean_tiger_county\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/config.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nConfiguration for Sanitizers.\n\"\"\"\nfrom typing import Sequence, Union, Optional, Pattern, Callable, Any, TYPE_CHECKING\nfrom collections import UserDict\nimport re\n\nfrom ...errors import UsageError\n\n# working around missing generics in Python < 3.8\n# See https://github.com/python/typing/issues/60#issuecomment-869757075\nif TYPE_CHECKING:\n _BaseUserDict = UserDict[str, Any]\nelse:\n _BaseUserDict = UserDict\n\n\nclass SanitizerConfig(_BaseUserDict):\n \"\"\" The `SanitizerConfig` class is a read-only dictionary\n with configuration options for the sanitizer.\n In addition to the usual dictionary functions, the class provides\n accessors to standard sanitizer options that are used by many of the\n sanitizers.\n \"\"\"\n\n def get_string_list(self, param: str, default: Sequence[str] = tuple()) -> Sequence[str]:\n \"\"\" Extract a configuration parameter as a string list.\n\n Arguments:\n param: Name of the configuration parameter.\n default: Takes a tuple or list of strings which will\n be returned if the parameter is missing in the\n sanitizer configuration.\n Note that if this default parameter is not\n provided then an empty list is returned.\n\n Returns:\n If the parameter value is a simple string, it is returned as a\n one-item list. If the parameter value does not exist, the given\n default is returned. If the parameter value is a list, it is\n checked to contain only strings before being returned.\n \"\"\"\n values = self.data.get(param, None)\n\n if values is None:\n return list(default)\n\n if isinstance(values, str):\n return [values] if values else []\n\n if not isinstance(values, (list, tuple)):\n raise UsageError(f\"Parameter '{param}' must be string or list of strings.\")\n\n if any(not isinstance(value, str) for value in values):\n raise UsageError(f\"Parameter '{param}' must be string or list of strings.\")\n\n return values\n\n def get_bool(self, param: str, default: Optional[bool] = None) -> bool:\n \"\"\" Extract a configuration parameter as a boolean.\n\n Arguments:\n param: Name of the configuration parameter. The parameter must\n contain one of the yaml boolean values or an\n UsageError will be raised.\n default: Value to return, when the parameter is missing.\n When set to `None`, the parameter must be defined.\n\n Returns:\n Boolean value of the given parameter.\n \"\"\"\n value = self.data.get(param, default)\n\n if not isinstance(value, bool):\n raise UsageError(f\"Parameter '{param}' must be a boolean value ('yes' or 'no').\")\n\n return value\n\n def get_delimiter(self, default: str = ',;') -> Pattern[str]:\n \"\"\" Return the 'delimiters' parameter in the configuration as a\n compiled regular expression that can be used to split strings on\n these delimiters.\n\n Arguments:\n default: Delimiters to be used when 'delimiters' parameter\n is not explicitly configured.\n\n Returns:\n A regular expression pattern which can be used to\n split a string. The regular expression makes sure that the\n resulting names are stripped and that repeated delimiters\n are ignored. It may still create empty fields on occasion. The\n code needs to filter those.\n \"\"\"\n delimiter_set = set(self.data.get('delimiters', default))\n if not delimiter_set:\n raise UsageError(\"Empty 'delimiter' parameter not allowed for sanitizer.\")\n\n return re.compile('\\\\s*[{}]+\\\\s*'.format(''.join('\\\\' + d for d in delimiter_set)))\n\n def get_filter(self, param: str, default: Union[str, Sequence[str]] = 'PASS_ALL'\n ) -> Callable[[str], bool]:\n \"\"\" Returns a filter function for the given parameter of the sanitizer\n configuration.\n\n The value provided for the parameter in sanitizer configuration\n should be a string or list of strings, where each string is a regular\n expression. These regular expressions will later be used by the\n filter function to filter strings.\n\n Arguments:\n param: The parameter for which the filter function\n will be created.\n default: Defines the behaviour of filter function if\n parameter is missing in the sanitizer configuration.\n Takes a string(PASS_ALL or FAIL_ALL) or a list of strings.\n Any other value of string or an empty list is not allowed,\n and will raise a ValueError. If the value is PASS_ALL, the filter\n function will let all strings to pass, if the value is FAIL_ALL,\n filter function will let no strings to pass.\n If value provided is a list of strings each string\n is treated as a regular expression. In this case these regular\n expressions will be used by the filter function.\n By default allow filter function to let all strings pass.\n\n Returns:\n A filter function that takes a target string as the argument and\n returns True if it fully matches any of the regular expressions\n otherwise returns False.\n \"\"\"\n filters = self.get_string_list(param) or default\n\n if filters == 'PASS_ALL':\n return lambda _: True\n if filters == 'FAIL_ALL':\n return lambda _: False\n\n if filters and isinstance(filters, (list, tuple)):\n regexes = [re.compile(regex) for regex in filters]\n return lambda target: any(regex.fullmatch(target) for regex in regexes)\n\n raise ValueError(\"Default parameter must be a non-empty list or a string value \\\n ('PASS_ALL' or 'FAIL_ALL').\")\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/delete_tags.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\r\n#\r\n# This file is part of Nominatim. (https://nominatim.org)\r\n#\r\n# Copyright (C) 2024 by the Nominatim developer community.\r\n# For a full list of authors see the git log.\r\n\"\"\"\r\nSanitizer which prevents certain tags from getting into the search index.\r\nIt remove tags which matches all properties given below.\r\n\r\n\r\nArguments:\r\n type: Define which type of tags should be considered for removal.\r\n There are two types of tags 'name' and 'address' tags.\r\n Takes a string 'name' or 'address'. (default: 'name')\r\n\r\n filter-kind: Define which 'kind' of tags should be removed.\r\n Takes a string or list of strings where each\r\n string is a regular expression. A tag is considered\r\n to be a candidate for removal if its 'kind' property\r\n fully matches any of the given regular expressions.\r\n Note that by default all 'kind' of tags are considered.\r\n\r\n suffix: Define the 'suffix' property of the tags which should be\r\n removed. Takes a string or list of strings where each\r\n string is a regular expression. A tag is considered to be a\r\n candidate for removal if its 'suffix' property fully\r\n matches any of the given regular expressions. Note that by\r\n default tags with any suffix value are considered including\r\n those which don't have a suffix at all.\r\n\r\n name: Define the 'name' property corresponding to the 'kind' property\r\n of the tag. Takes a string or list of strings where each string\r\n is a regular expression. A tag is considered to be a candidate\r\n for removal if its name fully matches any of the given regular\r\n expressions. Note that by default tags with any 'name' are\r\n considered.\r\n\r\n country_code: Define the country code of places whose tags should be\r\n considered for removed. Takes a string or list of strings\r\n where each string is a two-letter lower-case country code.\r\n Note that by default tags of places with any country code\r\n are considered including those which don't have a country\r\n code at all.\r\n\r\n rank_address: Define the address rank of places whose tags should be\r\n considered for removal. Takes a string or list of strings\r\n where each string is a number or range of number or the\r\n form -.\r\n Note that default is '0-30', which means that tags of all\r\n places are considered.\r\n See https://nominatim.org/release-docs/latest/customize/Ranking/#address-rank\r\n to learn more about address rank.\r\n\r\n\r\n\"\"\"\r\nfrom typing import Callable, List, Tuple, Sequence\r\n\r\nfrom ...data.place_name import PlaceName\r\nfrom .base import ProcessInfo\r\nfrom .config import SanitizerConfig\r\n\r\n\r\nclass _TagSanitizer:\r\n\r\n def __init__(self, config: SanitizerConfig) -> None:\r\n self.type = config.get('type', 'name')\r\n self.filter_kind = config.get_filter('filter-kind')\r\n self.country_codes = config.get_string_list('country_code', [])\r\n self.filter_suffix = config.get_filter('suffix')\r\n self.filter_name = config.get_filter('name')\r\n self.allowed_ranks = self._set_allowed_ranks(\r\n config.get_string_list(\"rank_address\", [\"0-30\"])\r\n )\r\n\r\n self.has_country_code = config.get('country_code', None) is not None\r\n\r\n def __call__(self, obj: ProcessInfo) -> None:\r\n tags = obj.names if self.type == 'name' else obj.address\r\n\r\n if not tags \\\r\n or not self.allowed_ranks[obj.place.rank_address] \\\r\n or self.has_country_code \\\r\n and obj.place.country_code not in self.country_codes:\r\n return\r\n\r\n filtered_tags: List[PlaceName] = []\r\n\r\n for tag in tags:\r\n\r\n if not self.filter_kind(tag.kind) \\\r\n or not self.filter_suffix(tag.suffix or '') \\\r\n or not self.filter_name(tag.name):\r\n filtered_tags.append(tag)\r\n\r\n if self.type == 'name':\r\n obj.names = filtered_tags\r\n else:\r\n obj.address = filtered_tags\r\n\r\n def _set_allowed_ranks(self, ranks: Sequence[str]) -> Tuple[bool, ...]:\r\n \"\"\" Returns a tuple of 31 boolean values corresponding to the\r\n address ranks 0-30. Value at index 'i' is True if rank 'i'\r\n is present in the ranks or lies in the range of any of the\r\n ranks provided in the sanitizer configuration, otherwise\r\n the value is False.\r\n \"\"\"\r\n allowed_ranks = [False] * 31\r\n\r\n for rank in ranks:\r\n intvl = [int(x) for x in rank.split('-')]\r\n\r\n start, end = intvl[0], intvl[0] if len(intvl) == 1 else intvl[1]\r\n\r\n for i in range(start, end + 1):\r\n allowed_ranks[i] = True\r\n\r\n return tuple(allowed_ranks)\r\n\r\n\r\ndef create(config: SanitizerConfig) -> Callable[[ProcessInfo], None]:\r\n \"\"\" Create a function to process removal of certain tags.\r\n \"\"\"\r\n\r\n return _TagSanitizer(config)\r\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/split_name_list.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSanitizer that splits lists of names into their components.\n\nArguments:\n delimiters: Define the set of characters to be used for\n splitting the list. (default: ',;')\n\"\"\"\nfrom typing import Callable\n\nfrom .base import ProcessInfo\nfrom .config import SanitizerConfig\n\n\ndef create(config: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\" Create a name processing function that splits name values with\n multiple values into their components.\n \"\"\"\n regexp = config.get_delimiter()\n\n def _process(obj: ProcessInfo) -> None:\n if not obj.names:\n return\n\n new_names = []\n for name in obj.names:\n split_names = regexp.split(name.name)\n if len(split_names) == 1:\n new_names.append(name)\n else:\n new_names.extend(name.clone(name=n) for n in split_names if n)\n\n obj.names = new_names\n\n return _process\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/strip_brace_terms.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nThis sanitizer creates additional name variants for names that have\naddendums in brackets (e.g. \"Halle (Saale)\"). The additional variant contains\nonly the main name part with the bracket part removed.\n\"\"\"\nfrom typing import Callable\n\nfrom .base import ProcessInfo\nfrom .config import SanitizerConfig\n\n\ndef create(_: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\" Create a name processing function that creates additional name variants\n for bracket addendums.\n \"\"\"\n def _process(obj: ProcessInfo) -> None:\n \"\"\" Add variants for names that have a bracket extension.\n \"\"\"\n if obj.names:\n new_names = []\n for name in (n for n in obj.names if '(' in n.name):\n if ')' in name.name and not name.name.endswith(')'):\n continue\n new_name = name.name.split('(')[0].strip()\n if new_name:\n new_names.append(name.clone(name=new_name))\n\n obj.names.extend(new_names)\n\n return _process\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/tag_analyzer_by_language.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nThis sanitizer sets the `analyzer` property depending on the\nlanguage of the tag. The language is taken from the suffix of the name.\nIf a name already has an analyzer tagged, then this is kept.\n\nArguments:\n\n filter-kind: Restrict the names the sanitizer should be applied to\n the given tags. The parameter expects a list of\n regular expressions which are matched against 'kind'.\n Note that a match against the full string is expected.\n whitelist: Restrict the set of languages that should be tagged.\n Expects a list of acceptable suffixes. When unset,\n all 2- and 3-letter lower-case codes are accepted.\n use-defaults: Configure what happens when the name has no suffix.\n When set to 'all', a variant is created for\n each of the default languages in the country\n the feature is in. When set to 'mono', a variant is\n only created, when exactly one language is spoken\n in the country. The default is to do nothing with\n the default languages of a country.\n mode: Define how the variants are created and may be 'replace' or\n 'append'. When set to 'append' the original name (without\n any analyzer tagged) is retained. (default: replace)\n\n\"\"\"\nfrom typing import Callable, Dict, Optional, List\n\nfrom ...data import country_info\nfrom .base import ProcessInfo\nfrom .config import SanitizerConfig\n\n\nclass _AnalyzerByLanguage:\n \"\"\" Processor for tagging the language of names in a place.\n \"\"\"\n\n def __init__(self, config: SanitizerConfig) -> None:\n self.filter_kind = config.get_filter('filter-kind')\n self.replace = config.get('mode', 'replace') != 'append'\n self.whitelist = config.get('whitelist')\n\n self._compute_default_languages(config.get('use-defaults', 'no'))\n\n def _compute_default_languages(self, use_defaults: str) -> None:\n self.deflangs: Dict[Optional[str], List[str]] = {}\n\n if use_defaults in ('mono', 'all'):\n for ccode, clangs in country_info.iterate('languages'):\n if len(clangs) == 1 or use_defaults == 'all':\n if self.whitelist:\n self.deflangs[ccode] = [cl for cl in clangs if cl in self.whitelist]\n else:\n self.deflangs[ccode] = clangs\n\n def _suffix_matches(self, suffix: str) -> bool:\n if self.whitelist is None:\n return len(suffix) in (2, 3) and suffix.islower()\n\n return suffix in self.whitelist\n\n def __call__(self, obj: ProcessInfo) -> None:\n if not obj.names:\n return\n\n more_names = []\n\n for name in (n for n in obj.names\n if not n.has_attr('analyzer') and self.filter_kind(n.kind)):\n if name.suffix:\n langs = [name.suffix] if self._suffix_matches(name.suffix) else None\n else:\n langs = self.deflangs.get(obj.place.country_code)\n\n if langs:\n if self.replace:\n name.set_attr('analyzer', langs[0])\n else:\n more_names.append(name.clone(attr={'analyzer': langs[0]}))\n\n more_names.extend(name.clone(attr={'analyzer': lg}) for lg in langs[1:])\n\n obj.names.extend(more_names)\n\n\ndef create(config: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\" Create a function that sets the analyzer property depending on the\n language of the tag.\n \"\"\"\n return _AnalyzerByLanguage(config)\n"
},
{
"path": "src/nominatim_db/tokenizer/sanitizers/tag_japanese.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nThis sanitizer maps OSM data to Japanese block addresses.\nIt replaces blocknumber and housenumber with housenumber,\nand quarter and neighbourhood with place.\n\"\"\"\n\n\nfrom typing import Callable\nfrom typing import List, Optional\n\nfrom .base import ProcessInfo\nfrom .config import SanitizerConfig\nfrom ...data.place_name import PlaceName\n\n\ndef create(_: SanitizerConfig) -> Callable[[ProcessInfo], None]:\n \"\"\"Set up the sanitizer\n \"\"\"\n return tag_japanese\n\n\ndef reconbine_housenumber(\n new_address: List[PlaceName],\n tmp_housenumber: Optional[str],\n tmp_blocknumber: Optional[str]\n) -> List[PlaceName]:\n \"\"\" Recombine the tag of housenumber by using housenumber and blocknumber\n \"\"\"\n if tmp_blocknumber and tmp_housenumber:\n new_address.append(\n PlaceName(\n kind='housenumber',\n name=f'{tmp_blocknumber}-{tmp_housenumber}',\n suffix=''\n )\n )\n elif tmp_blocknumber:\n new_address.append(\n PlaceName(\n kind='housenumber',\n name=tmp_blocknumber,\n suffix=''\n )\n )\n elif tmp_housenumber:\n new_address.append(\n PlaceName(\n kind='housenumber',\n name=tmp_housenumber,\n suffix=''\n )\n )\n return new_address\n\n\ndef reconbine_place(\n new_address: List[PlaceName],\n tmp_neighbourhood: Optional[str],\n tmp_quarter: Optional[str]\n) -> List[PlaceName]:\n \"\"\" Recombine the tag of place by using neighbourhood and quarter\n \"\"\"\n if tmp_neighbourhood and tmp_quarter:\n new_address.append(\n PlaceName(\n kind='place',\n name=f'{tmp_quarter}{tmp_neighbourhood}',\n suffix=''\n )\n )\n elif tmp_neighbourhood:\n new_address.append(\n PlaceName(\n kind='place',\n name=tmp_neighbourhood,\n suffix=''\n )\n )\n elif tmp_quarter:\n new_address.append(\n PlaceName(\n kind='place',\n name=tmp_quarter,\n suffix=''\n )\n )\n return new_address\n\n\ndef tag_japanese(obj: ProcessInfo) -> None:\n \"\"\"Recombine kind of address\n \"\"\"\n if obj.place.country_code != 'jp':\n return\n tmp_housenumber = None\n tmp_blocknumber = None\n tmp_neighbourhood = None\n tmp_quarter = None\n\n new_address = []\n for item in obj.address:\n if item.kind == 'housenumber':\n tmp_housenumber = item.name\n elif item.kind == 'block_number':\n tmp_blocknumber = item.name\n elif item.kind == 'neighbourhood':\n tmp_neighbourhood = item.name\n elif item.kind == 'quarter':\n tmp_quarter = item.name\n else:\n new_address.append(item)\n\n new_address = reconbine_housenumber(new_address, tmp_housenumber, tmp_blocknumber)\n new_address = reconbine_place(new_address, tmp_neighbourhood, tmp_quarter)\n\n obj.address = [item for item in new_address if item.name is not None]\n"
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/base.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nCommon data types and protocols for analysers.\n\"\"\"\nfrom typing import Mapping, List, Any, Union, Tuple\n\nfrom ...typing import Protocol\nfrom ...data.place_name import PlaceName\n\n\nclass Analyzer(Protocol):\n \"\"\" The `create()` function of an analysis module needs to return an\n object that implements the following functions.\n \"\"\"\n\n def get_canonical_id(self, name: PlaceName) -> str:\n \"\"\" Return the canonical form of the given name. The canonical ID must\n be unique (the same ID must always yield the same variants) and\n must be a form from which the variants can be derived.\n\n Arguments:\n name: Extended place name description as prepared by\n the sanitizers.\n\n Returns:\n ID string with a canonical form of the name. The string may\n be empty, when the analyzer cannot analyze the name at all,\n for example because the character set in use does not match.\n \"\"\"\n\n def compute_variants(self, canonical_id: str) -> Union[List[str], Tuple[List[str], List[str]]]:\n \"\"\" Compute the transliterated spelling variants for the given\n canonical ID.\n\n Arguments:\n canonical_id: ID string previously computed with\n `get_canonical_id()`.\n\n Returns:\n A list of possible spelling variants. All strings must have\n been transformed with the global normalizer and\n transliterator ICU rules. Otherwise they cannot be matched\n against the input by the query frontend.\n The list may be empty, when there are no useful\n spelling variants. This may happen when an analyzer only\n usually outputs additional variants to the canonical spelling\n and there are no such variants.\n \"\"\"\n\n\nclass AnalysisModule(Protocol):\n \"\"\" The setup of the token analysis is split into two parts:\n configuration and analyser factory. A token analysis module must\n therefore implement the two functions here described.\n \"\"\"\n\n def configure(self, rules: Mapping[str, Any],\n normalizer: Any, transliterator: Any) -> Any:\n \"\"\" Prepare the configuration of the analysis module.\n This function should prepare all data that can be shared\n between instances of this analyser.\n\n Arguments:\n rules: A dictionary with the additional configuration options\n as specified in the tokenizer configuration.\n normalizer: an ICU Transliterator with the compiled\n global normalization rules.\n transliterator: an ICU Transliterator with the compiled\n global transliteration rules.\n\n Returns:\n A data object with configuration data. This will be handed\n as is into the `create()` function and may be\n used freely by the analysis module as needed.\n \"\"\"\n\n def create(self, normalizer: Any, transliterator: Any, config: Any) -> Analyzer:\n \"\"\" Create a new instance of the analyser.\n A separate instance of the analyser is created for each thread\n when used in multi-threading context.\n\n Arguments:\n normalizer: an ICU Transliterator with the compiled normalization\n rules.\n transliterator: an ICU Transliterator with the compiled\n transliteration rules.\n config: The object that was returned by the call to configure().\n\n Returns:\n A new analyzer instance. This must be an object that implements\n the Analyzer protocol.\n \"\"\"\n"
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/config_variants.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nParser for configuration for variants.\n\"\"\"\nfrom typing import Any, Iterator, Tuple, List, Optional, Set, NamedTuple\nfrom collections import defaultdict\nimport itertools\nimport re\n\nfrom ...config import flatten_config_list\nfrom ...errors import UsageError\n\n\nclass ICUVariant(NamedTuple):\n \"\"\" A single replacement rule for variant creation.\n \"\"\"\n source: str\n replacement: str\n\n\ndef get_variant_config(in_rules: Any,\n normalizer: Any) -> Tuple[List[Tuple[str, List[str]]], str]:\n \"\"\" Convert the variant definition from the configuration into\n replacement sets.\n\n Returns a tuple containing the replacement set and the list of characters\n used in the replacements.\n \"\"\"\n immediate = defaultdict(list)\n chars: Set[str] = set()\n\n if in_rules:\n vset: Set[ICUVariant] = set()\n rules = flatten_config_list(in_rules, 'variants')\n\n vmaker = _VariantMaker(normalizer)\n\n for section in rules:\n for rule in (section.get('words') or []):\n vset.update(vmaker.compute(rule))\n\n # Intermediate reorder by source. Also compute required character set.\n for variant in vset:\n if variant.source[-1] == ' ' and variant.replacement[-1] == ' ':\n replstr = variant.replacement[:-1]\n else:\n replstr = variant.replacement\n immediate[variant.source].append(replstr)\n chars.update(variant.source)\n\n return list(immediate.items()), ''.join(chars)\n\n\nclass _VariantMaker:\n \"\"\" Generator for all necessary ICUVariants from a single variant rule.\n\n All text in rules is normalized to make sure the variants match later.\n \"\"\"\n\n def __init__(self, normalizer: Any) -> None:\n self.norm = normalizer\n\n def compute(self, rule: Any) -> Iterator[ICUVariant]:\n \"\"\" Generator for all ICUVariant tuples from a single variant rule.\n \"\"\"\n parts = re.split(r'(\\|)?([=-])>', rule)\n if len(parts) != 4:\n raise UsageError(f\"Syntax error in variant rule: {rule}\")\n\n decompose = parts[1] is None\n src_terms = [self._parse_variant_word(t) for t in parts[0].split(',')]\n repl_terms = (self.norm.transliterate(t).strip() for t in parts[3].split(','))\n\n # If the source should be kept, add a 1:1 replacement\n if parts[2] == '-':\n for src in src_terms:\n if src:\n for froms, tos in _create_variants(*src, src[0], decompose):\n yield ICUVariant(froms, tos)\n\n for src, repl in itertools.product(src_terms, repl_terms):\n if src and repl:\n for froms, tos in _create_variants(*src, repl, decompose):\n yield ICUVariant(froms, tos)\n\n def _parse_variant_word(self, name: str) -> Optional[Tuple[str, str, str]]:\n name = name.strip()\n match = re.fullmatch(r'([~^]?)([^~$^]*)([~$]?)', name)\n if match is None or (match.group(1) == '~' and match.group(3) == '~'):\n raise UsageError(f\"Invalid variant word descriptor '{name}'\")\n norm_name = self.norm.transliterate(match.group(2)).strip()\n if not norm_name:\n return None\n\n return norm_name, match.group(1), match.group(3)\n\n\n_FLAG_MATCH = {'^': '^ ',\n '$': ' ^',\n '': ' '}\n\n\ndef _create_variants(src: str, preflag: str, postflag: str,\n repl: str, decompose: bool) -> Iterator[Tuple[str, str]]:\n if preflag == '~':\n postfix = _FLAG_MATCH[postflag]\n # suffix decomposition\n src = src + postfix\n repl = repl + postfix\n\n yield src, repl\n yield ' ' + src, ' ' + repl\n\n if decompose:\n yield src, ' ' + repl\n yield ' ' + src, repl\n elif postflag == '~':\n # prefix decomposition\n prefix = _FLAG_MATCH[preflag]\n src = prefix + src\n repl = prefix + repl\n\n yield src, repl\n yield src + ' ', repl + ' '\n\n if decompose:\n yield src, repl + ' '\n yield src + ' ', repl\n else:\n prefix = _FLAG_MATCH[preflag]\n postfix = _FLAG_MATCH[postflag]\n\n yield prefix + src + postfix, prefix + repl + postfix\n"
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/generic.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nGeneric processor for names that creates abbreviation variants.\n\"\"\"\nfrom typing import Mapping, Dict, Any, Iterable, Optional, List, cast, Tuple\nimport itertools\n\nfrom ...errors import UsageError\nfrom ...data.place_name import PlaceName\nfrom .config_variants import get_variant_config\nfrom .generic_mutation import MutationVariantGenerator\nfrom .simple_trie import SimpleTrie\n\n# Configuration section\n\n\ndef configure(rules: Mapping[str, Any], normalizer: Any, _: Any) -> Dict[str, Any]:\n \"\"\" Extract and preprocess the configuration for this module.\n \"\"\"\n config: Dict[str, Any] = {}\n\n config['replacements'], _ = get_variant_config(rules.get('variants'), normalizer)\n config['variant_only'] = rules.get('mode', '') == 'variant-only'\n\n # parse mutation rules\n config['mutations'] = []\n for rule in rules.get('mutations', []):\n if 'pattern' not in rule:\n raise UsageError(\"Missing field 'pattern' in mutation configuration.\")\n if not isinstance(rule['pattern'], str):\n raise UsageError(\"Field 'pattern' in mutation configuration \"\n \"must be a simple text field.\")\n if 'replacements' not in rule:\n raise UsageError(\"Missing field 'replacements' in mutation configuration.\")\n if not isinstance(rule['replacements'], list):\n raise UsageError(\"Field 'replacements' in mutation configuration \"\n \"must be a list of texts.\")\n\n config['mutations'].append((rule['pattern'], rule['replacements']))\n\n return config\n\n\n# Analysis section\n\ndef create(normalizer: Any, transliterator: Any,\n config: Mapping[str, Any]) -> 'GenericTokenAnalysis':\n \"\"\" Create a new token analysis instance for this module.\n \"\"\"\n return GenericTokenAnalysis(normalizer, transliterator, config)\n\n\nclass GenericTokenAnalysis:\n \"\"\" Collects the different transformation rules for normalisation of names\n and provides the functions to apply the transformations.\n \"\"\"\n\n def __init__(self, norm: Any, to_ascii: Any, config: Mapping[str, Any]) -> None:\n self.norm = norm\n self.to_ascii = to_ascii\n self.variant_only = config['variant_only']\n\n # Set up datrie\n self.replacements: Optional[SimpleTrie[List[str]]] = \\\n SimpleTrie(config['replacements']) if config['replacements'] else None\n\n # set up mutation rules\n self.mutations = [MutationVariantGenerator(*cfg) for cfg in config['mutations']]\n\n def get_canonical_id(self, name: PlaceName) -> str:\n \"\"\" Return the normalized form of the name. This is the standard form\n from which possible variants for the name can be derived.\n \"\"\"\n return cast(str, self.norm.transliterate(name.name)).strip()\n\n def compute_variants(self, norm_name: str) -> Tuple[List[str], List[str]]:\n \"\"\" Compute the spelling variants for the given normalized name\n and transliterate the result.\n \"\"\"\n variants = self._generate_word_variants(norm_name)\n\n for mutation in self.mutations:\n variants = mutation.generate(variants)\n\n varset = set(map(str.strip, variants))\n if self.variant_only:\n varset.discard(norm_name)\n\n trans = []\n norm = []\n\n for var in varset:\n t = self.to_ascii.transliterate(var).strip()\n if t:\n trans.append(t)\n norm.append(var)\n\n return trans, norm\n\n def _generate_word_variants(self, norm_name: str) -> Iterable[str]:\n baseform = '^ ' + norm_name + ' ^'\n baselen = len(baseform)\n partials = ['']\n\n startpos = 0\n if self.replacements is not None:\n pos = 0\n force_space = False\n while pos < baselen:\n frm = pos\n repl, pos = self.replacements.longest_prefix(baseform, pos)\n if repl is not None:\n done = baseform[startpos:frm]\n partials = [v + done + r\n for v, r in itertools.product(partials, repl)\n if not force_space or r.startswith(' ')]\n if len(partials) > 128:\n # If too many variants are produced, they are unlikely\n # to be helpful. Only use the original term.\n startpos = 0\n break\n if baseform[pos - 1] == ' ':\n pos -= 1\n force_space = True\n startpos = pos\n else:\n pos += 1\n force_space = False\n\n # No variants detected? Fast return.\n if startpos == 0:\n return (norm_name, )\n\n if startpos < baselen:\n return (part[1:] + baseform[startpos:-1] for part in partials)\n\n return (part[1:-1] for part in partials)\n"
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/generic_mutation.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nCreator for mutation variants for the generic token analysis.\n\"\"\"\nfrom typing import Sequence, Iterable, Iterator, Tuple\nimport itertools\nimport logging\nimport re\n\nfrom ...errors import UsageError\n\nLOG = logging.getLogger()\n\n\ndef _zigzag(outer: Iterable[str], inner: Iterable[str]) -> Iterator[str]:\n return itertools.chain.from_iterable(itertools.zip_longest(outer, inner, fillvalue=''))\n\n\nclass MutationVariantGenerator:\n \"\"\" Generates name variants by applying a regular expression to the name\n and replacing it with one or more variants. When the regular expression\n matches more than once, each occurrence is replaced with all replacement\n patterns.\n \"\"\"\n\n def __init__(self, pattern: str, replacements: Sequence[str]):\n self.pattern = re.compile(pattern)\n self.replacements = replacements\n\n if self.pattern.groups > 0:\n LOG.fatal(\"The mutation pattern %s contains a capturing group. \"\n \"This is not allowed.\", pattern)\n raise UsageError(\"Bad mutation pattern in configuration.\")\n\n def generate(self, names: Iterable[str]) -> Iterator[str]:\n \"\"\" Generator function for the name variants. 'names' is an iterable\n over a set of names for which the variants are to be generated.\n \"\"\"\n for name in names:\n parts = self.pattern.split(name)\n if len(parts) == 1:\n yield name\n else:\n for seps in self._fillers(len(parts)):\n yield ''.join(_zigzag(parts, seps))\n\n def _fillers(self, num_parts: int) -> Iterator[Tuple[str, ...]]:\n \"\"\" Returns a generator for strings to join the given number of string\n parts in all possible combinations.\n \"\"\"\n return itertools.product(self.replacements, repeat=num_parts - 1)\n"
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/housenumbers.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSpecialized processor for housenumbers. Analyses common housenumber patterns\nand creates variants for them.\n\"\"\"\nfrom typing import Any, List, cast\nimport re\n\nfrom ...data.place_name import PlaceName\nfrom .generic_mutation import MutationVariantGenerator\n\nRE_NON_DIGIT = re.compile('[^0-9]')\nRE_DIGIT_ALPHA = re.compile(r'(\\d)\\s*([^\\d\\s␣])')\nRE_ALPHA_DIGIT = re.compile(r'([^\\s\\d␣])\\s*(\\d)')\nRE_NAMED_PART = re.compile(r'[a-z]{4}')\n\n# Configuration section\n\n\ndef configure(*_: Any) -> None:\n \"\"\" All behaviour is currently hard-coded.\n \"\"\"\n return None\n\n# Analysis section\n\n\ndef create(normalizer: Any, transliterator: Any, config: None) -> 'HousenumberTokenAnalysis':\n \"\"\" Create a new token analysis instance for this module.\n \"\"\"\n return HousenumberTokenAnalysis(normalizer, transliterator)\n\n\nclass HousenumberTokenAnalysis:\n \"\"\" Detects common housenumber patterns and normalizes them.\n \"\"\"\n def __init__(self, norm: Any, trans: Any) -> None:\n self.norm = norm\n self.trans = trans\n\n self.mutator = MutationVariantGenerator('␣', (' ', ''))\n\n def get_canonical_id(self, name: PlaceName) -> str:\n \"\"\" Return the normalized form of the housenumber.\n \"\"\"\n # shortcut for number-only numbers, which make up 90% of the data.\n if RE_NON_DIGIT.search(name.name) is None:\n return name.name\n\n norm = cast(str, self.trans.transliterate(self.norm.transliterate(name.name)))\n # If there is a significant non-numeric part, use as is.\n if RE_NAMED_PART.search(norm) is None:\n # Otherwise add optional spaces between digits and letters.\n (norm_opt, cnt1) = RE_DIGIT_ALPHA.subn(r'\\1␣\\2', norm)\n (norm_opt, cnt2) = RE_ALPHA_DIGIT.subn(r'\\1␣\\2', norm_opt)\n # Avoid creating too many variants per number.\n if cnt1 + cnt2 <= 4:\n return norm_opt\n\n return norm\n\n def compute_variants(self, norm_name: str) -> List[str]:\n \"\"\" Compute the spelling variants for the given normalized housenumber.\n\n Generates variants for optional spaces (marked with '␣').\n \"\"\"\n return list(self.mutator.generate([norm_name]))\n"
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/postcodes.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSpecialized processor for postcodes. Supports a 'lookup' variant of the\ntoken, which produces variants with optional spaces.\n\"\"\"\nfrom typing import Any, List\n\nfrom ...data.place_name import PlaceName\nfrom .generic_mutation import MutationVariantGenerator\n\n# Configuration section\n\n\ndef configure(*_: Any) -> None:\n \"\"\" All behaviour is currently hard-coded.\n \"\"\"\n return None\n\n# Analysis section\n\n\ndef create(normalizer: Any, transliterator: Any, config: None) -> 'PostcodeTokenAnalysis':\n \"\"\" Create a new token analysis instance for this module.\n \"\"\"\n return PostcodeTokenAnalysis(normalizer, transliterator)\n\n\nclass PostcodeTokenAnalysis:\n \"\"\" Special normalization and variant generation for postcodes.\n\n This analyser must not be used with anything but postcodes as\n it follows some special rules: the canonial ID is the form that\n is used for the output. `compute_variants` then needs to ensure that\n the generated variants once more follow the standard normalization\n and transliteration, so that postcodes are correctly recognised by\n the search algorithm.\n \"\"\"\n def __init__(self, norm: Any, trans: Any) -> None:\n self.norm = norm\n self.trans = trans\n\n self.mutator = MutationVariantGenerator(' ', (' ', ''))\n\n def get_canonical_id(self, name: PlaceName) -> str:\n \"\"\" Return the standard form of the postcode.\n \"\"\"\n return name.name.strip().upper()\n\n def compute_variants(self, norm_name: str) -> List[str]:\n \"\"\" Compute the spelling variants for the given normalized postcode.\n\n Takes the canonical form of the postcode, normalizes it using the\n standard rules and then creates variants of the result where\n all spaces are optional.\n \"\"\"\n # Postcodes follow their own transliteration rules.\n # Make sure at this point, that the terms are normalized in a way\n # that they are searchable with the standard transliteration rules.\n return [self.trans.transliterate(term) for term in\n self.mutator.generate([self.norm.transliterate(norm_name)]) if term]\n"
},
{
"path": "src/nominatim_db/tokenizer/token_analysis/simple_trie.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nSimple dict-based implementation of a trie structure.\n\"\"\"\nfrom typing import TypeVar, Generic, Tuple, Optional, List, Dict\nfrom collections import defaultdict\n\nT = TypeVar('T')\n\n\nclass SimpleTrie(Generic[T]):\n \"\"\" A simple read-only trie structure.\n This structure supports examply one lookup operation,\n which is longest-prefix lookup.\n \"\"\"\n\n def __init__(self, data: Optional[List[Tuple[str, T]]] = None) -> None:\n self._tree: Dict[str, 'SimpleTrie[T]'] = defaultdict(SimpleTrie[T])\n self._value: Optional[T] = None\n self._prefix = ''\n\n if data:\n for key, value in data:\n self._add(key, 0, value)\n\n self._make_compact()\n\n def _add(self, word: str, pos: int, value: T) -> None:\n \"\"\" (Internal) Add a sub-word to the trie.\n The word is added from index 'pos'. If the sub-word to add\n is empty, then the trie saves the given value.\n \"\"\"\n if pos < len(word):\n self._tree[word[pos]]._add(word, pos + 1, value)\n else:\n self._value = value\n\n def _make_compact(self) -> None:\n \"\"\" (Internal) Compress tree where there is exactly one subtree\n and no value.\n\n Compression works recursively starting at the leaf.\n \"\"\"\n for t in self._tree.values():\n t._make_compact()\n\n if len(self._tree) == 1 and self._value is None:\n assert not self._prefix\n for k, v in self._tree.items():\n self._prefix = k + v._prefix\n self._tree = v._tree\n self._value = v._value\n\n def longest_prefix(self, word: str, start: int = 0) -> Tuple[Optional[T], int]:\n \"\"\" Return the longest prefix match for the given word starting at\n the position 'start'.\n\n The function returns a tuple with the value for the longest match and\n the position of the word after the match. If no match was found at\n all, the function returns (None, start).\n \"\"\"\n cur = self\n pos = start\n result: Tuple[Optional[T], int] = None, start\n\n while True:\n if cur._prefix:\n if not word.startswith(cur._prefix, pos):\n return result\n pos += len(cur._prefix)\n\n if cur._value:\n result = cur._value, pos\n\n if pos >= len(word) or word[pos] not in cur._tree:\n return result\n\n cur = cur._tree[word[pos]]\n pos += 1\n"
},
{
"path": "src/nominatim_db/tools/__init__.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nModule with functions for importing, updating Nominatim databases\nas well as general maintenance helpers.\n\"\"\"\n"
},
{
"path": "src/nominatim_db/tools/add_osm_data.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunction to add additional OSM data from a file or the API into the database.\n\"\"\"\nfrom typing import Any, MutableMapping\nfrom pathlib import Path\nimport logging\nimport urllib\n\nfrom ..db.connection import connect\nfrom ..utils.url_utils import get_url\nfrom .exec_utils import run_osm2pgsql\n\nLOG = logging.getLogger()\n\n\ndef _run_osm2pgsql(dsn: str, options: MutableMapping[str, Any]) -> None:\n run_osm2pgsql(options)\n\n # Handle deletions\n with connect(dsn) as conn:\n with conn.cursor() as cur:\n cur.execute('SELECT flush_deleted_places()')\n conn.commit()\n\n\ndef add_data_from_file(dsn: str, fname: str, options: MutableMapping[str, Any]) -> int:\n \"\"\" Adds data from a OSM file to the database. The file may be a normal\n OSM file or a diff file in all formats supported by libosmium.\n \"\"\"\n options['import_file'] = Path(fname)\n options['append'] = True\n _run_osm2pgsql(dsn, options)\n\n # No status update. We don't know where the file came from.\n return 0\n\n\ndef add_osm_object(dsn: str, osm_type: str, osm_id: int, use_main_api: bool,\n options: MutableMapping[str, Any]) -> int:\n \"\"\" Add or update a single OSM object from the latest version of the\n API.\n \"\"\"\n if use_main_api:\n base_url = f'https://www.openstreetmap.org/api/0.6/{osm_type}/{osm_id}'\n if osm_type in ('way', 'relation'):\n base_url += '/full'\n else:\n # use Overpass API\n if osm_type == 'node':\n data = f'node({osm_id});out meta;'\n elif osm_type == 'way':\n data = f'(way({osm_id});>;);out meta;'\n else:\n data = f'(rel(id:{osm_id});>;);out meta;'\n base_url = 'https://overpass-api.de/api/interpreter?' \\\n + urllib.parse.urlencode({'data': data})\n\n options['append'] = True\n options['import_data'] = get_url(base_url).encode('utf-8')\n\n _run_osm2pgsql(dsn, options)\n\n return 0\n"
},
{
"path": "src/nominatim_db/tools/admin.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for database analysis and maintenance.\n\"\"\"\nfrom typing import Optional, Tuple, Any, cast\nimport logging\n\nimport psycopg\nfrom psycopg.types.json import Json\n\nfrom ..typing import DictCursorResult\nfrom ..config import Configuration\nfrom ..db.connection import connect, Cursor, register_hstore\nfrom ..db.sql_preprocessor import SQLPreprocessor\nfrom ..errors import UsageError\nfrom ..tokenizer import factory as tokenizer_factory\nfrom ..data.place_info import PlaceInfo\n\nLOG = logging.getLogger()\n\n\ndef _get_place_info(cursor: Cursor, osm_id: Optional[str],\n place_id: Optional[int]) -> DictCursorResult:\n sql = \"\"\"SELECT place_id, extra.*\n FROM placex, LATERAL placex_indexing_prepare(placex) as extra\n \"\"\"\n\n values: Tuple[Any, ...]\n if osm_id:\n osm_type = osm_id[0].upper()\n if osm_type not in 'NWR' or not osm_id[1:].isdigit():\n LOG.fatal('OSM ID must be of form . Got: %s', osm_id)\n raise UsageError(\"OSM ID parameter badly formatted\")\n\n sql += ' WHERE placex.osm_type = %s AND placex.osm_id = %s'\n values = (osm_type, int(osm_id[1:]))\n elif place_id is not None:\n sql += ' WHERE placex.place_id = %s'\n values = (place_id, )\n else:\n LOG.fatal(\"No OSM object given to index.\")\n raise UsageError(\"OSM object not found\")\n\n cursor.execute(sql + ' LIMIT 1', values)\n\n if cursor.rowcount < 1:\n LOG.fatal(\"OSM object %s not found in database.\", osm_id)\n raise UsageError(\"OSM object not found\")\n\n return cast(DictCursorResult, cursor.fetchone())\n\n\ndef analyse_indexing(config: Configuration, osm_id: Optional[str] = None,\n place_id: Optional[int] = None) -> None:\n \"\"\" Analyse indexing of a single Nominatim object.\n \"\"\"\n with connect(config.get_libpq_dsn()) as conn:\n register_hstore(conn)\n with conn.cursor(row_factory=psycopg.rows.dict_row) as cur:\n place = _get_place_info(cur, osm_id, place_id)\n\n cur.execute(\"update placex set indexed_status = 2 where place_id = %s\",\n (place['place_id'], ))\n\n cur.execute(\"\"\"SET auto_explain.log_min_duration = '0';\n SET auto_explain.log_analyze = 'true';\n SET auto_explain.log_nested_statements = 'true';\n LOAD 'auto_explain';\n SET client_min_messages = LOG;\n SET log_min_messages = FATAL\"\"\")\n\n tokenizer = tokenizer_factory.get_tokenizer_for_db(config)\n\n # Enable printing of messages.\n conn.add_notice_handler(lambda diag: print(diag.message_primary))\n\n with tokenizer.name_analyzer() as analyzer:\n cur.execute(\"\"\"UPDATE placex\n SET indexed_status = 0, address = %s, token_info = %s,\n name = %s, linked_place_id = %s\n WHERE place_id = %s\"\"\",\n (place['address'],\n Json(analyzer.process_place(PlaceInfo(place))),\n place['name'], place['linked_place_id'], place['place_id']))\n\n # we do not want to keep the results\n conn.rollback()\n\n\ndef clean_deleted_relations(config: Configuration, age: str) -> None:\n \"\"\" Clean deleted relations older than a given age\n \"\"\"\n with connect(config.get_libpq_dsn()) as conn:\n with conn.cursor() as cur:\n try:\n cur.execute(\"\"\"SELECT place_force_delete(p.place_id)\n FROM import_polygon_delete d, placex p\n WHERE p.osm_type = d.osm_type AND p.osm_id = d.osm_id\n AND age(p.indexed_date) > %s::interval\"\"\",\n (age, ))\n except psycopg.DataError as exc:\n raise UsageError('Invalid PostgreSQL time interval format') from exc\n conn.commit()\n\n\ndef grant_ro_access(dsn: str, config: Configuration) -> None:\n \"\"\" Grant read-only access to the web user for all Nominatim tables.\n This can be used to grant access to a different user after import.\n \"\"\"\n with connect(dsn) as conn:\n sql = SQLPreprocessor(conn, config)\n sql.run_sql_file(conn, 'grants.sql')\n"
},
{
"path": "src/nominatim_db/tools/check_database.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nCollection of functions that check if the database is complete and functional.\n\"\"\"\nfrom typing import Callable, Optional, Any, Union, Tuple, Mapping, List\nfrom enum import Enum\nfrom textwrap import dedent\n\nfrom ..config import Configuration\nfrom ..db.connection import connect, Connection, \\\n index_exists, table_exists, execute_scalar\nfrom ..db import properties\nfrom ..errors import UsageError\nfrom ..tokenizer import factory as tokenizer_factory\nfrom . import freeze\nfrom ..version import NOMINATIM_VERSION, parse_version\n\nCHECKLIST = []\n\n\nclass CheckState(Enum):\n \"\"\" Possible states of a check. FATAL stops check execution entirely.\n \"\"\"\n OK = 0\n FAIL = 1\n FATAL = 2\n NOT_APPLICABLE = 3\n WARN = 4\n\n\nCheckResult = Union[CheckState, Tuple[CheckState, Mapping[str, Any]]]\nCheckFunc = Callable[[Connection, Configuration], CheckResult]\n\n\ndef _check(hint: Optional[str] = None) -> Callable[[CheckFunc], CheckFunc]:\n \"\"\" Decorator for checks. It adds the function to the list of\n checks to execute and adds the code for printing progress messages.\n \"\"\"\n def decorator(func: CheckFunc) -> CheckFunc:\n title = (func.__doc__ or '').split('\\n', 1)[0].strip()\n\n def run_check(conn: Connection, config: Configuration) -> CheckState:\n print(title, end=' ... ')\n ret = func(conn, config)\n if isinstance(ret, tuple):\n ret, params = ret\n else:\n params = {}\n if ret == CheckState.OK:\n print('\\033[92mOK\\033[0m')\n elif ret == CheckState.WARN:\n print('\\033[93mWARNING\\033[0m')\n if hint:\n print('')\n print(dedent(hint.format(**params)))\n elif ret == CheckState.NOT_APPLICABLE:\n print('not applicable')\n else:\n print('\\x1B[31mFailed\\033[0m')\n if hint:\n print(dedent(hint.format(**params)))\n return ret\n\n CHECKLIST.append(run_check)\n return run_check\n\n return decorator\n\n\nclass _BadConnection:\n\n def __init__(self, msg: str) -> None:\n self.msg = msg\n\n def close(self) -> None:\n \"\"\" Dummy function to provide the implementation.\n \"\"\"\n\n\ndef check_database(config: Configuration) -> int:\n \"\"\" Run a number of checks on the database and return the status.\n \"\"\"\n try:\n conn = connect(config.get_libpq_dsn())\n except UsageError as err:\n conn = _BadConnection(str(err)) # type: ignore[assignment]\n\n overall_result = 0\n for check in CHECKLIST:\n ret = check(conn, config)\n if ret == CheckState.FATAL:\n conn.close()\n return 1\n if ret in (CheckState.FATAL, CheckState.FAIL):\n overall_result = 1\n\n conn.close()\n return overall_result\n\n\ndef _get_indexes(conn: Connection) -> List[str]:\n indexes = ['idx_place_addressline_address_place_id',\n 'idx_placex_rank_search',\n 'idx_placex_rank_address',\n 'idx_placex_parent_place_id',\n 'idx_placex_geometry_reverse_lookupplacenode',\n 'idx_placex_geometry_reverse_lookuppolygon',\n 'idx_placex_geometry_placenode',\n 'idx_osmline_parent_place_id',\n 'idx_osmline_parent_osm_id',\n 'idx_location_postcodes_id',\n 'idx_location_postcodes_postcode'\n ]\n\n # These won't exist if --reverse-only import was used\n if table_exists(conn, 'search_name'):\n indexes.extend(('idx_search_name_nameaddress_vector',\n 'idx_search_name_name_vector',\n 'idx_search_name_centroid',\n 'idx_placex_housenumber',\n 'idx_osmline_parent_osm_id_with_hnr'))\n\n # These won't exist if --no-updates import was used\n if table_exists(conn, 'place'):\n indexes.extend(('idx_location_area_country_place_id',\n 'idx_place_osm_unique',\n 'idx_placex_rank_address_sector',\n 'idx_placex_rank_boundaries_sector'))\n\n return indexes\n\n\n# CHECK FUNCTIONS\n#\n# Functions are executed in the order they appear here.\n\n@_check(hint=\"\"\"\\\n {error}\n\n Hints:\n * Is the database server started?\n * Check the NOMINATIM_DATABASE_DSN variable in your local .env\n * Try connecting to the database with the same settings\n\n Project directory: {config.project_dir}\n Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}\n \"\"\")\ndef check_connection(conn: Any, config: Configuration) -> CheckResult:\n \"\"\" Checking database connection\n \"\"\"\n if isinstance(conn, _BadConnection):\n return CheckState.FATAL, dict(error=conn.msg, config=config)\n\n return CheckState.OK\n\n\n@_check(hint=\"\"\"\\\n Database version ({db_version}) doesn't match Nominatim version ({nom_version})\n\n Hints:\n {instruction}\n\n Project directory: {config.project_dir}\n Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}\n \"\"\")\ndef check_database_version(conn: Connection, config: Configuration) -> CheckResult:\n \"\"\" Checking database_version matches Nominatim software version\n \"\"\"\n\n db_version_str = None\n if not table_exists(conn, 'nominatim_properties'):\n instruction = 'Are you connecting to the correct database?'\n else:\n db_version_str = properties.get_property(conn, 'database_version')\n\n if db_version_str is None:\n instruction = 'Database version not found. Did the import finish?'\n else:\n db_version = parse_version(db_version_str)\n\n if db_version == NOMINATIM_VERSION:\n return CheckState.OK\n\n instruction = (\n \"Run migrations: 'nominatim admin --migrate'\"\n if db_version < NOMINATIM_VERSION\n else 'You need to upgrade the Nominatim software.'\n ) + ' Check the Migration chapter of the Administration Guide.'\n\n return CheckState.FATAL, dict(db_version=db_version_str,\n nom_version=NOMINATIM_VERSION,\n instruction=instruction,\n config=config)\n\n\n@_check(hint=\"\"\"\\\n placex table not found\n\n Hints:\n * Are you connecting to the correct database?\n * Did the import process finish without errors?\n\n Project directory: {config.project_dir}\n Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}\n \"\"\")\ndef check_placex_table(conn: Connection, config: Configuration) -> CheckResult:\n \"\"\" Checking for placex table\n \"\"\"\n if table_exists(conn, 'placex'):\n return CheckState.OK\n\n return CheckState.FATAL, dict(config=config)\n\n\n@_check(hint=\"\"\"placex table has no data. Did the import finish successfully?\"\"\")\ndef check_placex_size(conn: Connection, _: Configuration) -> CheckResult:\n \"\"\" Checking for placex content\n \"\"\"\n cnt = execute_scalar(conn, 'SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')\n\n return CheckState.OK if cnt > 0 else CheckState.FATAL\n\n\n@_check(hint=\"\"\"{msg}\"\"\")\ndef check_tokenizer(_: Connection, config: Configuration) -> CheckResult:\n \"\"\" Checking that tokenizer works\n \"\"\"\n try:\n tokenizer = tokenizer_factory.get_tokenizer_for_db(config)\n except UsageError:\n return CheckState.FAIL, dict(msg=\"\"\"\\\n Cannot load tokenizer. Did the import finish successfully?\"\"\")\n\n result = tokenizer.check_database(config)\n\n if result is None:\n return CheckState.OK\n\n return CheckState.FAIL, dict(msg=result)\n\n\n@_check(hint=\"\"\"\\\n Wikipedia/Wikidata importance tables missing.\n Quality of search results may be degraded. Reverse geocoding is unaffected.\n See https://nominatim.org/release-docs/latest/admin/Import/#wikipediawikidata-rankings\n \"\"\")\ndef check_existance_wikipedia(conn: Connection, _: Configuration) -> CheckResult:\n \"\"\" Checking for wikipedia/wikidata data\n \"\"\"\n if not table_exists(conn, 'search_name') or not table_exists(conn, 'place'):\n return CheckState.NOT_APPLICABLE\n\n if table_exists(conn, 'wikimedia_importance'):\n cnt = execute_scalar(conn, 'SELECT count(*) FROM wikimedia_importance')\n else:\n cnt = execute_scalar(conn, 'SELECT count(*) FROM wikipedia_article')\n\n return CheckState.WARN if cnt == 0 else CheckState.OK\n\n\n@_check(hint=\"\"\"\\\n The indexing didn't finish. {count} entries are not yet indexed.\n\n To index the remaining entries, run: {index_cmd}\n \"\"\")\ndef check_indexing(conn: Connection, _: Configuration) -> CheckResult:\n \"\"\" Checking indexing status\n \"\"\"\n cnt = execute_scalar(conn, 'SELECT count(*) FROM placex WHERE indexed_status > 0')\n\n if cnt == 0:\n return CheckState.OK\n\n if freeze.is_frozen(conn):\n index_cmd = \"\"\"\\\n Database is marked frozen, it cannot be updated.\n Low counts of unindexed places are fine.\"\"\"\n return CheckState.WARN, dict(count=cnt, index_cmd=index_cmd)\n\n if index_exists(conn, 'idx_placex_rank_search'):\n # Likely just an interrupted update.\n index_cmd = 'nominatim index'\n else:\n # Looks like the import process got interrupted.\n index_cmd = 'nominatim import --continue indexing'\n\n return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)\n\n\n@_check(hint=\"\"\"\\\n The following indexes are missing:\n {indexes}\n\n Rerun the index creation with: nominatim import --continue db-postprocess\n \"\"\")\ndef check_database_indexes(conn: Connection, _: Configuration) -> CheckResult:\n \"\"\" Checking that database indexes are complete\n \"\"\"\n missing = []\n for index in _get_indexes(conn):\n if not index_exists(conn, index):\n missing.append(index)\n\n if missing:\n return CheckState.FAIL, dict(indexes='\\n '.join(missing))\n\n return CheckState.OK\n\n\n@_check(hint=\"\"\"\\\n At least one index is invalid. That can happen, e.g. when index creation was\n disrupted and later restarted. You should delete the affected indices\n and recreate them.\n\n Invalid indexes:\n {indexes}\n \"\"\")\ndef check_database_index_valid(conn: Connection, _: Configuration) -> CheckResult:\n \"\"\" Checking that all database indexes are valid\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"\"\" SELECT relname FROM pg_class, pg_index\n WHERE pg_index.indisvalid = false\n AND pg_index.indexrelid = pg_class.oid\"\"\")\n\n broken = [c[0] for c in cur]\n\n if broken:\n return CheckState.FAIL, dict(indexes='\\n '.join(broken))\n\n return CheckState.OK\n\n\n@_check(hint=\"\"\"\\\n {error}\n Run TIGER import again: nominatim add-data --tiger-data \n \"\"\")\ndef check_tiger_table(conn: Connection, config: Configuration) -> CheckResult:\n \"\"\" Checking TIGER external data table.\n \"\"\"\n if not config.get_bool('USE_US_TIGER_DATA'):\n return CheckState.NOT_APPLICABLE\n\n if not table_exists(conn, 'location_property_tiger'):\n return CheckState.FAIL, dict(error='TIGER data table not found.')\n\n if execute_scalar(conn, 'SELECT count(*) FROM location_property_tiger') == 0:\n return CheckState.FAIL, dict(error='TIGER data table is empty.')\n\n return CheckState.OK\n"
},
{
"path": "src/nominatim_db/tools/collect_os_info.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nCollection of host system information including software versions, memory,\nstorage, and database configuration.\n\"\"\"\nimport os\nimport subprocess\nimport sys\nfrom pathlib import Path\nfrom typing import List, Optional, Union\n\nimport psutil\n\nfrom ..config import Configuration\nfrom ..db.connection import connect, server_version_tuple, execute_scalar\nfrom ..version import NOMINATIM_VERSION\n\n\ndef friendly_memory_string(mem: float) -> str:\n \"\"\"Create a user friendly string for the amount of memory specified as mem\"\"\"\n mem_magnitude = (\"bytes\", \"KB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\", \"ZB\", \"YB\")\n mag = 0\n # determine order of magnitude\n while mem > 1000:\n mem /= 1000\n mag += 1\n\n return f\"{mem:.1f} {mem_magnitude[mag]}\"\n\n\ndef run_command(cmd: Union[str, List[str]]) -> str:\n \"\"\"Runs a command using the shell and returns the output from stdout\"\"\"\n try:\n if sys.version_info < (3, 7):\n cap_out = subprocess.run(cmd, stdout=subprocess.PIPE, check=False)\n else:\n cap_out = subprocess.run(cmd, capture_output=True, check=False)\n return cap_out.stdout.decode(\"utf-8\")\n except FileNotFoundError:\n # non-Linux system should end up here\n return f\"Unknown (unable to find the '{cmd}' command)\"\n\n\ndef os_name_info() -> str:\n \"\"\"Obtain Operating System Name (and possibly the version)\"\"\"\n os_info = None\n # man page os-release(5) details meaning of the fields\n if Path(\"/etc/os-release\").is_file():\n os_info = from_file_find_line_portion(\n \"/etc/os-release\", \"PRETTY_NAME\", \"=\")\n # alternative location\n elif Path(\"/usr/lib/os-release\").is_file():\n os_info = from_file_find_line_portion(\n \"/usr/lib/os-release\", \"PRETTY_NAME\", \"=\"\n )\n\n # fallback on Python's os name\n if os_info is None or os_info == \"\":\n os_info = os.name\n\n # if the above is insufficient, take a look at neofetch's approach to OS detection\n return os_info\n\n\n# Note: Intended to be used on informational files like /proc\ndef from_file_find_line_portion(\n filename: str, start: str, sep: str, fieldnum: int = 1\n) -> Optional[str]:\n \"\"\"open filename, finds the line starting with the 'start' string.\n Splits the line using separator and returns a \"fieldnum\" from the split.\"\"\"\n with open(filename, encoding='utf8') as file:\n result = \"\"\n for line in file:\n if line.startswith(start):\n result = line.split(sep)[fieldnum].strip()\n return result\n\n\ndef get_postgresql_config(version: int) -> str:\n \"\"\"Retrieve postgres configuration file\"\"\"\n try:\n with open(f\"/etc/postgresql/{version}/main/postgresql.conf\", encoding='utf8') as file:\n db_config = file.read()\n file.close()\n return db_config\n except IOError:\n return f\"**Could not read '/etc/postgresql/{version}/main/postgresql.conf'**\"\n\n\ndef report_system_information(config: Configuration) -> None:\n \"\"\"Generate a report about the host system including software versions, memory,\n storage, and database configuration.\"\"\"\n\n with connect(config.get_libpq_dsn(), dbname='postgres') as conn:\n postgresql_ver: str = '.'.join(map(str, server_version_tuple(conn)))\n\n with conn.cursor() as cur:\n cur.execute(\"SELECT datname FROM pg_catalog.pg_database WHERE datname=%s\",\n (config.get_database_params()['dbname'], ))\n nominatim_db_exists = cur.rowcount > 0\n\n if nominatim_db_exists:\n with connect(config.get_libpq_dsn()) as conn:\n postgis_ver: str = execute_scalar(conn, 'SELECT postgis_lib_version()')\n else:\n postgis_ver = \"Unable to connect to database\"\n\n postgresql_config: str = get_postgresql_config(int(float(postgresql_ver)))\n\n # Note: psutil.disk_partitions() is similar to run_command(\"lsblk\")\n\n # Note: run_command(\"systemd-detect-virt\") only works on Linux, on other OSes\n # should give a message: \"Unknown (unable to find the 'systemd-detect-virt' command)\"\n\n # Generates the Markdown report.\n\n report = f\"\"\"\n **Instructions**\n Use this information in your issue report at https://github.com/osm-search/Nominatim/issues\n Redirect the output to a file:\n $ ./collect_os_info.py > report.md\n\n\n **Software Environment:**\n - Python version: {sys.version}\n - Nominatim version: {NOMINATIM_VERSION!s}\n - PostgreSQL version: {postgresql_ver}\n - PostGIS version: {postgis_ver}\n - OS: {os_name_info()}\n\n\n **Hardware Configuration:**\n - RAM: {friendly_memory_string(psutil.virtual_memory().total)}\n - number of CPUs: {psutil.cpu_count(logical=False)}\n - bare metal/AWS/other cloud service (per systemd-detect-virt(1)):\n {run_command(\"systemd-detect-virt\")}\n - type and size of disks:\n **`df -h` - df - report file system disk space usage: **\n ```\n {run_command([\"df\", \"-h\"])}\n ```\n\n **lsblk - list block devices: **\n ```\n {run_command(\"lsblk\")}\n ```\n\n\n **Postgresql Configuration:**\n ```\n {postgresql_config}\n ```\n **Notes**\n Please add any notes about anything above anything above that is incorrect.\n\"\"\"\n print(report)\n"
},
{
"path": "src/nominatim_db/tools/convert_sqlite.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nExporting a Nominatim database to SQlite.\n\"\"\"\nfrom typing import Set, Any, Optional, Union\nimport datetime as dt\nimport logging\nfrom pathlib import Path\n\nimport sqlalchemy as sa\n\nimport nominatim_api as napi\nfrom nominatim_api.search.query_analyzer_factory import make_query_analyzer\nfrom nominatim_api.typing import SaSelect, SaRow\nfrom nominatim_api.sql.sqlalchemy_types import Geometry, IntArray\n\nLOG = logging.getLogger()\n\n\nasync def convert(project_dir: Optional[Union[str, Path]],\n outfile: Path, options: Set[str]) -> None:\n \"\"\" Export an existing database to sqlite. The resulting database\n will be usable against the Python frontend of Nominatim.\n \"\"\"\n api = napi.NominatimAPIAsync(project_dir)\n\n try:\n outapi = napi.NominatimAPIAsync(project_dir,\n {'NOMINATIM_DATABASE_DSN': f\"sqlite:dbname={outfile}\",\n 'NOMINATIM_DATABASE_RW': '1'})\n\n try:\n async with api.begin() as src, outapi.begin() as dest:\n writer = SqliteWriter(src, dest, options)\n await writer.write()\n finally:\n await outapi.close()\n finally:\n await api.close()\n\n\nclass SqliteWriter:\n \"\"\" Worker class which creates a new SQLite database.\n \"\"\"\n\n def __init__(self, src: napi.SearchConnection,\n dest: napi.SearchConnection, options: Set[str]) -> None:\n self.src = src\n self.dest = dest\n self.options = options\n\n async def write(self) -> None:\n \"\"\" Create the database structure and copy the data from\n the source database to the destination.\n \"\"\"\n LOG.warning('Setting up spatialite')\n await self.dest.execute(sa.select(sa.func.InitSpatialMetaData(True, 'WGS84')))\n\n await self.create_tables()\n await self.copy_data()\n if 'search' in self.options:\n await self.create_word_table()\n await self.create_indexes()\n\n async def create_tables(self) -> None:\n \"\"\" Set up the database tables.\n \"\"\"\n LOG.warning('Setting up tables')\n if 'search' not in self.options:\n self.dest.t.meta.remove(self.dest.t.search_name)\n else:\n await self.create_class_tables()\n\n await self.dest.connection.run_sync(self.dest.t.meta.create_all)\n\n # Convert all Geometry columns to Spatialite geometries\n for table in self.dest.t.meta.sorted_tables:\n for col in table.c:\n if isinstance(col.type, Geometry):\n await self.dest.execute(sa.select(\n sa.func.RecoverGeometryColumn(table.name, col.name, 4326,\n col.type.subtype.upper(), 'XY')))\n\n async def create_class_tables(self) -> None:\n \"\"\" Set up the table that serve class/type-specific geometries.\n \"\"\"\n sql = sa.text(\"\"\"SELECT tablename FROM pg_tables\n WHERE tablename LIKE 'place_classtype_%'\"\"\")\n for res in await self.src.execute(sql):\n for db in (self.src, self.dest):\n sa.Table(res[0], db.t.meta,\n sa.Column('place_id', sa.BigInteger),\n sa.Column('centroid', Geometry))\n\n async def create_word_table(self) -> None:\n \"\"\" Create the word table.\n This table needs the property information to determine the\n correct format. Therefore needs to be done after all other\n data has been copied.\n \"\"\"\n await make_query_analyzer(self.src)\n await make_query_analyzer(self.dest)\n src = self.src.t.meta.tables['word']\n dest = self.dest.t.meta.tables['word']\n\n await self.dest.connection.run_sync(dest.create)\n\n LOG.warning(\"Copying word table\")\n async_result = await self.src.connection.stream(sa.select(src))\n\n async for partition in async_result.partitions(10000):\n data = [{k: getattr(r, k) for k in r._fields} for r in partition]\n await self.dest.execute(dest.insert(), data)\n\n await self.dest.connection.run_sync(sa.Index('idx_word_woken', dest.c.word_token).create)\n\n async def copy_data(self) -> None:\n \"\"\" Copy data for all registered tables.\n \"\"\"\n def _getfield(row: SaRow, key: str) -> Any:\n value = getattr(row, key)\n if isinstance(value, dt.datetime):\n if value.tzinfo is not None:\n value = value.astimezone(dt.timezone.utc)\n return value\n\n for table in self.dest.t.meta.sorted_tables:\n LOG.warning(\"Copying '%s'\", table.name)\n async_result = await self.src.connection.stream(self.select_from(table.name))\n\n async for partition in async_result.partitions(10000):\n data = [{('class_' if k == 'class' else k): _getfield(r, k)\n for k in r._fields}\n for r in partition]\n await self.dest.execute(table.insert(), data)\n\n # Set up a minimal copy of pg_tables used to look up the class tables later.\n pg_tables = sa.Table('pg_tables', self.dest.t.meta,\n sa.Column('schemaname', sa.Text, default='public'),\n sa.Column('tablename', sa.Text))\n await self.dest.connection.run_sync(pg_tables.create)\n data = [{'tablename': t} for t in self.dest.t.meta.tables]\n await self.dest.execute(pg_tables.insert().values(data))\n\n async def create_indexes(self) -> None:\n \"\"\" Add indexes necessary for the frontend.\n \"\"\"\n # reverse place node lookup needs an extra table to simulate a\n # partial index with adaptive buffering.\n await self.dest.execute(sa.text(\n \"\"\" CREATE TABLE placex_place_node_areas AS\n SELECT place_id, ST_Expand(geometry,\n 14.0 * exp(-0.2 * rank_search) - 0.03) as geometry\n FROM placex\n WHERE rank_address between 5 and 25\n and osm_type = 'N'\n and linked_place_id is NULL \"\"\"))\n await self.dest.execute(sa.select(\n sa.func.RecoverGeometryColumn('placex_place_node_areas', 'geometry',\n 4326, 'GEOMETRY', 'XY')))\n await self.dest.execute(sa.select(sa.func.CreateSpatialIndex(\n 'placex_place_node_areas', 'geometry')))\n\n # Remaining indexes.\n await self.create_spatial_index('country_grid', 'geometry')\n await self.create_spatial_index('placex', 'geometry')\n await self.create_spatial_index('osmline', 'linegeo')\n await self.create_spatial_index('tiger', 'linegeo')\n await self.create_index('placex', 'place_id')\n await self.create_index('placex', 'parent_place_id')\n await self.create_index('placex', 'rank_address')\n await self.create_index('addressline', 'place_id')\n await self.create_index('postcode', 'place_id')\n await self.create_index('osmline', 'place_id')\n await self.create_index('tiger', 'place_id')\n\n if 'search' in self.options:\n await self.create_spatial_index('postcode', 'geometry')\n await self.create_spatial_index('search_name', 'centroid')\n await self.create_index('search_name', 'place_id')\n await self.create_index('osmline', 'parent_place_id')\n await self.create_index('tiger', 'parent_place_id')\n await self.create_search_index()\n\n for t in self.dest.t.meta.tables:\n if t.startswith('place_classtype_'):\n await self.dest.execute(sa.select(\n sa.func.CreateSpatialIndex(t, 'centroid')))\n\n async def create_spatial_index(self, table: str, column: str) -> None:\n \"\"\" Create a spatial index on the given table and column.\n \"\"\"\n await self.dest.execute(sa.select(\n sa.func.CreateSpatialIndex(getattr(self.dest.t, table).name, column)))\n\n async def create_index(self, table_name: str, column: str) -> None:\n \"\"\" Create a simple index on the given table and column.\n \"\"\"\n table = getattr(self.dest.t, table_name)\n await self.dest.connection.run_sync(\n sa.Index(f\"idx_{table}_{column}\", getattr(table.c, column)).create)\n\n async def create_search_index(self) -> None:\n \"\"\" Create the tables and indexes needed for word lookup.\n \"\"\"\n LOG.warning(\"Creating reverse search table\")\n rsn = sa.Table('reverse_search_name', self.dest.t.meta,\n sa.Column('word', sa.Integer()),\n sa.Column('column', sa.Text()),\n sa.Column('places', IntArray))\n await self.dest.connection.run_sync(rsn.create)\n\n tsrc = self.src.t.search_name\n for column in ('name_vector', 'nameaddress_vector'):\n sql = sa.select(sa.func.unnest(getattr(tsrc.c, column)).label('word'),\n sa.func.ArrayAgg(tsrc.c.place_id).label('places'))\\\n .group_by('word')\n\n async_result = await self.src.connection.stream(sql)\n async for partition in async_result.partitions(100):\n data = []\n for row in partition:\n row.places.sort()\n data.append({'word': row.word,\n 'column': column,\n 'places': row.places})\n await self.dest.execute(rsn.insert(), data)\n\n await self.dest.connection.run_sync(\n sa.Index('idx_reverse_search_name_word', rsn.c.word).create)\n\n def select_from(self, table: str) -> SaSelect:\n \"\"\" Create the SQL statement to select the source columns and rows.\n \"\"\"\n columns = self.src.t.meta.tables[table].c\n\n if table == 'placex':\n # SQLite struggles with Geometries that are larger than 5MB,\n # so simplify those.\n return sa.select(*(c for c in columns if not isinstance(c.type, Geometry)),\n sa.func.ST_AsText(columns.centroid).label('centroid'),\n sa.func.ST_AsText(\n sa.case((sa.func.ST_MemSize(columns.geometry) < 5000000,\n columns.geometry),\n else_=sa.func.ST_SimplifyPreserveTopology(\n columns.geometry, 0.0001)\n )).label('geometry'))\n\n sql = sa.select(*(sa.func.ST_AsText(c).label(c.name)\n if isinstance(c.type, Geometry) else c for c in columns))\n\n return sql\n"
},
{
"path": "src/nominatim_db/tools/database_import.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for setting up and importing a new Nominatim database.\n\"\"\"\nfrom typing import Tuple, Optional, Union, Sequence, MutableMapping, Any\nimport logging\nimport os\nimport subprocess\nimport asyncio\nfrom pathlib import Path\n\nimport psutil\nimport psycopg\nfrom psycopg import sql as pysql\n\nfrom ..errors import UsageError\nfrom ..config import Configuration\nfrom ..db.connection import connect, get_pg_env, Connection, server_version_tuple, \\\n postgis_version_tuple, drop_tables, table_exists, execute_scalar\nfrom ..db.sql_preprocessor import SQLPreprocessor\nfrom ..db.query_pool import QueryPool\nfrom .exec_utils import run_osm2pgsql\nfrom ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION\n\nLOG = logging.getLogger()\n\n\ndef _require_version(module: str, actual: Tuple[int, int], expected: Tuple[int, int]) -> None:\n \"\"\" Compares the version for the given module and raises an exception\n if the actual version is too old.\n \"\"\"\n if actual < expected:\n LOG.fatal('Minimum supported version of %s is %d.%d. '\n 'Found version %d.%d.',\n module, expected[0], expected[1], actual[0], actual[1])\n raise UsageError(f'{module} is too old.')\n\n\ndef _require_loaded(extension_name: str, conn: Connection) -> None:\n \"\"\" Check that the given extension is loaded. \"\"\"\n with conn.cursor() as cur:\n cur.execute('SELECT * FROM pg_extension WHERE extname = %s', (extension_name, ))\n if cur.rowcount <= 0:\n LOG.fatal('Required module %s is not loaded.', extension_name)\n raise UsageError(f'{extension_name} is not loaded.')\n\n\ndef check_existing_database_plugins(dsn: str) -> None:\n \"\"\" Check that the database has the required plugins installed.\"\"\"\n with connect(dsn) as conn:\n _require_version('PostgreSQL server',\n server_version_tuple(conn),\n POSTGRESQL_REQUIRED_VERSION)\n _require_version('PostGIS',\n postgis_version_tuple(conn),\n POSTGIS_REQUIRED_VERSION)\n _require_loaded('hstore', conn)\n\n\ndef setup_database_skeleton(dsn: str, rouser: Optional[str] = None) -> None:\n \"\"\" Create a new database for Nominatim and populate it with the\n essential extensions.\n\n The function fails when the database already exists or Postgresql or\n PostGIS versions are too old.\n\n Uses `createdb` to create the database.\n\n If 'rouser' is given, then the function also checks that the user\n with that given name exists.\n\n Requires superuser rights by the caller.\n \"\"\"\n proc = subprocess.run(['createdb'], env=get_pg_env(dsn), check=False)\n\n if proc.returncode != 0:\n raise UsageError('Creating new database failed.')\n\n with connect(dsn) as conn:\n _require_version('PostgreSQL server',\n server_version_tuple(conn),\n POSTGRESQL_REQUIRED_VERSION)\n\n if rouser is not None:\n cnt = execute_scalar(conn, 'SELECT count(*) FROM pg_user where usename = %s',\n (rouser, ))\n if cnt == 0:\n LOG.fatal(\"Web user '%s' does not exist. Create it with:\\n\"\n \"\\n createuser %s\", rouser, rouser)\n raise UsageError('Missing read-only user.')\n\n # Create extensions.\n with conn.cursor() as cur:\n cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')\n cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')\n cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster')\n\n conn.commit()\n\n _require_version('PostGIS',\n postgis_version_tuple(conn),\n POSTGIS_REQUIRED_VERSION)\n\n\ndef import_osm_data(osm_files: Union[Path, Sequence[Path]],\n options: MutableMapping[str, Any],\n drop: bool = False, ignore_errors: bool = False) -> None:\n \"\"\" Import the given OSM files. 'options' contains the list of\n default settings for osm2pgsql.\n \"\"\"\n options['import_file'] = osm_files\n options['append'] = False\n options['threads'] = 1\n\n if not options['flatnode_file'] and options['osm2pgsql_cache'] == 0:\n # Make some educated guesses about cache size based on the size\n # of the import file and the available memory.\n mem = psutil.virtual_memory()\n fsize = 0\n if isinstance(osm_files, list):\n for fname in osm_files:\n fsize += os.stat(str(fname)).st_size\n else:\n fsize = os.stat(str(osm_files)).st_size\n options['osm2pgsql_cache'] = int(min((mem.available + getattr(mem, 'cached', 0)) * 0.75,\n fsize * 2) / 1024 / 1024) + 1\n\n run_osm2pgsql(options)\n\n with connect(options['dsn']) as conn:\n if not ignore_errors:\n with conn.cursor() as cur:\n cur.execute('SELECT true FROM place LIMIT 1')\n if cur.rowcount == 0:\n raise UsageError('No data imported by osm2pgsql.')\n\n if drop:\n drop_tables(conn, 'planet_osm_nodes')\n conn.commit()\n\n if drop and options['flatnode_file']:\n Path(options['flatnode_file']).unlink()\n\n\ndef create_tables(conn: Connection, config: Configuration, reverse_only: bool = False) -> None:\n \"\"\" Create the set of basic tables.\n When `reverse_only` is True, then the main table for searching will\n be skipped and only reverse search is possible.\n \"\"\"\n SQLPreprocessor(conn, config).run_sql_file(conn, 'tables.sql',\n create_reverse_only=reverse_only)\n\n # reinitiate the preprocessor to get all the newly created tables\n SQLPreprocessor(conn, config).run_sql_file(conn, 'grants.sql')\n\n\ndef create_table_triggers(conn: Connection, config: Configuration) -> None:\n \"\"\" Create the triggers for the tables. The trigger functions must already\n have been imported with refresh.create_functions().\n \"\"\"\n sql = SQLPreprocessor(conn, config)\n sql.run_sql_file(conn, 'table-triggers.sql')\n\n\ndef create_partition_tables(conn: Connection, config: Configuration) -> None:\n \"\"\" Create tables that have explicit partitioning.\n \"\"\"\n sql = SQLPreprocessor(conn, config)\n sql.run_sql_file(conn, 'partition-tables.src.sql')\n\n\ndef truncate_data_tables(conn: Connection) -> None:\n \"\"\" Truncate all data tables to prepare for a fresh load.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute('TRUNCATE placex')\n cur.execute('TRUNCATE place_addressline')\n cur.execute('TRUNCATE location_area')\n cur.execute('TRUNCATE location_area_country')\n cur.execute('TRUNCATE location_property_tiger')\n cur.execute('TRUNCATE location_property_osmline')\n cur.execute('TRUNCATE location_postcodes')\n if table_exists(conn, 'search_name'):\n cur.execute('TRUNCATE search_name')\n cur.execute('DROP SEQUENCE IF EXISTS seq_place')\n cur.execute('CREATE SEQUENCE seq_place start 100000')\n\n cur.execute(\"\"\"SELECT tablename FROM pg_tables\n WHERE tablename LIKE 'location_road_%'\"\"\")\n\n for table in [r[0] for r in list(cur)]:\n cur.execute(pysql.SQL('TRUNCATE {}').format(pysql.Identifier(table)))\n\n conn.commit()\n\n\n_COPY_COLUMNS = pysql.SQL(',').join(map(pysql.Identifier,\n ('osm_type', 'osm_id', 'class', 'type',\n 'name', 'admin_level', 'address',\n 'extratags', 'geometry')))\n\n\nasync def load_data(dsn: str, threads: int) -> None:\n \"\"\" Copy data into the word and placex table.\n \"\"\"\n placex_threads = max(1, threads - 1)\n\n progress = asyncio.create_task(_progress_print())\n\n async with QueryPool(dsn, placex_threads + 1) as pool:\n # Copy data from place to placex in chunks.\n for imod in range(placex_threads):\n await pool.put_query(\n pysql.SQL(\"\"\"INSERT INTO placex ({columns})\n SELECT {columns} FROM place\n WHERE osm_id % {total} = {mod}\n \"\"\").format(columns=_COPY_COLUMNS,\n total=pysql.Literal(placex_threads),\n mod=pysql.Literal(imod)), None)\n\n # Interpolations need to be copied separately\n await pool.put_query(\"\"\"\n INSERT INTO location_property_osmline (osm_id, type, address, linegeo)\n SELECT osm_id, type, address, geometry\n FROM place_interpolation\n \"\"\", None)\n\n progress.cancel()\n\n async with await psycopg.AsyncConnection.connect(dsn) as aconn:\n await aconn.execute('ANALYSE')\n\n\nasync def _progress_print() -> None:\n while True:\n try:\n await asyncio.sleep(1)\n except asyncio.CancelledError:\n print('', flush=True)\n break\n print('.', end='', flush=True)\n\n\nasync def create_search_indices(conn: Connection, config: Configuration,\n drop: bool = False, threads: int = 1) -> None:\n \"\"\" Create tables that have explicit partitioning.\n \"\"\"\n\n # If index creation failed and left an index invalid, they need to be\n # cleaned out first, so that the script recreates them.\n with conn.cursor() as cur:\n cur.execute(\"\"\"SELECT relname FROM pg_class, pg_index\n WHERE pg_index.indisvalid = false\n AND pg_index.indexrelid = pg_class.oid\"\"\")\n bad_indices = [row[0] for row in list(cur)]\n for idx in bad_indices:\n LOG.info(\"Drop invalid index %s.\", idx)\n cur.execute(pysql.SQL('DROP INDEX {}').format(pysql.Identifier(idx)))\n conn.commit()\n\n sql = SQLPreprocessor(conn, config)\n\n await sql.run_parallel_sql_file(config.get_libpq_dsn(),\n 'indices.sql', min(8, threads), drop=drop)\n"
},
{
"path": "src/nominatim_db/tools/exec_utils.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelper functions for executing external programs.\n\"\"\"\nfrom typing import Any, Mapping, List, Optional\nimport logging\nimport os\nimport re\nimport subprocess\nimport shutil\n\nfrom ..db.connection import get_pg_env\nfrom ..errors import UsageError\nfrom ..version import OSM2PGSQL_REQUIRED_VERSION\n\nLOG = logging.getLogger()\n\n\ndef run_osm2pgsql(options: Mapping[str, Any]) -> None:\n \"\"\" Run osm2pgsql with the given options.\n \"\"\"\n _check_osm2pgsql_version(options['osm2pgsql'])\n\n env = get_pg_env(options['dsn'])\n\n cmd = [_find_osm2pgsql_cmd(options['osm2pgsql']),\n '--append' if options['append'] else '--create',\n '--slim',\n '--log-progress', 'true',\n '--number-processes', '1' if options['append'] else str(options['threads']),\n '--cache', str(options['osm2pgsql_cache']),\n '--style', str(options['osm2pgsql_style'])\n ]\n\n env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),\n os.environ.get('LUA_PATH', ';')))\n env['THEMEPARK_PATH'] = str(options['osm2pgsql_style_path'] / 'themes')\n if 'THEMEPARK_PATH' in os.environ:\n env['THEMEPARK_PATH'] += ':' + os.environ['THEMEPARK_PATH']\n cmd.extend(('--output', 'flex'))\n\n for flavour in ('data', 'index'):\n if options['tablespaces'][f\"main_{flavour}\"]:\n env[f\"NOMINATIM_TABLESPACE_PLACE_{flavour.upper()}\"] = \\\n options['tablespaces'][f\"main_{flavour}\"]\n\n if options['flatnode_file']:\n cmd.extend(('--flat-nodes', options['flatnode_file']))\n\n cmd.extend(_mk_tablespace_options('slim', options))\n\n if options.get('disable_jit', False):\n env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'\n\n if 'import_data' in options:\n cmd.extend(('-r', 'xml', '-'))\n elif isinstance(options['import_file'], list):\n for fname in options['import_file']:\n cmd.append(str(fname))\n else:\n cmd.append(str(options['import_file']))\n\n subprocess.run(cmd, cwd=options.get('cwd', '.'),\n input=options.get('import_data'),\n env=env, check=True)\n\n\ndef _mk_tablespace_options(ttype: str, options: Mapping[str, Any]) -> List[str]:\n cmds: List[str] = []\n for flavour in ('data', 'index'):\n if options['tablespaces'][f\"{ttype}_{flavour}\"]:\n cmds.extend((f\"--tablespace-{ttype}-{flavour}\",\n options['tablespaces'][f\"{ttype}_{flavour}\"]))\n\n return cmds\n\n\ndef _find_osm2pgsql_cmd(cmdline: Optional[str]) -> str:\n if cmdline:\n return cmdline\n\n in_path = shutil.which('osm2pgsql')\n if in_path is None:\n raise UsageError('osm2pgsql executable not found. Please install osm2pgsql first.')\n\n return str(in_path)\n\n\ndef _check_osm2pgsql_version(cmdline: Optional[str]) -> None:\n cmd = [_find_osm2pgsql_cmd(cmdline), '--version']\n\n result = subprocess.run(cmd, capture_output=True, check=True)\n\n if not result.stderr:\n raise UsageError(\"osm2pgsql does not print version information.\")\n\n verinfo = result.stderr.decode('UTF-8')\n\n match = re.search(r'osm2pgsql version (\\d+)\\.(\\d+)', verinfo)\n if match is None:\n raise UsageError(f\"No version information found in output: {verinfo}\")\n\n if (int(match[1]), int(match[2])) < OSM2PGSQL_REQUIRED_VERSION:\n raise UsageError(f\"osm2pgsql is too old. Found version {match[1]}.{match[2]}. \"\n f\"Need at least version {'.'.join(map(str, OSM2PGSQL_REQUIRED_VERSION))}.\")\n"
},
{
"path": "src/nominatim_db/tools/freeze.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for removing unnecessary data from the database.\n\"\"\"\nfrom typing import Optional\nfrom pathlib import Path\n\nfrom psycopg import sql as pysql\n\nfrom ..db.connection import Connection, drop_tables, table_exists\n\nUPDATE_TABLES = [\n 'address_levels',\n 'gb_postcode',\n 'import_osmosis_log',\n 'location_area%',\n 'location_road%',\n 'place',\n 'place_associated_street',\n 'place_entrance',\n 'place_postcode',\n 'planet_osm_%',\n 'search_name_%',\n 'us_postcode',\n 'wikipedia_%'\n]\n\n\ndef drop_update_tables(conn: Connection) -> None:\n \"\"\" Drop all tables only necessary for updating the database from\n OSM replication data.\n \"\"\"\n parts = (pysql.SQL(\"(tablename LIKE {})\").format(pysql.Literal(t)) for t in UPDATE_TABLES)\n\n with conn.cursor() as cur:\n cur.execute(pysql.SQL(\"SELECT tablename FROM pg_tables WHERE \")\n + pysql.SQL(' or ').join(parts))\n tables = [r[0] for r in cur]\n\n drop_tables(conn, *tables, cascade=True)\n conn.commit()\n\n\ndef drop_flatnode_file(fpath: Optional[Path]) -> None:\n \"\"\" Remove the flatnode file if it exists.\n \"\"\"\n if fpath and fpath.exists():\n fpath.unlink()\n\n\ndef is_frozen(conn: Connection) -> bool:\n \"\"\" Returns true if database is in a frozen state\n \"\"\"\n return table_exists(conn, 'place') is False\n"
},
{
"path": "src/nominatim_db/tools/migration.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for database migration to newer software versions.\n\"\"\"\nfrom typing import List, Tuple, Callable, Any\nimport logging\n\nfrom ..errors import UsageError\nfrom ..config import Configuration\nfrom ..db import properties\nfrom ..db.connection import connect, Connection, \\\n table_exists, register_hstore, table_has_column\nfrom ..db.sql_preprocessor import SQLPreprocessor\nfrom ..version import NominatimVersion, NOMINATIM_VERSION, parse_version\nfrom ..tokenizer import factory as tokenizer_factory\nfrom ..data.country_info import create_country_names, setup_country_config\nfrom .freeze import is_frozen\nfrom . import refresh\n\nLOG = logging.getLogger()\n\n_MIGRATION_FUNCTIONS: List[Tuple[NominatimVersion, Callable[..., None]]] = []\n\n\ndef migrate(config: Configuration, paths: Any) -> int:\n \"\"\" Check for the current database version and execute migrations,\n if necessary.\n \"\"\"\n with connect(config.get_libpq_dsn()) as conn:\n register_hstore(conn)\n if table_exists(conn, 'nominatim_properties'):\n db_version_str = properties.get_property(conn, 'database_version')\n else:\n db_version_str = None\n\n if db_version_str is not None:\n db_version = parse_version(db_version_str)\n else:\n db_version = None\n\n if db_version is None or db_version < (4, 3, 0, 0):\n LOG.fatal('Your database version is older than 4.3. '\n 'Direct migration is not possible.\\n'\n 'You should strongly consider a reimport. If that is not possible\\n'\n 'please upgrade to 4.3 first and then to the newest version.')\n raise UsageError('Migration not possible.')\n\n if db_version == NOMINATIM_VERSION:\n LOG.warning(\"Database already at latest version (%s)\", db_version_str)\n return 0\n\n LOG.info(\"Detected database version: %s\", db_version_str)\n\n for version, func in _MIGRATION_FUNCTIONS:\n if db_version < version:\n title = func.__doc__ or ''\n LOG.warning(\"Running: %s (%s)\", title.split('\\n', 1)[0], version)\n kwargs = dict(conn=conn, config=config, paths=paths)\n func(**kwargs)\n conn.commit()\n\n LOG.warning('Updating SQL functions.')\n refresh.create_functions(conn, config)\n tokenizer = tokenizer_factory.get_tokenizer_for_db(config)\n tokenizer.update_sql_functions(config)\n\n properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))\n\n conn.commit()\n\n return 0\n\n\ndef _migration(major: int, minor: int, patch: int = 0,\n dbpatch: int = 0) -> Callable[[Callable[..., None]], Callable[..., None]]:\n \"\"\" Decorator for a single migration step. The parameters describe the\n version after which the migration is applicable, i.e before changing\n from the given version to the next, the migration is required.\n\n All migrations are run in the order in which they are defined in this\n file. Do not run global SQL scripts for migrations as you cannot be sure\n that these scripts do the same in later versions.\n\n Functions will always be reimported in full at the end of the migration\n process, so the migration functions may leave a temporary state behind\n there.\n \"\"\"\n def decorator(func: Callable[..., None]) -> Callable[..., None]:\n version = NominatimVersion(major, minor, patch, dbpatch)\n _MIGRATION_FUNCTIONS.append((version, func))\n return func\n\n return decorator\n\n\n@_migration(4, 4, 99, 0)\ndef create_postcode_area_lookup_index(conn: Connection, **_: Any) -> None:\n \"\"\" Create index needed for looking up postcode areas from postocde points.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"\"\"CREATE INDEX IF NOT EXISTS idx_placex_postcode_areas\n ON placex USING BTREE (country_code, postcode)\n WHERE osm_type = 'R' AND class = 'boundary' AND type = 'postal_code'\n \"\"\")\n\n\n@_migration(4, 4, 99, 1)\ndef create_postcode_parent_index(conn: Connection, **_: Any) -> None:\n \"\"\" Create index needed for updating postcodes when a parent changes.\n \"\"\"\n if table_exists(conn, 'planet_osm_ways'):\n with conn.cursor() as cur:\n cur.execute(\"\"\"CREATE INDEX IF NOT EXISTS\n idx_location_postcode_parent_place_id\n ON location_postcode USING BTREE (parent_place_id)\"\"\")\n\n\n@_migration(5, 1, 99, 0)\ndef create_placex_entrance_table(conn: Connection, config: Configuration, **_: Any) -> None:\n \"\"\" Add the placex_entrance table to store linked-up entrance nodes\n \"\"\"\n if not table_exists(conn, 'placex_entrance'):\n sqlp = SQLPreprocessor(conn, config)\n sqlp.run_string(conn, \"\"\"\n -- Table to store location of entrance nodes\n CREATE TABLE placex_entrance (\n place_id BIGINT NOT NULL,\n osm_id BIGINT NOT NULL,\n type TEXT NOT NULL,\n location GEOMETRY(Point, 4326) NOT NULL,\n extratags HSTORE\n );\n CREATE UNIQUE INDEX idx_placex_entrance_place_id_osm_id ON placex_entrance\n USING BTREE (place_id, osm_id) {{db.tablespace.search_index}};\n GRANT SELECT ON placex_entrance TO \"{{config.DATABASE_WEBUSER}}\" ;\n \"\"\")\n\n\n@_migration(5, 1, 99, 1)\ndef create_place_entrance_table(conn: Connection, config: Configuration, **_: Any) -> None:\n \"\"\" Add the place_entrance table to store incoming entrance nodes\n \"\"\"\n if not table_exists(conn, 'place_entrance'):\n with conn.cursor() as cur:\n cur.execute(\"\"\"\n -- Table to store location of entrance nodes\n CREATE TABLE place_entrance (\n osm_id BIGINT NOT NULL,\n type TEXT NOT NULL,\n extratags HSTORE,\n geometry GEOMETRY(Point, 4326) NOT NULL\n );\n CREATE UNIQUE INDEX place_entrance_osm_id_idx ON place_entrance\n USING BTREE (osm_id);\n \"\"\")\n\n\n@_migration(5, 2, 99, 0)\ndef convert_country_tokens(conn: Connection, config: Configuration, **_: Any) -> None:\n \"\"\" Convert country word tokens\n\n Country tokens now save the country in the info field instead of the\n word. This migration removes all country tokens from the word table\n and reimports the default country name. This means that custom names\n are lost. If you need them back, invalidate the OSM objects containing\n the names by setting indexed_status to 2 and then reindex the database.\n \"\"\"\n tokenizer = tokenizer_factory.get_tokenizer_for_db(config)\n # There is only one tokenizer at the time of migration, so we make\n # some assumptions here about the structure of the database. This will\n # fail if somebody has written a custom tokenizer.\n with conn.cursor() as cur:\n cur.execute(\"DELETE FROM word WHERE type = 'C'\")\n conn.commit()\n\n setup_country_config(config)\n create_country_names(conn, tokenizer, config.get_str_list('LANGUAGES'))\n\n\n@_migration(5, 2, 99, 1)\ndef create_place_postcode_table(conn: Connection, config: Configuration, **_: Any) -> None:\n \"\"\" Restructure postcode tables\n \"\"\"\n sqlp = SQLPreprocessor(conn, config)\n mutable = not is_frozen(conn)\n has_place_table = table_exists(conn, 'place_postcode')\n has_postcode_table = table_exists(conn, 'location_postcodes')\n if mutable and not has_place_table:\n with conn.cursor() as cur:\n cur.execute(\n \"\"\"\n CREATE TABLE place_postcode (\n osm_type CHAR(1) NOT NULL,\n osm_id BIGINT NOT NULL,\n postcode TEXT NOT NULL,\n country_code TEXT,\n centroid GEOMETRY(Point, 4326) NOT NULL,\n geometry GEOMETRY(Geometry, 4326)\n )\n \"\"\")\n # Move postcode points into the new table\n cur.execute(\"ALTER TABLE place DISABLE TRIGGER ALL\")\n cur.execute(\n \"\"\"\n WITH deleted AS (\n DELETE FROM place\n WHERE (class = 'place' AND type = 'postcode')\n OR (osm_type = 'R'\n AND class = 'boundary' AND type = 'postal_code')\n RETURNING osm_type, osm_id, address->'postcode' as postcode,\n ST_Centroid(geometry) as centroid,\n (CASE WHEN class = 'place' THEN NULL ELSE geometry END) as geometry)\n INSERT INTO place_postcode (osm_type, osm_id, postcode, centroid, geometry)\n (SELECT * FROM deleted\n WHERE deleted.postcode is not NULL AND deleted.centroid is not NULL)\n \"\"\")\n cur.execute(\n \"\"\"\n CREATE INDEX place_postcode_osm_id_idx ON place_postcode\n USING BTREE (osm_type, osm_id)\n \"\"\")\n cur.execute(\n \"\"\"\n CREATE INDEX place_postcode_postcode_idx ON place_postcode\n USING BTREE (postcode)\n \"\"\")\n cur.execute(\"ALTER TABLE place ENABLE TRIGGER ALL\")\n if not has_postcode_table:\n sqlp.run_sql_file(conn, 'functions/postcode_triggers.sql')\n with conn.cursor() as cur:\n # create a new location_postcode table which will replace the\n # old one atomically in the end\n cur.execute(\n \"\"\"\n CREATE TABLE location_postcodes (\n place_id BIGINT,\n osm_id BIGINT,\n rank_search SMALLINT,\n parent_place_id BIGINT,\n indexed_status SMALLINT,\n indexed_date TIMESTAMP,\n country_code VARCHAR(2),\n postcode TEXT,\n centroid Geometry(Point, 4326),\n geometry Geometry(Geometry, 4326) NOT NULL\n )\n \"\"\")\n sqlp.run_string(conn,\n 'GRANT SELECT ON location_postcodes TO \"{{config.DATABASE_WEBUSER}}\"')\n # remove postcodes from the various auxiliary tables\n cur.execute(\n \"\"\"\n DELETE FROM place_addressline\n WHERE address_place_id = ANY(\n SELECT place_id FROM placex\n WHERE osm_type = 'R'\n AND class = 'boundary' AND type = 'postal_code')\n \"\"\")\n if mutable:\n cur.execute(\n \"\"\"\n SELECT deleteLocationArea(partition, place_id, rank_search),\n deleteSearchName(partition, place_id)\n FROM placex\n WHERE osm_type = 'R' AND class = 'boundary' AND type = 'postal_code'\n \"\"\")\n if table_exists(conn, 'search_name'):\n cur.execute(\n \"\"\"\n DELETE FROM search_name\n WHERE place_id = ANY(\n SELECT place_id FROM placex\n WHERE osm_type = 'R'\n AND class = 'boundary' AND type = 'postal_code')\n \"\"\")\n # move postcode areas from placex to location_postcodes\n # avoiding automatic invalidation\n cur.execute(\"ALTER TABLE placex DISABLE TRIGGER ALL\")\n cur.execute(\n \"\"\"\n WITH deleted AS (\n DELETE FROM placex\n WHERE osm_type = 'R'\n AND class = 'boundary' AND type = 'postal_code'\n RETURNING place_id, osm_id, rank_search, parent_place_id,\n indexed_status, indexed_date,\n country_code, postcode, centroid, geometry)\n INSERT INTO location_postcodes (SELECT * from deleted)\n \"\"\")\n cur.execute(\"ALTER TABLE placex ENABLE TRIGGER ALL\")\n # remove any old postcode centroid that would overlap with areas\n cur.execute(\n \"\"\"\n DELETE FROM location_postcode o USING location_postcodes n\n WHERE o.country_code = n.country_code\n AND o.postcode = n.postcode\n \"\"\")\n # copy over old postcodes\n cur.execute(\n \"\"\"\n INSERT INTO location_postcodes\n (SELECT place_id, NULL, rank_search, parent_place_id,\n indexed_status, indexed_date, country_code,\n postcode, geometry,\n ST_Expand(geometry, 0.05)\n FROM location_postcode)\n \"\"\")\n # add indexes and triggers\n cur.execute(\"\"\"CREATE INDEX idx_location_postcodes_geometry\n ON location_postcodes USING GIST(geometry)\"\"\")\n cur.execute(\"\"\"CREATE INDEX idx_location_postcodes_id\n ON location_postcodes USING BTREE(place_id)\"\"\")\n cur.execute(\"\"\"CREATE INDEX idx_location_postcodes_osmid\n ON location_postcodes USING BTREE(osm_id)\"\"\")\n cur.execute(\"\"\"CREATE INDEX idx_location_postcodes_postcode\n ON location_postcodes USING BTREE(postcode, country_code)\"\"\")\n cur.execute(\"\"\"CREATE INDEX idx_location_postcodes_parent_place_id\n ON location_postcodes USING BTREE(parent_place_id)\"\"\")\n cur.execute(\"\"\"CREATE TRIGGER location_postcodes_before_update\n BEFORE UPDATE ON location_postcodes\n FOR EACH ROW EXECUTE PROCEDURE postcodes_update()\"\"\")\n cur.execute(\"\"\"CREATE TRIGGER location_postcodes_before_delete\n BEFORE DELETE ON location_postcodes\n FOR EACH ROW EXECUTE PROCEDURE postcodes_delete()\"\"\")\n cur.execute(\"\"\"CREATE TRIGGER location_postcodes_before_insert\n BEFORE INSERT ON location_postcodes\n FOR EACH ROW EXECUTE PROCEDURE postcodes_insert()\"\"\")\n sqlp.run_string(\n conn,\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPolygon_nopostcode\n ON placex USING gist (geometry) {{db.tablespace.search_index}}\n WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')\n AND rank_address between 4 and 25\n AND name is not null AND linked_place_id is null;\n\n CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPlaceNode_nopostcode\n ON placex USING gist (ST_Buffer(geometry, reverse_place_diameter(rank_search)))\n {{db.tablespace.search_index}}\n WHERE rank_address between 4 and 25\n AND name is not null AND linked_place_id is null AND osm_type = 'N';\n\n CREATE INDEX idx_placex_geometry_placenode_nopostcode ON placex\n USING SPGIST (geometry) {{db.tablespace.address_index}}\n WHERE osm_type = 'N' and rank_search < 26 and class = 'place';\n ANALYSE;\n \"\"\")\n\n\n@_migration(5, 2, 99, 3)\ndef create_place_interpolation_table(conn: Connection, config: Configuration, **_: Any) -> None:\n \"\"\" Create place_interpolation table\n \"\"\"\n sqlp = SQLPreprocessor(conn, config)\n mutable = not is_frozen(conn)\n has_place_table = table_exists(conn, 'place_interpolation')\n\n if mutable and not has_place_table:\n # create tables\n conn.execute(\"\"\"\n CREATE TABLE place_interpolation (\n osm_id BIGINT NOT NULL,\n type TEXT NOT NULL,\n address HSTORE,\n nodes BIGINT[] NOT NULL,\n geometry GEOMETRY(LineString, 4326)\n );\n\n CREATE TABLE IF NOT EXISTS place_interpolation_to_be_deleted (\n osm_id BIGINT NOT NULL\n );\n \"\"\")\n # copy data over\n conn.execute(\"\"\"\n ALTER TABLE place DISABLE TRIGGER ALL;\n\n WITH deleted AS (\n DELETE FROM place\n WHERE class='place' and type = 'houses'\n RETURNING osm_type, osm_id,\n address->'interpolation' as itype,\n address - 'interpolation'::TEXT as address,\n geometry)\n INSERT INTO place_interpolation (osm_id, type, address, nodes, geometry)\n (SELECT d.osm_id, d.itype, d.address, p.nodes, d.geometry\n FROM deleted d, planet_osm_ways p\n WHERE osm_type = 'W'\n AND d.osm_id = p.id\n AND itype is not null\n AND ST_GeometryType(geometry) = 'ST_LineString');\n\n ALTER TABLE place ENABLE TRIGGER ALL;\n \"\"\")\n\n # create indices\n conn.execute(\"\"\"\n CREATE INDEX place_interpolation_nodes_idx ON place_interpolation\n USING gin(nodes);\n CREATE INDEX place_interpolation_osm_id_idx ON place_interpolation\n USING btree(osm_id);\n \"\"\")\n # create triggers\n sqlp.run_sql_file(conn, 'functions/interpolation.sql')\n conn.execute(\"\"\"\n CREATE TRIGGER place_interpolation_before_insert BEFORE INSERT ON place_interpolation\n FOR EACH ROW EXECUTE PROCEDURE place_interpolation_insert();\n CREATE TRIGGER place_interpolation_before_delete BEFORE DELETE ON place_interpolation\n FOR EACH ROW EXECUTE PROCEDURE place_interpolation_delete();\n \"\"\")\n # mutate location_property_osmline table\n conn.execute(\"\"\"\n ALTER TABLE location_property_osmline ADD COLUMN type TEXT;\n\n UPDATE location_property_osmline\n SET type = coalesce(address->'interpolation', 'all'),\n address = address - 'interpolation'::TEXT;\n \"\"\")\n\n\n@_migration(5, 2, 99, 4)\ndef backfill_importance(conn: Connection, **_: Any) -> None:\n \"\"\" Backfill missing importance values.\n \"\"\"\n conn.execute(\"\"\"UPDATE placex\n SET importance = 0.40001 - (rank_search::float / 75)\n WHERE importance is NULL OR importance <= 0\n \"\"\")\n if table_exists(conn, 'search_name')\\\n and table_has_column(conn, 'search_name', 'search_rank'):\n conn.execute(\"\"\"UPDATE search_name\n SET importance = 0.40001 - (search_rank::float / 75)\n WHERE importance is NULL OR importance <= 0\n \"\"\")\n conn.execute(\"ALTER TABLE search_name DROP COLUMN search_rank\")\n\n\n@_migration(5, 2, 99, 5)\ndef create_place_associated_street_table(conn: Connection, **_: Any) -> None:\n \"\"\" Create place_associated_street table for associatedStreet relations\n \"\"\"\n if table_exists(conn, 'place_associated_street'):\n return\n\n with conn.cursor() as cur:\n cur.execute(\"\"\"\n CREATE TABLE place_associated_street (\n relation_id BIGINT NOT NULL,\n member_type TEXT NOT NULL,\n member_id BIGINT NOT NULL,\n member_role TEXT NOT NULL\n )\n \"\"\")\n\n # Check the osm2pgsql middle table format using the properties table.\n db_format = '1'\n if table_exists(conn, 'osm2pgsql_properties'):\n cur.execute(\"\"\"\n SELECT value FROM osm2pgsql_properties\n WHERE property = 'db_format'\n \"\"\")\n row = cur.fetchone()\n if row is not None:\n db_format = row[0]\n\n # Populate from planet_osm_rels if the JSONB members format is in use.\n if db_format != '1':\n cur.execute(\"\"\"\n INSERT INTO place_associated_street\n (relation_id, member_type, member_id, member_role)\n SELECT id,\n UPPER(x.type),\n x.ref,\n x.role\n FROM planet_osm_rels,\n LATERAL jsonb_to_recordset(members)\n AS x(ref bigint, role text, type text)\n WHERE tags->>'type' = 'associatedStreet'\n AND x.role IN ('street', 'house')\n \"\"\")\n else:\n LOG.warning(\n '\\n'\n 'WARNING: Your database uses an old osm2pgsql middle table format.\\n'\n 'The place_associated_street table has been created but is empty.\\n'\n 'A full reimport is strongly recommended.\\n'\n )\n\n cur.execute(\"\"\"\n CREATE INDEX place_associated_street_member_idx\n ON place_associated_street USING BTREE (member_type, member_id);\n CREATE INDEX place_associated_street_relation_id_idx\n ON place_associated_street USING BTREE (relation_id);\n \"\"\")\n\n\n@_migration(5, 3, 99, 0)\ndef create_associated_street_triggers(conn: Connection, config: Configuration, **_: Any) -> None:\n \"\"\" Create triggers for tables on associated street tables.\n \"\"\"\n SQLPreprocessor(conn, config).run_sql_file(conn, 'functions/associated_street_triggers.sql')\n with conn.cursor() as cur:\n cur.execute(\"\"\"CREATE OR REPLACE TRIGGER place_associated_street_update\n AFTER INSERT OR DELETE ON place_associated_street\n FOR EACH ROW EXECUTE FUNCTION invalidate_associated_street_members()\"\"\")\n"
},
{
"path": "src/nominatim_db/tools/postcodes.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for importing, updating and otherwise maintaining the table\nof artificial postcode centroids.\n\"\"\"\nfrom typing import Optional, Tuple, Dict, TextIO\nfrom collections import defaultdict\nfrom pathlib import Path\nimport csv\nimport gzip\nimport logging\nfrom math import isfinite\n\nfrom psycopg import sql as pysql\n\nfrom ..db.connection import connect, Connection, table_exists\nfrom ..utils.centroid import PointsCentroid\nfrom ..data.postcode_format import PostcodeFormatter, CountryPostcodeMatcher\nfrom ..tokenizer.base import AbstractAnalyzer, AbstractTokenizer\n\nLOG = logging.getLogger()\n\n\ndef _to_float(numstr: str, max_value: float) -> float:\n \"\"\" Convert the number in string into a float. The number is expected\n to be in the range of [-max_value, max_value]. Otherwise rises a\n ValueError.\n \"\"\"\n num = float(numstr)\n if not isfinite(num) or num <= -max_value or num >= max_value:\n raise ValueError()\n\n return num\n\n\ndef _extent_to_rank(extent: int) -> int:\n \"\"\" Guess a suitable search rank from the extent of a postcode.\n \"\"\"\n if extent <= 100:\n return 25\n if extent <= 3000:\n return 23\n return 21\n\n\nclass _PostcodeCollector:\n \"\"\" Collector for postcodes of a single country.\n \"\"\"\n\n def __init__(self, country: str, matcher: Optional[CountryPostcodeMatcher],\n default_extent: int, exclude: set[str] = set()):\n self.country = country\n self.matcher = matcher\n self.extent = default_extent\n self.exclude = exclude\n self.collected: Dict[str, PointsCentroid] = defaultdict(PointsCentroid)\n self.normalization_cache: Optional[Tuple[str, Optional[str]]] = None\n\n def add(self, postcode: str, x: float, y: float) -> None:\n \"\"\" Add the given postcode to the collection cache. If the postcode\n already existed, it is overwritten with the new centroid.\n \"\"\"\n if self.matcher is not None:\n normalized: Optional[str]\n if self.normalization_cache and self.normalization_cache[0] == postcode:\n normalized = self.normalization_cache[1]\n else:\n match = self.matcher.match(postcode)\n normalized = self.matcher.normalize(match) if match else None\n self.normalization_cache = (postcode, normalized)\n\n if normalized and normalized not in self.exclude:\n self.collected[normalized] += (x, y)\n\n def commit(self, conn: Connection, analyzer: AbstractAnalyzer,\n project_dir: Optional[Path], is_initial: bool) -> None:\n \"\"\" Update postcodes for the country from the postcodes selected so far.\n\n When 'project_dir' is set, then any postcode files found in this\n directory are taken into account as well.\n \"\"\"\n if project_dir is not None:\n self._update_from_external(analyzer, project_dir)\n\n if is_initial:\n to_delete = []\n else:\n with conn.cursor() as cur:\n cur.execute(\"\"\"SELECT postcode FROM location_postcodes\n WHERE country_code = %s AND osm_id is null\"\"\",\n (self.country, ))\n to_delete = [row[0] for row in cur if row[0] not in self.collected]\n\n to_add = [dict(zip(('pc', 'x', 'y'), (k, *v.centroid())))\n for k, v in self.collected.items()]\n self.collected = defaultdict(PointsCentroid)\n\n LOG.info(\"Processing country '%s' (%s added, %s deleted).\",\n self.country, len(to_add), len(to_delete))\n\n with conn.cursor() as cur:\n if to_add:\n columns = ['country_code',\n 'rank_search',\n 'postcode',\n 'centroid',\n 'geometry']\n values = [pysql.Literal(self.country),\n pysql.Literal(_extent_to_rank(self.extent)),\n pysql.Placeholder('pc'),\n pysql.SQL('ST_SetSRID(ST_MakePoint(%(x)s, %(y)s), 4326)'),\n pysql.SQL(\"\"\"expand_by_meters(\n ST_SetSRID(ST_MakePoint(%(x)s, %(y)s), 4326), {})\"\"\")\n .format(pysql.Literal(self.extent))]\n if is_initial:\n columns.extend(('place_id', 'indexed_status'))\n values.extend((pysql.SQL(\"nextval('seq_place')\"), pysql.Literal(1)))\n\n cur.executemany(pysql.SQL(\"INSERT INTO location_postcodes ({}) VALUES ({})\")\n .format(pysql.SQL(',')\n .join(pysql.Identifier(c) for c in columns),\n pysql.SQL(',').join(values)),\n to_add)\n if to_delete:\n cur.execute(\"\"\"DELETE FROM location_postcodes\n WHERE country_code = %s and postcode = any(%s)\n AND osm_id is null\n \"\"\", (self.country, to_delete))\n\n def _update_from_external(self, analyzer: AbstractAnalyzer, project_dir: Path) -> None:\n \"\"\" Look for an external postcode file for the active country in\n the project directory and add missing postcodes when found.\n \"\"\"\n csvfile = self._open_external(project_dir)\n if csvfile is None:\n return\n\n try:\n reader = csv.DictReader(csvfile)\n for row in reader:\n if 'postcode' not in row or 'lat' not in row or 'lon' not in row:\n LOG.warning(\"Bad format for external postcode file for country '%s'.\"\n \" Ignored.\", self.country)\n return\n postcode = analyzer.normalize_postcode(row['postcode'])\n if postcode not in self.collected:\n try:\n # Do float conversation separately, it might throw\n centroid = (_to_float(row['lon'], 180),\n _to_float(row['lat'], 90))\n self.collected[postcode] += centroid\n except ValueError:\n LOG.warning(\"Bad coordinates %s, %s in '%s' country postcode file.\",\n row['lat'], row['lon'], self.country)\n\n finally:\n csvfile.close()\n\n def _open_external(self, project_dir: Path) -> Optional[TextIO]:\n fname = project_dir / f'{self.country}_postcodes.csv'\n\n if fname.is_file():\n LOG.info(\"Using external postcode file '%s'.\", fname)\n return open(fname, 'r', encoding='utf-8')\n\n fname = project_dir / f'{self.country}_postcodes.csv.gz'\n\n if fname.is_file():\n LOG.info(\"Using external postcode file '%s'.\", fname)\n return gzip.open(fname, 'rt', encoding='utf-8')\n\n return None\n\n\ndef update_postcodes(dsn: str, project_dir: Optional[Path],\n tokenizer: AbstractTokenizer, force_reimport: bool = False) -> None:\n \"\"\" Update the table of postcodes from the input tables\n placex and place_postcode.\n \"\"\"\n matcher = PostcodeFormatter()\n with tokenizer.name_analyzer() as analyzer:\n with connect(dsn) as conn:\n # Backfill country_code column where required\n conn.execute(\"\"\"UPDATE place_postcode\n SET country_code = get_country_code(centroid)\n WHERE country_code is null\n \"\"\")\n if force_reimport:\n conn.execute(\"TRUNCATE location_postcodes\")\n is_initial = True\n else:\n is_initial = _is_postcode_table_empty(conn)\n if is_initial:\n conn.execute(\"\"\"ALTER TABLE location_postcodes\n DISABLE TRIGGER location_postcodes_before_insert\"\"\")\n # Now update first postcode areas\n _update_postcode_areas(conn, analyzer, matcher, is_initial)\n # Then fill with estimated postcode centroids from other info\n _update_guessed_postcode(conn, analyzer, matcher, project_dir, is_initial)\n if is_initial:\n conn.execute(\"\"\"ALTER TABLE location_postcodes\n ENABLE TRIGGER location_postcodes_before_insert\"\"\")\n conn.commit()\n\n analyzer.update_postcodes_from_db()\n\n\ndef _is_postcode_table_empty(conn: Connection) -> bool:\n \"\"\" Check if there are any entries in the location_postcodes table yet.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute(\"SELECT place_id FROM location_postcodes LIMIT 1\")\n return cur.fetchone() is None\n\n\ndef _insert_postcode_areas(conn: Connection, country_code: str,\n extent: int, pcs: list[dict[str, str]],\n is_initial: bool) -> None:\n if pcs:\n with conn.cursor() as cur:\n columns = ['osm_id', 'country_code',\n 'rank_search', 'postcode',\n 'centroid', 'geometry']\n values = [pysql.Identifier('osm_id'), pysql.Identifier('country_code'),\n pysql.Literal(_extent_to_rank(extent)), pysql.Placeholder('out'),\n pysql.Identifier('centroid'), pysql.Identifier('geometry')]\n if is_initial:\n columns.extend(('place_id', 'indexed_status'))\n values.extend((pysql.SQL(\"nextval('seq_place')\"), pysql.Literal(1)))\n\n cur.executemany(\n pysql.SQL(\n \"\"\" INSERT INTO location_postcodes ({})\n SELECT {} FROM place_postcode\n WHERE osm_type = 'R'\n and country_code = {} and postcode = %(in)s\n and geometry is not null\n \"\"\").format(pysql.SQL(',')\n .join(pysql.Identifier(c) for c in columns),\n pysql.SQL(',').join(values),\n pysql.Literal(country_code)),\n pcs)\n\n\ndef _update_postcode_areas(conn: Connection, analyzer: AbstractAnalyzer,\n matcher: PostcodeFormatter, is_initial: bool) -> None:\n \"\"\" Update the postcode areas made from postcode boundaries.\n \"\"\"\n # first delete all areas that have gone\n if not is_initial:\n conn.execute(\"\"\" DELETE FROM location_postcodes pc\n WHERE pc.osm_id is not null\n AND NOT EXISTS(\n SELECT * FROM place_postcode pp\n WHERE pp.osm_type = 'R' and pp.osm_id = pc.osm_id\n and geometry is not null)\n \"\"\")\n # now insert all in country batches, triggers will ensure proper updates\n with conn.cursor() as cur:\n cur.execute(\"\"\" SELECT country_code, postcode FROM place_postcode\n WHERE geometry is not null and osm_type = 'R'\n ORDER BY country_code\n \"\"\")\n country_code = None\n fmt = None\n pcs = []\n for cc, postcode in cur:\n if country_code is None:\n country_code = cc\n fmt = matcher.get_matcher(country_code)\n elif country_code != cc:\n _insert_postcode_areas(conn, country_code,\n matcher.get_postcode_extent(country_code), pcs,\n is_initial)\n country_code = cc\n fmt = matcher.get_matcher(country_code)\n pcs = []\n\n if fmt is not None:\n if (m := fmt.match(postcode)):\n pcs.append({'out': fmt.normalize(m), 'in': postcode})\n\n if country_code is not None and pcs:\n _insert_postcode_areas(conn, country_code,\n matcher.get_postcode_extent(country_code), pcs,\n is_initial)\n\n\ndef _update_guessed_postcode(conn: Connection, analyzer: AbstractAnalyzer,\n matcher: PostcodeFormatter, project_dir: Optional[Path],\n is_initial: bool) -> None:\n \"\"\" Computes artificial postcode centroids from the placex table,\n potentially enhances it with external data and then updates the\n postcodes in the table 'location_postcodes'.\n \"\"\"\n # First get the list of countries that currently have postcodes.\n # (Doing this before starting to insert, so it is fast on import.)\n if is_initial:\n todo_countries: set[str] = set()\n else:\n with conn.cursor() as cur:\n cur.execute(\"\"\"SELECT DISTINCT country_code FROM location_postcodes\n WHERE osm_id is null\"\"\")\n todo_countries = {row[0] for row in cur}\n\n # Next, get the list of postcodes that are already covered by areas.\n area_pcs = defaultdict(set)\n with conn.cursor() as cur:\n cur.execute(\"\"\"SELECT country_code, postcode\n FROM location_postcodes WHERE osm_id is not null\n ORDER BY country_code\"\"\")\n for cc, pc in cur:\n area_pcs[cc].add(pc)\n\n # Create a temporary table which contains coverage of the postcode areas.\n with conn.cursor() as cur:\n cur.execute(\"DROP TABLE IF EXISTS _global_postcode_area\")\n cur.execute(\"\"\"CREATE TABLE _global_postcode_area AS\n (SELECT ST_SubDivide(ST_SimplifyPreserveTopology(\n ST_Union(geometry), 0.00001), 128) as geometry\n FROM place_postcode WHERE geometry is not null)\n \"\"\")\n cur.execute(\"CREATE INDEX ON _global_postcode_area USING gist(geometry)\")\n\n # Recompute the list of valid postcodes from placex.\n with conn.cursor(name=\"placex_postcodes\") as cur:\n cur.execute(\"\"\"\n SELECT country_code, postcode, ST_X(centroid), ST_Y(centroid)\n FROM (\n (SELECT country_code, address->'postcode' as postcode, centroid\n FROM placex WHERE address ? 'postcode')\n UNION\n (SELECT country_code, postcode, centroid\n FROM place_postcode WHERE geometry is null)\n ) x\n WHERE not postcode like '%,%' and not postcode like '%;%'\n AND NOT EXISTS(SELECT * FROM _global_postcode_area g\n WHERE ST_Intersects(x.centroid, g.geometry))\n ORDER BY country_code\"\"\")\n\n collector = None\n\n for country, postcode, x, y in cur:\n if collector is None or country != collector.country:\n if collector is not None:\n collector.commit(conn, analyzer, project_dir, is_initial)\n collector = _PostcodeCollector(country, matcher.get_matcher(country),\n matcher.get_postcode_extent(country),\n exclude=area_pcs[country])\n todo_countries.discard(country)\n collector.add(postcode, x, y)\n\n if collector is not None:\n collector.commit(conn, analyzer, project_dir, is_initial)\n\n # Now handle any countries that are only in the postcode table.\n for country in todo_countries:\n fmt = matcher.get_matcher(country)\n ext = matcher.get_postcode_extent(country)\n _PostcodeCollector(country, fmt, ext,\n exclude=area_pcs[country]).commit(conn, analyzer, project_dir, False)\n\n conn.execute(\"DROP TABLE IF EXISTS _global_postcode_area\")\n\n\ndef can_compute(dsn: str) -> bool:\n \"\"\" Check that the necessary tables exist so that postcodes can be computed.\n \"\"\"\n with connect(dsn) as conn:\n return table_exists(conn, 'place_postcode')\n"
},
{
"path": "src/nominatim_db/tools/refresh.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for bringing auxiliary data in the database up-to-date.\n\"\"\"\nfrom typing import MutableSequence, Tuple, Any, Mapping, Sequence, List\nimport csv\nimport gzip\nimport logging\nfrom pathlib import Path\n\nfrom psycopg import sql as pysql\n\nfrom ..config import Configuration\nfrom ..db.connection import Connection, connect, drop_tables\nfrom ..db.utils import execute_file\nfrom ..db.sql_preprocessor import SQLPreprocessor\n\nLOG = logging.getLogger()\n\nOSM_TYPE = {'N': 'node', 'W': 'way', 'R': 'relation'}\n\n\ndef _add_address_level_rows_from_entry(rows: MutableSequence[Tuple[Any, ...]],\n entry: Mapping[str, Any]) -> None:\n \"\"\" Converts a single entry from the JSON format for address rank\n descriptions into a flat format suitable for inserting into a\n PostgreSQL table and adds these lines to `rows`.\n \"\"\"\n countries = entry.get('countries') or (None, )\n for key, values in entry['tags'].items():\n for value, ranks in values.items():\n if isinstance(ranks, list):\n rank_search, rank_address = ranks\n else:\n rank_search = rank_address = ranks\n if not value:\n value = None\n for country in countries:\n rows.append((country, key, value, rank_search, rank_address))\n\n\ndef load_address_levels(conn: Connection, table: str, levels: Sequence[Mapping[str, Any]]) -> None:\n \"\"\" Replace the `address_levels` table with the contents of `levels'.\n\n A new table is created any previously existing table is dropped.\n The table has the following columns:\n country, class, type, rank_search, rank_address\n \"\"\"\n rows: List[Tuple[Any, ...]] = []\n for entry in levels:\n _add_address_level_rows_from_entry(rows, entry)\n\n drop_tables(conn, table)\n\n with conn.cursor() as cur:\n cur.execute(pysql.SQL(\"\"\"CREATE TABLE {} (\n country_code varchar(2),\n class TEXT,\n type TEXT,\n rank_search SMALLINT,\n rank_address SMALLINT)\n \"\"\").format(pysql.Identifier(table)))\n\n cur.executemany(pysql.SQL(\"INSERT INTO {} VALUES (%s, %s, %s, %s, %s)\")\n .format(pysql.Identifier(table)), rows)\n\n cur.execute(pysql.SQL('CREATE UNIQUE INDEX ON {} (country_code, class, type)')\n .format(pysql.Identifier(table)))\n\n conn.commit()\n\n\ndef load_address_levels_from_config(conn: Connection, config: Configuration) -> None:\n \"\"\" Replace the `address_levels` table with the content as\n defined in the given configuration. Uses the parameter\n NOMINATIM_ADDRESS_LEVEL_CONFIG to determine the location of the\n configuration file.\n \"\"\"\n cfg = config.load_sub_configuration('', config='ADDRESS_LEVEL_CONFIG')\n load_address_levels(conn, 'address_levels', cfg)\n\n\ndef create_functions(conn: Connection, config: Configuration,\n enable_diff_updates: bool = True,\n enable_debug: bool = False) -> None:\n \"\"\" (Re)create the PL/pgSQL functions.\n \"\"\"\n sql = SQLPreprocessor(conn, config)\n\n sql.run_sql_file(conn, 'functions.sql',\n disable_diff_updates=not enable_diff_updates,\n debug=enable_debug)\n\n\ndef import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:\n \"\"\" Replaces the wikipedia importance tables with new data.\n The import is run in a single transaction so that the new data\n is replace seamlessly.\n\n Returns 0 if all was well and 1 if the importance file could not\n be found. Throws an exception if there was an error reading the file.\n \"\"\"\n if import_importance_csv(dsn, data_path / 'wikimedia-importance.csv.gz') == 0 \\\n or import_importance_sql(dsn, data_path / 'wikimedia-importance.sql.gz',\n ignore_errors) == 0:\n return 0\n\n return 1\n\n\ndef import_importance_csv(dsn: str, data_file: Path) -> int:\n \"\"\" Replace wikipedia importance table with data from a\n single CSV file.\n\n The file must be a gzipped CSV and have the following columns:\n language, title, importance, wikidata_id\n\n Other columns may be present but will be ignored.\n \"\"\"\n if not data_file.exists():\n return 1\n\n # Only import the first occurrence of a wikidata ID.\n # This keeps indexes and table small.\n wd_done = set()\n\n with connect(dsn) as conn:\n drop_tables(conn, 'wikipedia_article', 'wikipedia_redirect', 'wikimedia_importance')\n with conn.cursor() as cur:\n cur.execute(\"\"\"CREATE TABLE wikimedia_importance (\n language TEXT NOT NULL,\n title TEXT NOT NULL,\n importance double precision NOT NULL,\n wikidata TEXT\n ) \"\"\")\n\n copy_cmd = \"\"\"COPY wikimedia_importance(language, title, importance, wikidata)\n FROM STDIN\"\"\"\n with gzip.open(\n str(data_file), 'rt', encoding='utf-8') as fd, \\\n cur.copy(copy_cmd) as copy:\n for row in csv.DictReader(fd, delimiter='\\t', quotechar='|'):\n wd_id = int(row['wikidata_id'][1:])\n copy.write_row((row['language'],\n row['title'],\n row['importance'],\n None if wd_id in wd_done else row['wikidata_id']))\n wd_done.add(wd_id)\n\n cur.execute(\"\"\"CREATE INDEX IF NOT EXISTS idx_wikimedia_importance_title\n ON wikimedia_importance (title)\"\"\")\n cur.execute(\"\"\"CREATE INDEX IF NOT EXISTS idx_wikimedia_importance_wikidata\n ON wikimedia_importance (wikidata)\n WHERE wikidata is not null\"\"\")\n\n conn.commit()\n\n return 0\n\n\ndef import_importance_sql(dsn: str, data_file: Path, ignore_errors: bool) -> int:\n \"\"\" Replace wikipedia importance table with data from an SQL file.\n \"\"\"\n if not data_file.exists():\n return 1\n\n pre_code = \"\"\"BEGIN;\n DROP TABLE IF EXISTS \"wikipedia_article\";\n DROP TABLE IF EXISTS \"wikipedia_redirect\";\n DROP TABLE IF EXISTS \"wikipedia_importance\";\n \"\"\"\n post_code = \"COMMIT\"\n execute_file(dsn, data_file, ignore_errors=ignore_errors,\n pre_code=pre_code, post_code=post_code)\n\n return 0\n\n\ndef import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:\n \"\"\" Replaces the secondary importance raster data table with new data.\n\n Returns 0 if all was well and 1 if the raster SQL file could not\n be found. Throws an exception if there was an error reading the file.\n \"\"\"\n datafile = data_path / 'secondary_importance.sql.gz'\n if not datafile.exists():\n return 1\n\n execute_file(dsn, datafile, ignore_errors=ignore_errors)\n\n return 0\n\n\ndef recompute_importance(conn: Connection) -> None:\n \"\"\" Recompute wikipedia links and importance for all entries in placex.\n This is a long-running operations that must not be executed in\n parallel with updates.\n \"\"\"\n with conn.cursor() as cur:\n cur.execute('ALTER TABLE placex DISABLE TRIGGER ALL')\n cur.execute(\"\"\"\n UPDATE placex SET (wikipedia, importance) =\n (SELECT wikipedia, importance\n FROM compute_importance(extratags, country_code, rank_search, centroid))\n \"\"\")\n cur.execute(\"\"\"\n UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance\n FROM placex d\n WHERE s.place_id = d.linked_place_id and d.wikipedia is not null\n and (s.wikipedia is null or s.importance < d.importance);\n \"\"\")\n cur.execute(\"\"\"\n UPDATE search_name s SET importance = p.importance\n FROM placex p\n WHERE s.place_id = p.place_id AND s.importance != p.importance\n \"\"\")\n\n cur.execute('ALTER TABLE placex ENABLE TRIGGER ALL')\n conn.commit()\n\n\ndef invalidate_osm_object(osm_type: str, osm_id: int, conn: Connection,\n recursive: bool = True) -> None:\n \"\"\" Mark the given OSM object for reindexing. When 'recursive' is set\n to True (the default), then all dependent objects are marked for\n reindexing as well.\n\n 'osm_type' must be on of 'N' (node), 'W' (way) or 'R' (relation).\n If the given object does not exist, then nothing happens.\n \"\"\"\n assert osm_type in ('N', 'R', 'W')\n\n LOG.warning(\"Invalidating OSM %s %s%s.\",\n OSM_TYPE[osm_type], osm_id,\n ' and its dependent places' if recursive else '')\n\n with conn.cursor() as cur:\n if recursive:\n sql = \"\"\"SELECT place_force_update(place_id)\n FROM placex WHERE osm_type = %s and osm_id = %s\"\"\"\n else:\n sql = \"\"\"UPDATE placex SET indexed_status = 2\n WHERE osm_type = %s and osm_id = %s\"\"\"\n\n cur.execute(sql, (osm_type, osm_id))\n"
},
{
"path": "src/nominatim_db/tools/replication.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for updating a database from a replication source.\n\"\"\"\nfrom typing import ContextManager, MutableMapping, Any, Generator, cast, Iterator\nfrom contextlib import contextmanager\nimport datetime as dt\nfrom enum import Enum\nimport logging\nimport time\nimport types\nimport urllib.request as urlrequest\n\nfrom ..errors import UsageError\nfrom ..db import status\nfrom ..db.connection import Connection, connect\nfrom .exec_utils import run_osm2pgsql\n\ntry:\n from osmium.replication.server import ReplicationServer\n from osmium import WriteHandler\n from osmium import version as pyo_version\n import requests\nexcept ModuleNotFoundError as exc:\n logging.getLogger().critical(\"pyosmium not installed. Replication functions not available.\\n\"\n \"To install pyosmium via pip: pip install osmium\")\n raise UsageError(\"replication tools not available\") from exc\n\nLOG = logging.getLogger()\n\n\ndef init_replication(conn: Connection, base_url: str,\n socket_timeout: int = 60) -> None:\n \"\"\" Set up replication for the server at the given base URL.\n \"\"\"\n LOG.info(\"Using replication source: %s\", base_url)\n date = status.compute_database_date(conn)\n\n # margin of error to make sure we get all data\n date -= dt.timedelta(hours=3)\n\n with _make_replication_server(base_url, socket_timeout) as repl:\n seq = repl.timestamp_to_sequence(date)\n\n if seq is None:\n LOG.fatal(\"Cannot reach the configured replication service '%s'.\\n\"\n \"Does the URL point to a directory containing OSM update data?\",\n base_url)\n raise UsageError(\"Failed to reach replication service\")\n\n status.set_status(conn, date=date, seq=seq)\n\n LOG.warning(\"Updates initialised at sequence %s (%s)\", seq, date)\n\n\ndef check_for_updates(conn: Connection, base_url: str,\n socket_timeout: int = 60) -> int:\n \"\"\" Check if new data is available from the replication service at the\n given base URL.\n \"\"\"\n _, seq, _ = status.get_status(conn)\n\n if seq is None:\n LOG.error(\"Replication not set up. \"\n \"Please run 'nominatim replication --init' first.\")\n return 254\n\n with _make_replication_server(base_url, socket_timeout) as repl:\n state = repl.get_state_info()\n\n if state is None:\n LOG.error(\"Cannot get state for URL %s.\", base_url)\n return 253\n\n if state.sequence <= seq:\n LOG.warning(\"Database is up to date.\")\n return 2\n\n LOG.warning(\"New data available (%i => %i).\", seq, state.sequence)\n return 0\n\n\nclass UpdateState(Enum):\n \"\"\" Possible states after an update has run.\n \"\"\"\n\n UP_TO_DATE = 0\n MORE_PENDING = 2\n NO_CHANGES = 3\n\n\ndef update(dsn: str, options: MutableMapping[str, Any],\n socket_timeout: int = 60) -> UpdateState:\n \"\"\" Update database from the next batch of data. Returns the state of\n updates according to `UpdateState`.\n \"\"\"\n with connect(dsn) as conn:\n startdate, startseq, indexed = status.get_status(conn)\n conn.commit()\n\n if startseq is None:\n LOG.error(\"Replication not set up. \"\n \"Please run 'nominatim replication --init' first.\")\n raise UsageError(\"Replication not set up.\")\n\n assert startdate is not None\n\n if not indexed and options['indexed_only']:\n LOG.info(\"Skipping update. There is data that needs indexing.\")\n return UpdateState.MORE_PENDING\n\n last_since_update = dt.datetime.now(dt.timezone.utc) - startdate\n update_interval = dt.timedelta(seconds=options['update_interval'])\n if last_since_update < update_interval:\n duration = (update_interval - last_since_update).seconds\n LOG.warning(\"Sleeping for %s sec before next update.\", duration)\n time.sleep(duration)\n\n if options['import_file'].exists():\n options['import_file'].unlink()\n\n # Read updates into file.\n with _make_replication_server(options['base_url'], socket_timeout) as repl:\n outhandler = WriteHandler(str(options['import_file']))\n endseq = repl.apply_diffs(outhandler, startseq + 1,\n max_size=options['max_diff_size'] * 1024)\n outhandler.close()\n\n if endseq is None:\n return UpdateState.NO_CHANGES\n\n with connect(dsn) as conn:\n run_osm2pgsql_updates(conn, options)\n\n # Write the current status to the file\n endstate = repl.get_state_info(endseq)\n status.set_status(conn, endstate.timestamp if endstate else None,\n seq=endseq, indexed=False)\n conn.commit()\n\n return UpdateState.UP_TO_DATE\n\n\ndef run_osm2pgsql_updates(conn: Connection, options: MutableMapping[str, Any]) -> None:\n \"\"\" Run osm2pgsql in append mode.\n \"\"\"\n # Remove any stale deletion marks.\n with conn.cursor() as cur:\n cur.execute('TRUNCATE place_to_be_deleted')\n conn.commit()\n\n # Consume updates with osm2pgsql.\n options['append'] = True\n options['disable_jit'] = True\n run_osm2pgsql(options)\n\n # Handle deletions\n with conn.cursor() as cur:\n cur.execute('SELECT flush_deleted_places()')\n conn.commit()\n\n\ndef _make_replication_server(url: str, timeout: int) -> ContextManager[ReplicationServer]:\n \"\"\" Returns a ReplicationServer in form of a context manager.\n\n Creates a light wrapper around older versions of pyosmium that did\n not support the context manager interface.\n \"\"\"\n if hasattr(ReplicationServer, '__enter__'):\n # Patches the open_url function for pyosmium >= 3.2\n # where the socket timeout is no longer respected.\n def patched_open_url(self: ReplicationServer, url: urlrequest.Request) -> Any:\n \"\"\" Download a resource from the given URL and return a byte sequence\n of the content.\n \"\"\"\n headers = {\"User-Agent\": f\"Nominatim (pyosmium/{pyo_version.pyosmium_release})\"}\n\n if self.session is not None:\n return self.session.get(url.get_full_url(),\n headers=headers, timeout=timeout or None,\n stream=True)\n\n @contextmanager\n def _get_url_with_session() -> Iterator[requests.Response]:\n with requests.Session() as session:\n request = session.get(url.get_full_url(),\n headers=headers, timeout=timeout or None,\n stream=True)\n yield request\n\n return _get_url_with_session()\n\n repl = ReplicationServer(url)\n setattr(repl, 'open_url', types.MethodType(patched_open_url, repl))\n\n return cast(ContextManager[ReplicationServer], repl)\n\n @contextmanager\n def get_cm() -> Generator[ReplicationServer, None, None]:\n yield ReplicationServer(url)\n\n return get_cm()\n"
},
{
"path": "src/nominatim_db/tools/special_phrases/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/tools/special_phrases/importer_statistics.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\n Contains the class which handles statistics for the\n import of special phrases.\n\"\"\"\nimport logging\nLOG = logging.getLogger()\n\n\nclass SpecialPhrasesImporterStatistics():\n \"\"\"\n Class handling statistics of the import\n process of special phrases.\n \"\"\"\n def __init__(self) -> None:\n self._intialize_values()\n\n def _intialize_values(self) -> None:\n \"\"\"\n Set all counts for the global\n import to 0.\n \"\"\"\n self.tables_created = 0\n self.tables_deleted = 0\n self.tables_ignored = 0\n self.invalids = 0\n\n def notify_one_phrase_invalid(self) -> None:\n \"\"\"\n Add +1 to the count of invalid entries\n fetched from the wiki.\n \"\"\"\n self.invalids += 1\n\n def notify_one_table_created(self) -> None:\n \"\"\"\n Add +1 to the count of created tables.\n \"\"\"\n self.tables_created += 1\n\n def notify_one_table_deleted(self) -> None:\n \"\"\"\n Add +1 to the count of deleted tables.\n \"\"\"\n self.tables_deleted += 1\n\n def notify_one_table_ignored(self) -> None:\n \"\"\"\n Add +1 to the count of ignored tables.\n \"\"\"\n self.tables_ignored += 1\n\n def notify_import_done(self) -> None:\n \"\"\"\n Print stats for the whole import process\n and reset all values.\n \"\"\"\n LOG.info('====================================================================')\n LOG.info('Final statistics of the import:')\n LOG.info('- %s phrases were invalid.', self.invalids)\n if self.invalids > 0:\n LOG.info(' Those invalid phrases have been skipped.')\n LOG.info('- %s tables were ignored as they already exist on the database',\n self.tables_ignored)\n LOG.info('- %s tables were created', self.tables_created)\n LOG.info('- %s tables were deleted from the database', self.tables_deleted)\n if self.tables_deleted > 0:\n LOG.info(' They were deleted as they are not valid anymore.')\n\n if self.invalids > 0:\n LOG.warning('%s phrases were invalid and have been skipped during the whole process.',\n self.invalids)\n\n self._intialize_values()\n"
},
{
"path": "src/nominatim_db/tools/special_phrases/sp_csv_loader.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\n Module containing the SPCsvLoader class.\n\n The class allows to load phrases from a csv file.\n\"\"\"\nfrom typing import Iterable\nimport csv\nimport os\n\nfrom ...errors import UsageError\nfrom .special_phrase import SpecialPhrase\n\n\nclass SPCsvLoader:\n \"\"\"\n Handles loading of special phrases from external csv file.\n \"\"\"\n def __init__(self, csv_path: str) -> None:\n self.csv_path = csv_path\n\n def generate_phrases(self) -> Iterable[SpecialPhrase]:\n \"\"\" Open and parse the given csv file.\n Create the corresponding SpecialPhrases.\n \"\"\"\n self._check_csv_validity()\n\n with open(self.csv_path, encoding='utf-8') as fd:\n reader = csv.DictReader(fd, delimiter=',')\n for row in reader:\n yield SpecialPhrase(row['phrase'], row['class'], row['type'], row['operator'])\n\n def _check_csv_validity(self) -> None:\n \"\"\"\n Check that the csv file has the right extension.\n \"\"\"\n _, extension = os.path.splitext(self.csv_path)\n\n if extension != '.csv':\n raise UsageError(f'The file {self.csv_path} is not a csv file.')\n"
},
{
"path": "src/nominatim_db/tools/special_phrases/sp_importer.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\n Module containing the class handling the import\n of the special phrases.\n\n Phrases are analyzed and imported into the database.\n\n The phrases already present in the database which are not\n valids anymore are removed.\n\"\"\"\nfrom typing import Iterable, Tuple, Mapping, Sequence, Optional, Set\nimport logging\nimport re\nfrom psycopg.sql import Identifier, SQL\n\nfrom ...typing import Protocol\nfrom ...config import Configuration\nfrom ...db.connection import Connection, drop_tables, index_exists\nfrom .importer_statistics import SpecialPhrasesImporterStatistics\nfrom .special_phrase import SpecialPhrase\nfrom ...tokenizer.base import AbstractTokenizer\n\nLOG = logging.getLogger()\n\n\ndef _classtype_table(phrase_class: str, phrase_type: str) -> str:\n \"\"\" Return the name of the table for the given class and type.\n \"\"\"\n return f'place_classtype_{phrase_class}_{phrase_type}'\n\n\nclass SpecialPhraseLoader(Protocol):\n \"\"\" Protocol for classes implementing a loader for special phrases.\n \"\"\"\n\n def generate_phrases(self) -> Iterable[SpecialPhrase]:\n \"\"\" Generates all special phrase terms this loader can produce.\n \"\"\"\n\n\nclass SPImporter():\n \"\"\"\n Class handling the process of special phrases importation into the database.\n\n Take a sp loader which load the phrases from an external source.\n \"\"\"\n def __init__(self, config: Configuration, conn: Connection,\n sp_loader: SpecialPhraseLoader) -> None:\n self.config = config\n self.db_connection = conn\n self.sp_loader = sp_loader\n self.statistics_handler = SpecialPhrasesImporterStatistics()\n self.black_list, self.white_list = self._load_white_and_black_lists()\n self.sanity_check_pattern = re.compile(r'^\\w+$')\n # This set will contain all existing phrases to be added.\n # It contains tuples with the following format: (label, class, type, operator)\n self.word_phrases: Set[Tuple[str, str, str, str]] = set()\n # This set will contain all existing place_classtype tables which doesn't match any\n # special phrases class/type on the wiki.\n self.table_phrases_to_delete: Set[str] = set()\n\n def get_classtype_pairs(self, min: int = 0) -> Set[Tuple[str, str]]:\n \"\"\"\n Returns list of allowed special phrases from the database,\n restricting to a list of combinations of classes and types\n which occur equal to or more than a specified amount of times.\n\n Default value for this is 0, which allows everything in database.\n \"\"\"\n db_combinations = set()\n\n query = f\"\"\"\n SELECT class AS CLS, type AS typ\n FROM placex\n GROUP BY class, type\n HAVING COUNT(*) >= {min}\n \"\"\"\n\n with self.db_connection.cursor() as db_cursor:\n db_cursor.execute(SQL(query))\n for row in db_cursor:\n db_combinations.add((row[0], row[1]))\n\n return db_combinations\n\n def import_phrases(self, tokenizer: AbstractTokenizer, should_replace: bool,\n min: int = 0) -> None:\n \"\"\"\n Iterate through all SpecialPhrases extracted from the\n loader and import them into the database.\n\n If should_replace is set to True only the loaded phrases\n will be kept into the database. All other phrases already\n in the database will be removed.\n \"\"\"\n LOG.warning('Special phrases importation starting')\n self._fetch_existing_place_classtype_tables()\n\n # Store pairs of class/type for further processing\n class_type_pairs = set()\n\n for phrase in self.sp_loader.generate_phrases():\n result = self._process_phrase(phrase)\n if result:\n class_type_pairs.add(result)\n\n self._create_classtype_table_and_indexes(class_type_pairs, min)\n if should_replace:\n self._remove_non_existent_tables_from_db()\n\n self.db_connection.commit()\n\n with tokenizer.name_analyzer() as analyzer:\n analyzer.update_special_phrases(self.word_phrases, should_replace)\n\n LOG.warning('Import done.')\n self.statistics_handler.notify_import_done()\n\n def _fetch_existing_place_classtype_tables(self) -> None:\n \"\"\"\n Fetch existing place_classtype tables.\n Fill the table_phrases_to_delete set of the class.\n \"\"\"\n query = \"\"\"\n SELECT table_name\n FROM information_schema.tables\n WHERE table_schema='public'\n AND table_name like 'place_classtype_%';\n \"\"\"\n with self.db_connection.cursor() as db_cursor:\n db_cursor.execute(SQL(query))\n for row in db_cursor:\n self.table_phrases_to_delete.add(row[0])\n\n def _load_white_and_black_lists(self) \\\n -> Tuple[Mapping[str, Sequence[str]], Mapping[str, Sequence[str]]]:\n \"\"\"\n Load white and black lists from phrases-settings.json.\n \"\"\"\n settings = self.config.load_sub_configuration('phrase-settings.json')\n\n return settings['blackList'], settings['whiteList']\n\n def _check_sanity(self, phrase: SpecialPhrase) -> bool:\n \"\"\"\n Check sanity of given inputs in case somebody added garbage in the wiki.\n If a bad class/type is detected the system will exit with an error.\n \"\"\"\n class_matchs = self.sanity_check_pattern.findall(phrase.p_class)\n type_matchs = self.sanity_check_pattern.findall(phrase.p_type)\n\n if not class_matchs or not type_matchs:\n LOG.warning(\"Bad class/type: %s=%s. It will not be imported\",\n phrase.p_class, phrase.p_type)\n return False\n return True\n\n def _process_phrase(self, phrase: SpecialPhrase) -> Optional[Tuple[str, str]]:\n \"\"\"\n Processes the given phrase by checking black and white list\n and sanity.\n Return the class/type pair corresponding to the phrase.\n \"\"\"\n\n # blacklisting: disallow certain class/type combinations\n if phrase.p_class in self.black_list.keys() \\\n and phrase.p_type in self.black_list[phrase.p_class]:\n return None\n\n # whitelisting: if class is in whitelist, allow only tags in the list\n if phrase.p_class in self.white_list.keys() \\\n and phrase.p_type not in self.white_list[phrase.p_class]:\n return None\n\n # sanity check, in case somebody added garbage in the wiki\n if not self._check_sanity(phrase):\n self.statistics_handler.notify_one_phrase_invalid()\n return None\n\n self.word_phrases.add((phrase.p_label, phrase.p_class,\n phrase.p_type, phrase.p_operator))\n\n return (phrase.p_class, phrase.p_type)\n\n def _create_classtype_table_and_indexes(self,\n class_type_pairs: Iterable[Tuple[str, str]],\n min: int = 0) -> None:\n \"\"\"\n Create table place_classtype for each given pair.\n Also create indexes on place_id and centroid.\n \"\"\"\n LOG.warning('Create tables and indexes...')\n\n sql_tablespace = self.config.TABLESPACE_AUX_DATA\n if sql_tablespace:\n sql_tablespace = ' TABLESPACE ' + sql_tablespace\n\n with self.db_connection.cursor() as db_cursor:\n db_cursor.execute(\"CREATE INDEX idx_placex_classtype ON placex (class, type)\")\n\n if min:\n allowed_special_phrases = self.get_classtype_pairs(min)\n\n for pair in class_type_pairs:\n phrase_class = pair[0]\n phrase_type = pair[1]\n\n # Will only filter if min is not 0\n if min and (phrase_class, phrase_type) not in allowed_special_phrases:\n LOG.warning(\"Skipping phrase %s=%s: not in allowed special phrases\",\n phrase_class, phrase_type)\n continue\n\n table_name = _classtype_table(phrase_class, phrase_type)\n\n if table_name in self.table_phrases_to_delete:\n self.statistics_handler.notify_one_table_ignored()\n # Remove this table from the ones to delete as it match a\n # class/type still existing on the special phrases of the wiki.\n self.table_phrases_to_delete.remove(table_name)\n # So don't need to create the table and indexes.\n continue\n\n # Table creation\n self._create_place_classtype_table(sql_tablespace, phrase_class, phrase_type)\n\n # Indexes creation\n self._create_place_classtype_indexes(sql_tablespace, phrase_class, phrase_type)\n\n # Grant access on read to the web user.\n self._grant_access_to_webuser(phrase_class, phrase_type)\n\n self.statistics_handler.notify_one_table_created()\n\n with self.db_connection.cursor() as db_cursor:\n db_cursor.execute(\"DROP INDEX idx_placex_classtype\")\n\n def _create_place_classtype_table(self, sql_tablespace: str,\n phrase_class: str, phrase_type: str) -> None:\n \"\"\"\n Create table place_classtype of the given phrase_class/phrase_type\n if doesn't exit.\n \"\"\"\n table_name = _classtype_table(phrase_class, phrase_type)\n with self.db_connection.cursor() as cur:\n cur.execute(SQL(\"\"\"CREATE TABLE IF NOT EXISTS {} {} AS\n SELECT place_id AS place_id,\n st_centroid(geometry) AS centroid\n FROM placex\n WHERE class = %s AND type = %s\n \"\"\").format(Identifier(table_name), SQL(sql_tablespace)),\n (phrase_class, phrase_type))\n\n def _create_place_classtype_indexes(self, sql_tablespace: str,\n phrase_class: str, phrase_type: str) -> None:\n \"\"\"\n Create indexes on centroid and place_id for the place_classtype table.\n \"\"\"\n index_prefix = f'idx_place_classtype_{phrase_class}_{phrase_type}_'\n base_table = _classtype_table(phrase_class, phrase_type)\n # Index on centroid\n if not index_exists(self.db_connection, index_prefix + 'centroid'):\n with self.db_connection.cursor() as db_cursor:\n db_cursor.execute(SQL(\"CREATE INDEX {} ON {} USING GIST (centroid) {}\")\n .format(Identifier(index_prefix + 'centroid'),\n Identifier(base_table),\n SQL(sql_tablespace)))\n\n # Index on place_id\n if not index_exists(self.db_connection, index_prefix + 'place_id'):\n with self.db_connection.cursor() as db_cursor:\n db_cursor.execute(SQL(\"CREATE INDEX {} ON {} USING btree(place_id) {}\")\n .format(Identifier(index_prefix + 'place_id'),\n Identifier(base_table),\n SQL(sql_tablespace)))\n\n def _grant_access_to_webuser(self, phrase_class: str, phrase_type: str) -> None:\n \"\"\"\n Grant access on read to the table place_classtype for the webuser.\n \"\"\"\n table_name = _classtype_table(phrase_class, phrase_type)\n with self.db_connection.cursor() as db_cursor:\n db_cursor.execute(SQL(\"\"\"GRANT SELECT ON {} TO {}\"\"\")\n .format(Identifier(table_name),\n Identifier(self.config.DATABASE_WEBUSER)))\n\n def _remove_non_existent_tables_from_db(self) -> None:\n \"\"\"\n Remove special phrases which doesn't exist on the wiki anymore.\n Delete the place_classtype tables.\n \"\"\"\n LOG.warning('Cleaning database...')\n\n # Delete place_classtype tables corresponding to class/type which\n # are not on the wiki anymore.\n drop_tables(self.db_connection, *self.table_phrases_to_delete)\n for _ in self.table_phrases_to_delete:\n self.statistics_handler.notify_one_table_deleted()\n"
},
{
"path": "src/nominatim_db/tools/special_phrases/sp_wiki_loader.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\n Module containing the SPWikiLoader class.\n\"\"\"\nfrom typing import Iterable\nimport re\nimport logging\n\nimport mwparserfromhell\n\nfrom ...config import Configuration\nfrom ...utils.url_utils import get_url\nfrom .special_phrase import SpecialPhrase\n\nLOG = logging.getLogger()\n\n\ndef _get_wiki_content(lang: str) -> str:\n \"\"\"\n Request and return the wiki page's content\n corresponding to special phrases for a given lang.\n Requested URL Example :\n https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/EN\n \"\"\"\n url = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/' \\\n + lang.upper()\n return get_url(url)\n\n\nclass SPWikiLoader:\n \"\"\"\n Handles loading of special phrases from the wiki.\n \"\"\"\n def __init__(self, config: Configuration) -> None:\n self.config = config\n # Hack around a bug where building=yes was imported with quotes into the wiki\n self.type_fix_pattern = re.compile(r'\\\"|"')\n\n self.languages = self.config.get_str_list('LANGUAGES') or \\\n ['af', 'ar', 'br', 'ca', 'cs', 'de', 'en', 'es',\n 'et', 'eu', 'fa', 'fi', 'fr', 'gl', 'hr', 'hu',\n 'ia', 'is', 'it', 'ja', 'mk', 'nl', 'no', 'pl',\n 'ps', 'pt', 'ru', 'sk', 'sl', 'sv', 'uk', 'vi',\n 'lv', 'tr']\n\n def generate_phrases(self) -> Iterable[SpecialPhrase]:\n \"\"\" Download the wiki pages for the configured languages\n and extract the phrases from the page.\n \"\"\"\n for lang in self.languages:\n LOG.warning('Importing phrases for lang: %s...', lang)\n loaded_xml = _get_wiki_content(lang)\n\n wikicode = mwparserfromhell.parse(loaded_xml)\n\n for table in wikicode.filter_tags(matches=lambda t: t.tag == 'table'):\n for row in table.contents.filter_tags(matches=lambda t: t.tag == 'tr'):\n cells = list(row.contents.filter_tags(matches=lambda t: t.tag == 'td'))\n\n if len(cells) < 5:\n continue\n\n label = cells[0].contents.strip_code().strip()\n cls = cells[1].contents.strip_code().strip()\n typ = cells[2].contents.strip_code().strip()\n operator = cells[3].contents.strip_code().strip()\n\n yield SpecialPhrase(label,\n cls,\n self.type_fix_pattern.sub('', typ),\n operator)\n"
},
{
"path": "src/nominatim_db/tools/special_phrases/special_phrase.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\n Module containing the class SpecialPhrase.\n\n This class is a model used to transfer a special phrase through\n the process of load and importation.\n\"\"\"\nfrom typing import Any\n\n\nclass SpecialPhrase:\n \"\"\"\n Model representing a special phrase.\n \"\"\"\n def __init__(self, p_label: str, p_class: str, p_type: str, p_operator: str) -> None:\n self.p_label = p_label.strip()\n self.p_class = p_class.strip()\n self.p_type = p_type.strip()\n # Needed if some operator in the wiki are not written in english\n p_operator = p_operator.strip().lower()\n self.p_operator = '-' if p_operator not in ('near', 'in') else p_operator\n\n def __eq__(self, other: Any) -> bool:\n if not isinstance(other, SpecialPhrase):\n return False\n\n return self.p_label == other.p_label \\\n and self.p_class == other.p_class \\\n and self.p_type == other.p_type \\\n and self.p_operator == other.p_operator\n\n def __hash__(self) -> int:\n return hash((self.p_label, self.p_class, self.p_type, self.p_operator))\n"
},
{
"path": "src/nominatim_db/tools/tiger_data.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for importing tiger data and handling tarbar and directory files\n\"\"\"\nfrom typing import Any, TextIO, List, Union, cast, Iterator, Dict\nimport csv\nimport io\nimport logging\nimport os\nimport tarfile\n\nfrom psycopg.types.json import Json\n\nfrom ..config import Configuration\nfrom ..db.connection import connect, table_exists\nfrom ..db.sql_preprocessor import SQLPreprocessor\nfrom ..errors import UsageError\nfrom ..db.query_pool import QueryPool\nfrom ..data.place_info import PlaceInfo\nfrom ..tokenizer.base import AbstractTokenizer\n\nLOG = logging.getLogger()\n\n\nclass TigerInput:\n \"\"\" Context manager that goes through Tiger input files which may\n either be in a directory or gzipped together in a tar file.\n \"\"\"\n\n def __init__(self, data_dir: str) -> None:\n self.tar_handle = None\n self.files: List[Union[str, tarfile.TarInfo]] = []\n\n if data_dir.endswith('.tar.gz'):\n try:\n self.tar_handle = tarfile.open(data_dir)\n except tarfile.ReadError as err:\n LOG.fatal(\"Cannot open '%s'. Is this a tar file?\", data_dir)\n raise UsageError(\"Cannot open Tiger data file.\") from err\n\n self.files = [i for i in self.tar_handle.getmembers() if i.name.endswith('.csv')]\n LOG.warning(\"Found %d CSV files in tarfile with path %s\", len(self.files), data_dir)\n else:\n files = os.listdir(data_dir)\n self.files = [os.path.join(data_dir, i) for i in files if i.endswith('.csv')]\n LOG.warning(\"Found %d CSV files in path %s\", len(self.files), data_dir)\n\n if not self.files:\n LOG.warning(\"Tiger data import selected but no files found at %s\", data_dir)\n\n def __enter__(self) -> 'TigerInput':\n return self\n\n def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:\n if self.tar_handle:\n self.tar_handle.close()\n self.tar_handle = None\n\n def __bool__(self) -> bool:\n return bool(self.files)\n\n def get_file(self, fname: Union[str, tarfile.TarInfo]) -> TextIO:\n \"\"\" Return a file handle to the next file to be processed.\n Raises an IndexError if there is no file left.\n \"\"\"\n if self.tar_handle is not None:\n extracted = self.tar_handle.extractfile(fname)\n assert extracted is not None\n return io.TextIOWrapper(extracted, encoding='utf-8')\n\n return open(cast(str, fname), encoding='utf-8')\n\n def __iter__(self) -> Iterator[Dict[str, Any]]:\n \"\"\" Iterate over the lines in each file.\n \"\"\"\n for fname in self.files:\n fd = self.get_file(fname)\n yield from csv.DictReader(fd, delimiter=';')\n\n\nasync def add_tiger_data(data_dir: str, config: Configuration, threads: int,\n tokenizer: AbstractTokenizer) -> int:\n \"\"\" Import tiger data from directory or tar file `data dir`.\n \"\"\"\n dsn = config.get_libpq_dsn()\n\n with TigerInput(data_dir) as tar:\n if not tar:\n return 1\n\n with connect(dsn) as conn:\n sql = SQLPreprocessor(conn, config)\n\n if not table_exists(conn, 'search_name'):\n raise UsageError(\n \"Cannot perform tiger import: required tables are missing. \"\n \"See https://github.com/osm-search/Nominatim/issues/2463 for details.\"\n )\n\n sql.run_sql_file(conn, 'tiger_import_start.sql')\n\n # Reading files and then for each file line handling\n # sql_query in chunks.\n place_threads = max(1, threads - 1)\n\n async with QueryPool(dsn, place_threads, autocommit=True) as pool:\n with tokenizer.name_analyzer() as analyzer:\n for lineno, row in enumerate(tar, 1):\n try:\n address = dict(street=row['street'], postcode=row['postcode'])\n args = ('SRID=4326;' + row['geometry'],\n int(row['from']), int(row['to']), row['interpolation'],\n Json(analyzer.process_place(PlaceInfo({'address': address}))),\n analyzer.normalize_postcode(row['postcode']))\n except ValueError:\n continue\n\n await pool.put_query(\n \"\"\"SELECT tiger_line_import(%s::GEOMETRY, %s::INT,\n %s::INT, %s::TEXT, %s::JSONB, %s::TEXT)\"\"\",\n args)\n\n if not lineno % 1000:\n print('.', end='', flush=True)\n\n print('', flush=True)\n\n LOG.warning(\"Creating indexes on Tiger data\")\n with connect(dsn) as conn:\n sql = SQLPreprocessor(conn, config)\n sql.run_sql_file(conn, 'tiger_import_finish.sql')\n\n return 0\n"
},
{
"path": "src/nominatim_db/typing.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nType definitions for typing annotations.\n\nComplex type definitions are moved here, to keep the source files readable.\n\"\"\"\nfrom typing import Any, Union, Mapping, TypeVar, Sequence, TYPE_CHECKING\n\n\nif TYPE_CHECKING:\n import os\n\nStrPath = Union[str, 'os.PathLike[str]']\n\nSysEnv = Mapping[str, str]\n\n# psycopg-related types\n\nT_ResultKey = TypeVar('T_ResultKey', int, str)\n\n\nclass DictCursorResult(Mapping[str, Any]):\n def __getitem__(self, x: Union[int, str]) -> Any: ...\n\n\nDictCursorResults = Sequence[DictCursorResult]\n\n# The following typing features require typing_extensions to work\n# on all supported Python versions.\n# Only require this for type checking but not for normal operations.\n\nif TYPE_CHECKING:\n from typing_extensions import (Protocol as Protocol,\n Final as Final,\n TypedDict as TypedDict)\nelse:\n Protocol = object\n Final = 'Final'\n TypedDict = dict\n"
},
{
"path": "src/nominatim_db/utils/__init__.py",
"content": ""
},
{
"path": "src/nominatim_db/utils/centroid.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFunctions for computation of centroids.\n\"\"\"\nfrom typing import Tuple, Any\nfrom collections.abc import Collection\n\n\nclass PointsCentroid:\n \"\"\" Centroid computation from single points using an online algorithm.\n More points may be added at any time.\n\n Coordinates are internally treated as a 7-digit fixed-point float\n (i.e. in OSM style).\n \"\"\"\n\n def __init__(self) -> None:\n self.sum_x = 0\n self.sum_y = 0\n self.count = 0\n\n def centroid(self) -> Tuple[float, float]:\n \"\"\" Return the centroid of all points collected so far.\n \"\"\"\n if self.count == 0:\n raise ValueError(\"No points available for centroid.\")\n\n return (self.sum_x / self.count / 10_000_000,\n self.sum_y / self.count / 10_000_000)\n\n def __len__(self) -> int:\n return self.count\n\n def __iadd__(self, other: Any) -> 'PointsCentroid':\n if isinstance(other, Collection) and len(other) == 2:\n if all(isinstance(p, (float, int)) for p in other):\n x, y = other\n self.sum_x += int(x * 10_000_000)\n self.sum_y += int(y * 10_000_000)\n self.count += 1\n return self\n\n raise ValueError(\"Can only add 2-element tuples to centroid.\")\n"
},
{
"path": "src/nominatim_db/utils/url_utils.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2024 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nHelper functions for accessing URL.\n\"\"\"\nfrom typing import IO # noqa\nimport logging\nimport urllib.request as urlrequest\n\nfrom ..version import NOMINATIM_VERSION\n\nLOG = logging.getLogger()\n\n\ndef get_url(url: str) -> str:\n \"\"\" Get the contents from the given URL and return it as a UTF-8 string.\n\n This version makes sure that an appropriate user agent is sent.\n \"\"\"\n headers = {\"User-Agent\": f\"Nominatim/{NOMINATIM_VERSION!s}\"}\n\n try:\n request = urlrequest.Request(url, headers=headers)\n with urlrequest.urlopen(request) as response: # type: IO[bytes]\n return response.read().decode('utf-8')\n except Exception:\n LOG.fatal('Failed to load URL: %s', url)\n raise\n"
},
{
"path": "src/nominatim_db/version.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2026 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nVersion information for Nominatim.\n\"\"\"\nfrom typing import NamedTuple, Optional\n\n\nclass NominatimVersion(NamedTuple):\n \"\"\" Version information for Nominatim. We follow semantic versioning.\n\n Major, minor and patch_level refer to the last released version.\n The database patch level tracks important changes between releases\n and must always be increased when there is a change to the database or code\n that requires a migration.\n\n When adding a migration on the development branch, raise the patch level\n to 99 to make sure that the migration is applied when updating from a\n patch release to the next minor version. Patch releases usually shouldn't\n have migrations in them. When they are needed, then make sure that the\n migration can be reapplied and set the migration version to the appropriate\n patch level when cherry-picking the commit with the migration.\n \"\"\"\n\n major: int\n minor: int\n patch_level: int\n db_patch_level: int\n\n def __str__(self) -> str:\n if self.db_patch_level is None:\n return f\"{self.major}.{self.minor}.{self.patch_level}\"\n\n return f\"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}\"\n\n def release_version(self) -> str:\n \"\"\" Return the release version in semantic versioning format.\n\n The release version does not include the database patch version.\n \"\"\"\n return f\"{self.major}.{self.minor}.{self.patch_level}\"\n\n\ndef parse_version(version: str) -> NominatimVersion:\n \"\"\" Parse a version string into a version consisting of a tuple of\n four ints: major, minor, patch level, database patch level\n\n This is the reverse operation of `version_str()`.\n \"\"\"\n parts = version.split('.')\n return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')])\n\n\nNOMINATIM_VERSION = parse_version('5.3.99-0')\n\nPOSTGRESQL_REQUIRED_VERSION = (12, 0)\nPOSTGIS_REQUIRED_VERSION = (3, 0)\nOSM2PGSQL_REQUIRED_VERSION = (1, 8)\n\n# Cmake sets a variable @GIT_HASH@ by executing 'git --log'. It is not run\n# on every execution of 'make'.\n# cmake/tool-installed.tmpl is used to build the binary 'nominatim'. Inside\n# there is a call to set the variable value below.\nGIT_COMMIT_HASH: Optional[str] = None\n"
},
{
"path": "test/bdd/conftest.py",
"content": "# SPDX-License-Identifier: GPL-3.0-or-later\n#\n# This file is part of Nominatim. (https://nominatim.org)\n#\n# Copyright (C) 2025 by the Nominatim developer community.\n# For a full list of authors see the git log.\n\"\"\"\nFixtures for BDD test steps\n\"\"\"\nimport sys\nimport json\nimport re\nfrom pathlib import Path\n\nimport psycopg\nfrom psycopg import sql as pysql\n\n# always test against the source\nSRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()\nsys.path.insert(0, str(SRC_DIR / 'src'))\n\nimport pytest\nfrom pytest_bdd.parsers import re as step_parse\nfrom pytest_bdd import given, when, then, scenario\nfrom pytest_bdd.feature import get_features\n\npytest.register_assert_rewrite('utils')\n\nfrom utils.api_runner import APIRunner\nfrom utils.api_result import APIResult\nfrom utils.checks import ResultAttr, COMPARATOR_TERMS\nfrom utils.geometry_alias import ALIASES\nfrom utils.grid import Grid\nfrom utils.db import DBManager\n\nfrom nominatim_db.config import Configuration\nfrom nominatim_db.data.country_info import setup_country_config\n\n\ndef _strlist(inp):\n return [s.strip() for s in inp.split(',')]\n\n\ndef _pretty_json(inp):\n return json.dumps(inp, indent=2)\n\n\ndef pytest_addoption(parser, pluginmanager):\n parser.addoption('--nominatim-purge', dest='NOMINATIM_PURGE', action='store_true',\n help='Force recreation of test databases from scratch.')\n parser.addoption('--nominatim-keep-db', dest='NOMINATIM_KEEP_DB', action='store_true',\n help='Do not drop the database after tests are finished.')\n parser.addoption('--nominatim-api-engine', dest='NOMINATIM_API_ENGINE',\n default='falcon',\n help='Chose the API engine to use when sending requests.')\n parser.addoption('--nominatim-tokenizer', dest='NOMINATIM_TOKENIZER',\n metavar='TOKENIZER',\n help='Use the specified tokenizer for importing data into '\n 'a Nominatim database.')\n\n parser.addini('nominatim_test_db', default='test_nominatim',\n help='Name of the database used for running a single test.')\n parser.addini('nominatim_api_test_db', default='test_api_nominatim',\n help='Name of the database for storing API test data.')\n parser.addini('nominatim_template_db', default='test_template_nominatim',\n help='Name of database used as a template for test databases.')\n\n\n@pytest.fixture\ndef datatable():\n \"\"\" Default fixture for datatables, so that their presence can be optional.\n \"\"\"\n return None\n\n\n@pytest.fixture\ndef node_grid():\n \"\"\" Default fixture for node grids. Nothing set.\n \"\"\"\n return Grid([[]], None, None)\n\n\n@pytest.fixture(scope='session', autouse=True)\ndef setup_country_info():\n setup_country_config(Configuration(None))\n\n\n@pytest.fixture(scope='session')\ndef template_db(pytestconfig):\n \"\"\" Create a template database containing the extensions and base data\n needed by Nominatim. Using the template instead of doing the full\n setup can speed up the tests.\n\n The template database will only be created if it does not exist yet\n or a purge has been explicitly requested.\n \"\"\"\n dbm = DBManager(purge=pytestconfig.option.NOMINATIM_PURGE)\n\n template_db = pytestconfig.getini('nominatim_template_db')\n\n template_config = Configuration(\n None, environ={'NOMINATIM_DATABASE_DSN': f\"pgsql:dbname={template_db}\"})\n\n dbm.setup_template_db(template_config)\n\n return template_db\n\n\n@pytest.fixture\ndef def_config(pytestconfig):\n dbname = pytestconfig.getini('nominatim_test_db')\n\n return Configuration(None,\n environ={'NOMINATIM_DATABASE_DSN': f\"pgsql:dbname={dbname}\"})\n\n\n@pytest.fixture\ndef db(template_db, pytestconfig):\n \"\"\" Set up an empty database for use with osm2pgsql.\n \"\"\"\n dbm = DBManager(purge=pytestconfig.option.NOMINATIM_PURGE)\n\n dbname = pytestconfig.getini('nominatim_test_db')\n\n dbm.create_db_from_template(dbname, template_db)\n\n yield dbname\n\n if not pytestconfig.option.NOMINATIM_KEEP_DB:\n dbm.drop_db(dbname)\n\n\n@pytest.fixture\ndef db_conn(db, def_config):\n with psycopg.connect(def_config.get_libpq_dsn()) as conn:\n info = psycopg.types.TypeInfo.fetch(conn, \"hstore\")\n psycopg.types.hstore.register_hstore(info, conn)\n yield conn\n\n\n@when(step_parse(r'reverse geocoding (?P[\\d.-]*),(?P[\\d.-]*)'),\n target_fixture='nominatim_result')\ndef reverse_geocode_via_api(test_config_env, pytestconfig, datatable, lat, lon):\n runner = APIRunner(test_config_env, pytestconfig.option.NOMINATIM_API_ENGINE)\n api_response = runner.run_step('reverse',\n {'lat': float(lat), 'lon': float(lon)},\n datatable, 'jsonv2', {})\n\n assert api_response.status == 200\n assert api_response.headers['content-type'] == 'application/json; charset=utf-8'\n\n result = APIResult('json', 'reverse', api_response.body)\n assert result.is_simple()\n\n assert isinstance(result.result['lat'], str)\n assert isinstance(result.result['lon'], str)\n result.result['centroid'] = f\"POINT({result.result['lon']} {result.result['lat']})\"\n\n return result\n\n\n@when(step_parse(r'reverse geocoding at node (?P[\\d]+)'),\n target_fixture='nominatim_result')\ndef reverse_geocode_via_api_and_grid(test_config_env, pytestconfig, node_grid, datatable, node):\n coords = node_grid.get(node)\n if coords is None:\n raise ValueError('Unknown node id')\n\n return reverse_geocode_via_api(test_config_env, pytestconfig, datatable, coords[1], coords[0])\n\n\n@when(step_parse(r'geocoding(?: \"(?P.*)\")?'),\n target_fixture='nominatim_result')\ndef forward_geocode_via_api(test_config_env, pytestconfig, datatable, query):\n runner = APIRunner(test_config_env, pytestconfig.option.NOMINATIM_API_ENGINE)\n\n params = {'addressdetails': '1'}\n if query:\n params['q'] = query\n\n api_response = runner.run_step('search', params, datatable, 'jsonv2', {})\n\n assert api_response.status == 200\n assert api_response.headers['content-type'] == 'application/json; charset=utf-8'\n\n result = APIResult('json', 'search', api_response.body)\n assert not result.is_simple()\n\n for res in result.result:\n assert isinstance(res['lat'], str)\n assert isinstance(res['lon'], str)\n res['centroid'] = f\"POINT({res['lon']} {res['lat']})\"\n\n return result\n\n\n@then(step_parse(r'(?P[a-z ]+) (?P\\d+) results? (?:are|is) returned'),\n converters={'num': int})\ndef check_number_of_results(nominatim_result, op, num):\n assert not nominatim_result.is_simple()\n assert COMPARATOR_TERMS[op](num, len(nominatim_result))\n\n\n@then(step_parse('the result metadata contains'))\ndef check_metadata_for_fields(nominatim_result, datatable):\n if datatable[0] == ['param', 'value']:\n pairs = datatable[1:]\n else:\n pairs = zip(datatable[0], datatable[1])\n\n for k, v in pairs:\n assert ResultAttr(nominatim_result.meta, k) == v\n\n\n@then(step_parse('the result metadata has no attributes (?P.*)'),\n converters={'attributes': _strlist})\ndef check_metadata_for_field_presence(nominatim_result, attributes):\n assert all(a not in nominatim_result.meta for a in attributes), \\\n f\"Unexpectedly have one of the attributes '{attributes}' in\\n\" \\\n f\"{_pretty_json(nominatim_result.meta)}\"\n\n\n@then(step_parse(r'the result contains(?: in field (?P\\S+))?'))\ndef check_result_for_fields(nominatim_result, datatable, node_grid, field):\n assert nominatim_result.is_simple()\n\n if datatable[0] == ['param', 'value']:\n pairs = datatable[1:]\n else:\n pairs = zip(datatable[0], datatable[1])\n\n prefix = field + '+' if field else ''\n\n for k, v in pairs:\n assert ResultAttr(nominatim_result.result, prefix + k, grid=node_grid) == v\n\n\n@then(step_parse('the result has attributes (?P.*)'),\n converters={'attributes': _strlist})\ndef check_result_for_field_presence(nominatim_result, attributes):\n assert nominatim_result.is_simple()\n assert all(a in nominatim_result.result for a in attributes)\n\n\n@then(step_parse('the result has no attributes (?P.*)'),\n converters={'attributes': _strlist})\ndef check_result_for_field_absence(nominatim_result, attributes):\n assert nominatim_result.is_simple()\n assert all(a not in nominatim_result.result for a in attributes)\n\n\n@then(step_parse(\n r'the result contains array field (?P\\S+) where element (?P\\d+) contains'),\n converters={'num': int})\ndef check_result_array_field_for_attributes(nominatim_result, datatable, field, num):\n assert nominatim_result.is_simple()\n\n if datatable[0] == ['param', 'value']:\n pairs = datatable[1:]\n else:\n pairs = zip(datatable[0], datatable[1])\n\n prefix = f\"{field}+{num}+\"\n\n for k, v in pairs:\n assert ResultAttr(nominatim_result.result, prefix + k) == v\n\n\n@then(step_parse('the result set contains(?P exactly)?'))\ndef check_result_list_match(nominatim_result, datatable, exact):\n assert not nominatim_result.is_simple()\n\n result_set = set(range(len(nominatim_result.result)))\n\n for row in datatable[1:]:\n for idx in result_set:\n for key, value in zip(datatable[0], row):\n if ResultAttr(nominatim_result.result[idx], key) != value:\n break\n else:\n # found a match\n result_set.remove(idx)\n break\n else:\n assert False, f\"Missing data row {row}. Full response:\\n{nominatim_result}\"\n\n if exact:\n assert not [nominatim_result.result[i] for i in result_set]\n\n\n@then(step_parse('all results have attributes (?P.*)'),\n converters={'attributes': _strlist})\ndef check_all_results_for_field_presence(nominatim_result, attributes):\n assert not nominatim_result.is_simple()\n assert len(nominatim_result) > 0\n for res in nominatim_result.result:\n assert all(a in res for a in attributes), \\\n f\"Missing one of the attributes '{attributes}' in\\n{_pretty_json(res)}\"\n\n\n@then(step_parse('all results have no attributes (?P.*)'),\n converters={'attributes': _strlist})\ndef check_all_result_for_field_absence(nominatim_result, attributes):\n assert not nominatim_result.is_simple()\n assert len(nominatim_result) > 0\n for res in nominatim_result.result:\n assert all(a not in res for a in attributes), \\\n f\"Unexpectedly have one of the attributes '{attributes}' in\\n{_pretty_json(res)}\"\n\n\n@then(step_parse(r'all results contain(?: in field (?P\\S+))?'))\ndef check_all_results_contain(nominatim_result, datatable, node_grid, field):\n assert not nominatim_result.is_simple()\n assert len(nominatim_result) > 0\n\n if datatable[0] == ['param', 'value']:\n pairs = datatable[1:]\n else:\n pairs = zip(datatable[0], datatable[1])\n\n prefix = field + '+' if field else ''\n\n for k, v in pairs:\n for r in nominatim_result.result:\n assert ResultAttr(r, prefix + k, grid=node_grid) == v\n\n\n@then(step_parse(r'result (?P\\d+) contains(?: in field (?P\\S+))?'),\n converters={'num': int})\ndef check_specific_result_for_fields(nominatim_result, datatable, num, field):\n assert not nominatim_result.is_simple()\n assert len(nominatim_result) > num\n\n if datatable[0] == ['param', 'value']:\n pairs = datatable[1:]\n else:\n pairs = zip(datatable[0], datatable[1])\n\n prefix = field + '+' if field else ''\n\n for k, v in pairs:\n assert ResultAttr(nominatim_result.result[num], prefix + k) == v\n\n\n@given(step_parse(r'the (?P[0-9.]+ )?grid(?: with origin (?P.*))?'),\n target_fixture='node_grid')\ndef set_node_grid(datatable, step, origin):\n if step is not None:\n step = float(step)\n\n if origin:\n if ',' in origin:\n coords = origin.split(',')\n if len(coords) != 2:\n raise RuntimeError('Grid origin expects origin with x,y coordinates.')\n origin = list(map(float, coords))\n elif origin in ALIASES:\n origin = ALIASES[origin]\n else:\n raise RuntimeError('Grid origin must be either coordinate or alias.')\n\n return Grid(datatable, step, origin)\n\n\n@then(step_parse('(?Pplacex?) has no entry for '\n r'(?P[NRW])(?P\\d+)(?::(?P\\S+))?'),\n converters={'osm_id': int})\ndef check_place_missing_lines(db_conn, table, osm_type, osm_id, osm_class):\n sql = pysql.SQL(\"\"\"SELECT count(*) FROM {}\n WHERE osm_type = %s and osm_id = %s\"\"\").format(pysql.Identifier(table))\n params = [osm_type, int(osm_id)]\n if osm_class:\n sql += pysql.SQL(' AND class = %s')\n params.append(osm_class)\n\n with db_conn.cursor() as cur:\n assert cur.execute(sql, params).fetchone()[0] == 0\n\n\nif pytest.version_tuple >= (8, 0, 0):\n def pytest_pycollect_makemodule(module_path, parent):\n return BddTestCollector.from_parent(parent, path=module_path)\n\n\nclass BddTestCollector(pytest.Module):\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n def collect(self):\n for item in super().collect():\n yield item\n\n if hasattr(self.obj, 'PYTEST_BDD_SCENARIOS'):\n for path in self.obj.PYTEST_BDD_SCENARIOS:\n for feature in get_features([str(Path(self.path.parent, path).resolve())]):\n yield FeatureFile.from_parent(self,\n name=str(Path(path, feature.rel_filename)),\n path=Path(feature.filename),\n feature=feature)\n\n\n# borrowed from pytest-bdd: src/pytest_bdd/scenario.py\ndef make_python_name(string: str) -> str:\n \"\"\"Make python attribute name out of a given string.\"\"\"\n string = re.sub(r\"\\W\", \"\", string.replace(\" \", \"_\"))\n return re.sub(r\"^\\d+_*\", \"\", string).lower()\n\n\nclass FeatureFile(pytest.File):\n class obj:\n pass\n\n def __init__(self, feature, **kwargs):\n self.feature = feature\n super().__init__(**kwargs)\n\n def collect(self):\n for sname, sobject in self.feature.scenarios.items():\n class_name = f\"L{sobject.line_number}\"\n test_name = \"test_\" + make_python_name(sname)\n\n @scenario(self.feature.filename, sname)\n def _test():\n pass\n\n tclass = type(class_name, (),\n {test_name: staticmethod(_test)})\n setattr(self.obj, class_name, tclass)\n\n yield pytest.Class.from_parent(self, name=class_name, obj=tclass)\n"
},
{
"path": "test/bdd/features/api/details/language.feature",
"content": "Feature: Localization of search results\n\n Scenario: default language\n When sending v1/details\n | osmtype | osmid |\n | R | 1155955 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | localname |\n | Liechtenstein |\n\n Scenario: accept-language first\n When sending v1/details\n | osmtype | osmid | accept-language |\n | R | 1155955 | zh,de |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | localname |\n | 列支敦士登 |\n\n Scenario: accept-language missing\n When sending v1/details\n | osmtype | osmid | accept-language |\n | R | 1155955 | xx,fr,en,de |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | localname |\n | Liechtenstein |\n\n Scenario: http accept language header first\n Given the HTTP header\n | accept-language |\n | fo;q=0.8,en-ca;q=0.5,en;q=0.3 |\n When sending v1/details\n | osmtype | osmid |\n | R | 1155955 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | localname |\n | Liktinstein |\n\n Scenario: http accept language header and accept-language\n Given the HTTP header\n | accept-language |\n | fr-ca,fr;q=0.8,en-ca;q=0.5,en;q=0.3 |\n When sending v1/details\n | osmtype | osmid | accept-language |\n | R | 1155955 | fo,en |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | localname |\n | Liktinstein |\n\n Scenario: http accept language header fallback\n Given the HTTP header\n | accept-language |\n | fo-ca,en-ca;q=0.5 |\n When sending v1/details\n | osmtype | osmid |\n | R | 1155955 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | localname |\n | Liktinstein |\n\n Scenario: http accept language header fallback (upper case)\n Given the HTTP header\n | accept-language |\n | fo-FR;q=0.8,en-ca;q=0.5 |\n When sending v1/details\n | osmtype | osmid |\n | R | 1155955 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | localname |\n | Liktinstein |\n"
},
{
"path": "test/bdd/features/api/details/params.feature",
"content": "Feature: Object details\n Testing different parameter options for details API.\n\n Scenario: Basic details\n When sending v1/details\n | osmtype | osmid |\n | W | 297699560 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes geometry\n And the result has no attributes keywords,address,linked_places,parentof\n And the result contains\n | geometry+type |\n | Point |\n\n Scenario: Basic details with pretty printing\n When sending v1/details\n | osmtype | osmid | pretty |\n | W | 297699560 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes geometry\n And the result has no attributes keywords,address,linked_places,parentof\n\n Scenario: Details with addressdetails\n When sending v1/details\n | osmtype | osmid | addressdetails |\n | W | 297699560 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes address\n\n Scenario: Details with entrances\n When sending v1/details\n | osmtype | osmid | entrances |\n | W | 429210603 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains array field entrances where element 0 contains\n | osm_id | type | lat | lon |\n | 6580031131 | yes | 47.2489382 | 9.5284033 |\n\n Scenario: Details with linkedplaces\n When sending v1/details\n | osmtype | osmid | linkedplaces |\n | R | 123924 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes linked_places\n\n Scenario: Details with hierarchy\n When sending v1/details\n | osmtype | osmid | hierarchy |\n | W | 297699560 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes hierarchy\n\n Scenario: Details with grouped hierarchy\n When sending v1/details\n | osmtype | osmid | hierarchy | group_hierarchy |\n | W | 297699560 | 1 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes hierarchy\n\n Scenario Outline: Details with keywords\n When sending v1/details\n | osmtype | osmid | keywords |\n | | | 1 |\n Then a HTTP 200 is returned\n Then the result is valid json\n And the result has attributes keywords\n\n Examples:\n | type | id |\n | W | 297699560 |\n | W | 243055645 |\n | W | 243055716 |\n | W | 43327921 |\n\n # ticket #1343\n Scenario: Details of a country with keywords\n When sending v1/details\n | osmtype | osmid | keywords |\n | R | 1155955 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes keywords\n\n Scenario Outline: Details with full geometry\n When sending v1/details\n | osmtype | osmid | polygon_geojson |\n | | | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes geometry\n And the result contains\n | geometry+type |\n | |\n\n Examples:\n | type | id | geometry |\n | W | 297699560 | LineString |\n | W | 243055645 | Polygon |\n | W | 243055716 | Polygon |\n | W | 43327921 | LineString |\n\n\n"
},
{
"path": "test/bdd/features/api/details/simple.feature",
"content": "Feature: Object details\n Check details page for correctness\n\n Scenario Outline: Details request with OSM id\n When sending v1/details\n | osmtype | osmid |\n | | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | osm_type | osm_id |\n | | |\n\n Examples:\n | type | id |\n | N | 5484325405 |\n | W | 43327921 |\n | R | 123924 |\n\n Scenario Outline: Details request with different class types for the same OSM id\n When sending v1/details\n | osmtype | osmid | class |\n | N | 300209696 | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | osm_type | osm_id | category |\n | N | 300209696 | |\n\n Examples:\n | class |\n | tourism |\n | mountain_pass |\n\n Scenario: Details request without osmtype\n When sending v1/details\n | osmid |\n | |\n Then a HTTP 400 is returned\n And the result is valid json\n\n Scenario: Details request with unknown OSM id\n When sending v1/details\n | osmtype | osmid |\n | R | 1 |\n Then a HTTP 404 is returned\n And the result is valid json\n\n Scenario: Details request with unknown class\n When sending v1/details\n | osmtype | osmid | class |\n | N | 300209696 | highway |\n Then a HTTP 404 is returned\n And the result is valid json\n\n Scenario: Details for interpolation way return the interpolation\n When sending v1/details\n | osmtype | osmid |\n | W | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | category | type | osm_type | osm_id | admin_level |\n | place | houses | W | 1 | 15 |\n\n\n @skip\n Scenario: Details for interpolation way return the interpolation\n When sending details query for 112871\n Then the result is valid json\n And the result contains\n | category | type | admin_level |\n | place | houses | 15 |\n And result has not attributes osm_type,osm_id\n\n\n @skip\n Scenario: Details for postcode\n When sending details query for 112820\n Then the result is valid json\n And the result contains\n | category | type | admin_level |\n | place | postcode | 15 |\n And result has not attributes osm_type,osm_id\n\n\n Scenario Outline: Details debug output returns no errors\n When sending v1/details\n | osmtype | osmid | debug |\n | | | 1 |\n Then a HTTP 200 is returned\n And the result is valid html\n\n Examples:\n | type | id |\n | N | 5484325405 |\n | W | 43327921 |\n | R | 123924 |\n\n"
},
{
"path": "test/bdd/features/api/lookup/simple.feature",
"content": "Feature: Tests for finding places by osm_type and osm_id\n Simple tests for response format.\n\n Scenario Outline: Address lookup for existing object\n When sending v1/lookup with format \n | osm_ids |\n | N5484325405,W43327921,,R123924,X99,N0 |\n Then a HTTP 200 is returned\n And the result is valid \n And exactly 3 results are returned\n\n Examples:\n | format | outformat |\n | xml | xml |\n | json | json |\n | jsonv2 | json |\n | geojson | geojson |\n | geocodejson | geocodejson |\n\n Scenario: Address lookup for non-existing or invalid object\n When sending v1/lookup\n | osm_ids |\n | X99,,N0,nN158845944,ABC,,W9 |\n Then a HTTP 200 is returned\n And the result is valid xml\n And exactly 0 results are returned\n\n Scenario Outline: Boundingbox is returned\n When sending v1/lookup with format \n | osm_ids |\n | N5484325405,W43327921 |\n Then the result is valid \n And the result set contains exactly\n | object | boundingbox!in_box |\n | N5484325405 | 47.135,47.14,9.52,9.525 |\n | W43327921 | 47.07,47.08,9.50,9.52 |\n\n Examples:\n | format | outformat |\n | xml | xml |\n | json | json |\n | jsonv2 | json |\n | geojson | geojson |\n\n Scenario Outline: Lookup with entrances\n When sending v1/lookup with format \n | osm_ids | entrances |\n | W429210603 | 1 |\n Then a HTTP 200 is returned\n And the result is valid \n And result 0 contains in field entrances+0\n | osm_id | type | lat | lon |\n | 6580031131 | yes | 47.2489382 | 9.5284033 |\n\n Examples:\n | format | outformat |\n | json | json |\n | jsonv2 | json |\n | geojson | geojson |\n\n Scenario: Linked places return information from the linkee\n When sending v1/lookup with format geocodejson\n | osm_ids |\n | N1932181216 |\n Then the result is valid geocodejson\n And exactly 1 result is returned\n And all results contain\n | name |\n | Vaduz |\n\n Scenario Outline: Force error by providing too many ids\n When sending v1/lookup with format \n | osm_ids |\n | N1,N2,N3,N4,N5,N6,N7,N8,N9,N10,N11,N12,N13,N14,N15,N16,N17,N18,N19,N20,N21,N22,N23,N24,N25,N26,N27,N28,N29,N30,N31,N32,N33,N34,N35,N36,N37,N38,N39,N40,N41,N42,N43,N44,N45,N46,N47,N48,N49,N50,N51 |\n Then a HTTP 400 is returned\n And the result is valid \n And the result contains\n | error+code | error+message |\n | 400 | Too many object IDs. |\n\n Examples:\n | format | outformat |\n | xml | xml |\n | json | json |\n | jsonv2 | json |\n | geojson | json |\n | geocodejson | json |\n"
},
{
"path": "test/bdd/features/api/reverse/geometry.feature",
"content": "Feature: Geometries for reverse geocoding\n Tests for returning geometries with reverse\n\n Scenario: Reverse - polygons are returned fully by default\n When sending v1/reverse\n | lat | lon | polygon_text |\n | 47.13803 | 9.52264 | 1 |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result contains\n | geotext!fm |\n | POLYGON\\(\\(9.5225302 47.138066, ?9.5225348 47.1379282, ?9.5226142 47.1379294, ?9.5226143 47.1379257, ?9.522615 47.137917, ?9.5226225 47.1379098, ?9.5226334 47.1379052, ?9.5226461 47.1379037, ?9.5226588 47.1379056, ?9.5226693 47.1379107, ?9.5226762 47.1379181, ?9.5226762 47.1379268, ?9.5226761 47.1379308, ?9.5227366 47.1379317, ?9.5227352 47.1379753, ?9.5227608 47.1379757, ?9.5227595 47.1380148, ?9.5227355 47.1380145, ?9.5227337 47.1380692, ?9.5225302 47.138066\\)\\) |\n\n\n Scenario: Reverse - polygons can be slightly simplified\n When sending v1/reverse\n | lat | lon | polygon_text | polygon_threshold |\n | 47.13803 | 9.52264 | 1 | 0.00001 |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result contains\n | geotext!fm |\n | POLYGON\\(\\(9.5225302 47.138066, ?9.5225348 47.1379282, ?9.5226142 47.1379294, ?9.5226225 47.1379098, ?9.5226588 47.1379056, ?9.5226761 47.1379308, ?9.5227366 47.1379317, ?9.5227352 47.1379753, ?9.5227608 47.1379757, ?9.5227595 47.1380148, ?9.5227355 47.1380145, ?9.5227337 47.1380692, ?9.5225302 47.138066\\)\\) |\n\n\n Scenario: Reverse - polygons can be much simplified\n When sending v1/reverse\n | lat | lon | polygon_text | polygon_threshold |\n | 47.13803 | 9.52264 | 1 | 0.9 |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result contains\n | geotext!fm |\n | POLYGON\\(\\([0-9. ]+, ?[0-9. ]+, ?[0-9. ]+, ?[0-9. ]+(, ?[0-9. ]+)?\\)\\) |\n\n\n Scenario: Reverse - for polygons return the centroid as center point\n When sending v1/reverse\n | lat | lon |\n | 47.13836 | 9.52304 |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result contains\n | lon | lat |\n | 9.5227108 | 47.1381805 |\n\n\n Scenario: Reverse - for streets return the closest point as center point\n When sending v1/reverse\n | lat | lon |\n | 47.13368 | 9.52942 |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result contains\n | lon | lat |\n | 9.5294315 | 47.1336817 |\n"
},
{
"path": "test/bdd/features/api/reverse/language.feature",
"content": "Feature: Localization of reverse search results\n\n Scenario: Reverse - default language\n When sending v1/reverse with format jsonv2\n | lat | lon |\n | 47.14 | 9.55 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | address+country |\n | Liechtenstein |\n\n Scenario: Reverse - accept-language parameter\n When sending v1/reverse with format jsonv2\n | lat | lon | accept-language |\n | 47.14 | 9.55 | ja,en |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | address+country |\n | リヒテンシュタイン |\n\n Scenario: Reverse - HTTP accept language header\n Given the HTTP header\n | accept-language |\n | fo-ca,fo;q=0.8,en-ca;q=0.5,en;q=0.3 |\n When sending v1/reverse with format jsonv2\n | lat | lon |\n | 47.14 | 9.55 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | address+country |\n | Liktinstein |\n\n Scenario: Reverse - accept-language parameter and HTTP header\n Given the HTTP header\n | accept-language |\n | fo-ca,fo;q=0.8,en-ca;q=0.5,en;q=0.3 |\n When sending v1/reverse with format jsonv2\n | lat | lon | accept-language |\n | 47.14 | 9.55 | en |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | address+country |\n | Liechtenstein |\n"
},
{
"path": "test/bdd/features/api/reverse/layers.feature",
"content": "Feature: Layer parameter in reverse geocoding\n Testing correct function of layer selection while reverse geocoding\n\n Scenario: POIs are selected by default\n When reverse geocoding 47.14077,9.52414\n Then the result contains\n | category | type |\n | tourism | viewpoint |\n\n Scenario Outline: Same address level POI with different layers\n When reverse geocoding 47.14077,9.52414\n | layer |\n | |\n Then the result contains\n | category |\n | |\n\n Examples:\n | layer | category |\n | address | highway |\n | poi,address | tourism |\n | address,poi | tourism |\n | natural | waterway |\n | address,natural | highway |\n | natural,poi | tourism |\n\n Scenario Outline: POIs are not selected without housenumber for address layer\n When reverse geocoding 47.13816,9.52168\n | layer |\n | |\n Then the result contains\n | category | type |\n | | |\n\n Examples:\n | layer | category | type |\n | address,poi | highway | bus_stop |\n | address | amenity | parking |\n\n Scenario: Between natural and low-zoom address prefer natural\n When reverse geocoding 47.13636,9.52094\n | layer | zoom |\n | natural,address | 15 |\n Then the result contains\n | category |\n | waterway |\n\n Scenario Outline: Search for mountain peaks begins at level 12\n When reverse geocoding 47.08293,9.57109\n | layer | zoom |\n | natural | |\n Then the result contains\n | category | type |\n | | |\n\n Examples:\n | zoom | category | type |\n | 12 | natural | peak |\n | 13 | waterway | river |\n\n Scenario Outline: Reverse search with manmade layers\n When reverse geocoding 32.46904,-86.44439\n | layer |\n | |\n Then the result contains\n | category | type |\n | | |\n\n Examples:\n | layer | category | type |\n | manmade | leisure | park |\n | address | highway | residential |\n | poi | leisure | pitch |\n | natural | waterway | river |\n | natural,manmade | leisure | park |\n"
},
{
"path": "test/bdd/features/api/reverse/queries.feature",
"content": "Feature: Reverse geocoding\n Testing the reverse function\n\n Scenario: Reverse - Unknown countries fall back to default country grid\n When reverse geocoding 45.174,-103.072\n Then the result contains\n | category | type | display_name |\n | place | country | United States |\n\n Scenario: Reverse - No TIGER house number for zoom < 18\n When reverse geocoding 32.4752389363,-86.4810198619\n | zoom |\n | 17 |\n Then the result contains\n | osm_type | category |\n | way | highway |\n And the result contains in field address\n | road | postcode | country_code |\n | Upper Kingston Road | 36067 | us |\n\n Scenario: Reverse - Address with non-numerical house number\n When reverse geocoding 47.107465,9.52838521614\n Then the result contains in field address\n | house_number | road |\n | 39A/B | Dorfstrasse |\n\n Scenario: Reverse - Address with numerical house number\n When reverse geocoding 47.168440329479594,9.511551699184338\n Then the result contains in field address\n | house_number | road |\n | 6 | Schmedgässle |\n\n Scenario Outline: Reverse - Zoom levels below 5 result in country\n When reverse geocoding 47.16,9.51\n | zoom |\n | |\n Then the result contains\n | display_name |\n | Liechtenstein |\n\n Examples:\n | zoom |\n | 0 |\n | 1 |\n | 2 |\n | 3 |\n | 4 |\n\n Scenario: Reverse - When on a street, the closest interpolation is shown\n When reverse geocoding 47.118457166193245,9.570678289621355\n | zoom |\n | 18 |\n Then the result contains\n | display_name |\n | 1021, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |\n\n # github 2214\n Scenario: Reverse - Interpolations do not override house numbers when they are closer\n When reverse geocoding 47.11778,9.57255\n | zoom |\n | 18 |\n Then the result contains\n | display_name |\n | 5, Grosssteg, Steg, Triesenberg, Oberland, 9497, Liechtenstein |\n\n Scenario: Reverse - Interpolations do not override house numbers when they are closer (2)\n When reverse geocoding 47.11834,9.57167\n | zoom |\n | 18 |\n Then the result contains\n | display_name |\n | 3, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |\n\n Scenario: Reverse - When on a street with zoom 18, the closest housenumber is returned\n When reverse geocoding 47.11755503977281,9.572722250405036\n | zoom |\n | 18 |\n Then the result contains in field address\n | house_number |\n | 7 |\n\n Scenario: Reverse - inherited address is shown by default\n When reverse geocoding 47.0629071,9.4879694\n Then the result contains\n | osm_type | category | display_name |\n | node | office | foo.li, 64, Hampfländer, Mäls, Balzers, Oberland, 9496, Liechtenstein |\n\n Scenario: Reverse - inherited address is not shown with address layer\n When reverse geocoding 47.0629071,9.4879694\n | layer |\n | address |\n Then the result contains\n | osm_type | category | display_name |\n | way | building | 64, Hampfländer, Mäls, Balzers, Oberland, 9496, Liechtenstein |\n"
},
{
"path": "test/bdd/features/api/reverse/v1_geocodejson.feature",
"content": "Feature: Geocodejson for Reverse API\n Testing correctness of geocodejson output (API version v1).\n\n Scenario Outline: Reverse geocodejson - Simple with no results\n When sending v1/reverse with format geocodejson\n | lat | lon |\n | | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | error |\n | Unable to geocode |\n\n Examples:\n | lat | lon |\n | 0.0 | 0.0 |\n | 91.3 | 0.4 |\n | -700 | 0.4 |\n | 0.2 | 324.44 |\n | 0.2 | -180.4 |\n\n Scenario Outline: Reverse geocodejson - Simple OSM result\n When sending v1/reverse with format geocodejson\n | lat | lon | addressdetails |\n | 47.066 | 9.504 | |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And the result metadata contains\n | version | licence | attribution!fm |\n | 0.1.0 | ODbL | Data © OpenStreetMap contributors, ODbL 1.0. https?://osm.org/copyright |\n And all results have country,postcode,county,city,district,street,housenumber,admin\n And all results contain\n | param | value |\n | osm_type | node |\n | osm_id | 6522627624 |\n | osm_key | shop |\n | osm_value | bakery |\n | type | house |\n | name | Dorfbäckerei Herrmann |\n | label | Dorfbäckerei Herrmann, 29, Gnetsch, Mäls, Balzers, Oberland, 9496, Liechtenstein |\n | geojson+type | Point |\n | geojson+coordinates | [9.5036065, 47.0660892] |\n\n Examples:\n | has_address | attributes |\n | 1 | attributes |\n | 0 | no attributes |\n\n Scenario: Reverse geocodejson - City housenumber-level address with street\n When sending v1/reverse with format geocodejson\n | lat | lon |\n | 47.1068011 | 9.52810091 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | housenumber | street | postcode | city | country |\n | 8 | Im Winkel | 9495 | Triesen | Liechtenstein |\n And all results contain\n | admin+level6 | admin+level8 |\n | Oberland | Triesen |\n\n Scenario: Reverse geocodejson - Town street-level address with street\n When sending v1/reverse with format geocodejson\n | lat | lon | zoom |\n | 47.066 | 9.504 | 16 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | name | city | postcode | country |\n | Gnetsch | Balzers | 9496 | Liechtenstein |\n\n Scenario: Reverse geocodejson - Poi street-level address with footway\n When sending v1/reverse with format geocodejson\n | lat | lon |\n | 47.06515 | 9.50083 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | street | city | postcode | country |\n | Burgweg | Balzers | 9496 | Liechtenstein |\n\n Scenario: Reverse geocodejson - City address with suburb\n When sending v1/reverse with format geocodejson\n | lat | lon |\n | 47.146861 | 9.511771 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | housenumber | street | district | city | postcode | country |\n | 5 | Lochgass | Ebenholz | Vaduz | 9490 | Liechtenstein |\n\n Scenario: Reverse geocodejson - Tiger address\n When sending v1/reverse with format geocodejson\n | lat | lon |\n | 32.4752389363 | -86.4810198619 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | osm_type | osm_id | osm_key | osm_value | type |\n | way | 396009653 | place | house | house |\n And all results contain\n | housenumber | street | city | county | postcode | country |\n | 707 | Upper Kingston Road | Prattville | Autauga County | 36067 | United States |\n\n Scenario: Reverse geocodejson - Interpolation address\n When sending v1/reverse with format geocodejson\n | lat | lon |\n | 47.118533 | 9.57056562 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | osm_type | osm_id | osm_key | osm_value | type |\n | way | 1 | place | house | house |\n And all results contain\n | label |\n | 1019, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |\n And all results have no attributes name\n\n Scenario: Reverse geocodejson - Line geometry output is supported\n When sending v1/reverse with format geocodejson\n | lat | lon | polygon_geojson |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | geojson+type |\n | LineString |\n\n Scenario Outline: Reverse geocodejson - Only geojson polygons are supported\n When sending v1/reverse with format geocodejson\n | lat | lon | |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid geocodejson with 1 result\n And all results contain\n | geojson+type |\n | Point |\n\n Examples:\n | param |\n | polygon_text |\n | polygon_svg |\n | polygon_kml |\n"
},
{
"path": "test/bdd/features/api/reverse/v1_geojson.feature",
"content": "Feature: Geojson for Reverse API\n Testing correctness of geojson output (API version v1).\n\n Scenario Outline: Reverse geojson - Simple with no results\n When sending v1/reverse with format geojson\n | lat | lon |\n | | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | error |\n | Unable to geocode |\n\n Examples:\n | lat | lon |\n | 0.0 | 0.0 |\n | 91.3 | 0.4 |\n | -700 | 0.4 |\n | 0.2 | 324.44 |\n | 0.2 | -180.4 |\n\n Scenario Outline: Reverse geojson - Simple OSM result\n When sending v1/reverse with format geojson\n | lat | lon | addressdetails |\n | 47.066 | 9.504 | |\n Then a HTTP 200 is returned\n And the result is valid geojson with 1 result\n And the result metadata contains\n | licence!fm |\n | Data © OpenStreetMap contributors, ODbL 1.0. http://osm.org/copyright |\n And all results have attributes place_id, importance\n And all results have address\n And all results contain\n | param | value |\n | osm_type | node |\n | osm_id | 6522627624 |\n | place_rank | 30 |\n | category | shop |\n | type | bakery |\n | addresstype | shop |\n | name | Dorfbäckerei Herrmann |\n | display_name | Dorfbäckerei Herrmann, 29, Gnetsch, Mäls, Balzers, Oberland, 9496, Liechtenstein |\n | boundingbox | [47.0660392, 47.0661392, 9.5035565, 9.5036565] |\n | geojson+type | Point |\n | geojson+coordinates | [9.5036065, 47.0660892] |\n\n Examples:\n | has_address | attributes |\n | 1 | attributes |\n | 0 | no attributes |\n\n Scenario: Reverse geojson - Tiger address\n When sending v1/reverse with format geojson\n | lat | lon |\n | 32.4752389363 | -86.4810198619 |\n Then a HTTP 200 is returned\n And the result is valid geojson with 1 result\n And all results contain\n | osm_type | osm_id | category | type | addresstype | place_rank |\n | way | 396009653 | place | house | place | 30 |\n\n Scenario: Reverse geojson - Interpolation address\n When sending v1/reverse with format geojson\n | lat | lon |\n | 47.118533 | 9.57056562 |\n Then a HTTP 200 is returned\n And the result is valid geojson with 1 result\n And all results contain\n | osm_type | osm_id | place_rank | category | type | addresstype |\n | way | 1 | 30 | place | house | place |\n And all results contain\n | boundingbox!in_box |\n | 47.118494, 47.118596, 9.570495, 9.570597 |\n And all results contain\n | display_name |\n | 1019, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |\n\n Scenario: Reverse geojson - Line geometry output is supported\n When sending v1/reverse with format geojson\n | lat | lon | polygon_geojson |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid geojson with 1 result\n And all results contain\n | geojson+type |\n | LineString |\n\n Scenario Outline: Reverse geojson - Only geojson polygons are supported\n When sending v1/reverse with format geojson\n | lat | lon | |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid geojson with 1 result\n And all results contain\n | geojson+type |\n | Point |\n\n Examples:\n | param |\n | polygon_text |\n | polygon_svg |\n | polygon_kml |\n"
},
{
"path": "test/bdd/features/api/reverse/v1_json.feature",
"content": "Feature: Json output for Reverse API\n Testing correctness of json and jsonv2 output (API version v1).\n\n Scenario Outline: Reverse json - Simple with no results\n When sending v1/reverse with format json\n | lat | lon |\n | | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | error |\n | Unable to geocode |\n When sending v1/reverse with format jsonv2\n | lat | lon |\n | | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | error |\n | Unable to geocode |\n\n Examples:\n | lat | lon |\n | 0.0 | 0.0 |\n | 91.3 | 0.4 |\n | -700 | 0.4 |\n | 0.2 | 324.44 |\n | 0.2 | -180.4 |\n\n Scenario Outline: Reverse json - OSM result with and without addresses\n When sending v1/reverse with format json\n | lat | lon | addressdetails |\n | 47.066 | 9.504 | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has address\n When sending v1/reverse with format jsonv2\n | lat | lon | addressdetails |\n | 47.066 | 9.504 | |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has address\n\n Examples:\n | has_address | attributes |\n | 1 | attributes |\n | 0 | no attributes |\n\n Scenario Outline: Reverse json - Simple OSM result\n When sending v1/reverse with format \n | lat | lon |\n | 47.066 | 9.504 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes place_id\n And the result contains\n | licence!fm |\n | Data © OpenStreetMap contributors, ODbL 1.0. https?://osm.org/copyright |\n And the result contains\n | osm_type | osm_id |\n | node | 6522627624 |\n And the result contains\n | lon | lat | boundingbox!in_box |\n | 9.5036065 | 47.0660892 | 47.0660391, 47.0661393, 9.5035564, 9.5036566 |\n And the result contains\n | display_name |\n | Dorfbäckerei Herrmann, 29, Gnetsch, Mäls, Balzers, Oberland, 9496, Liechtenstein |\n And the result has no attributes namedetails,extratags\n\n Examples:\n | format |\n | json |\n | jsonv2 |\n\n Scenario: Reverse json - Extra attributes of jsonv2 result\n When sending v1/reverse with format jsonv2\n | lat | lon |\n | 47.066 | 9.504 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result has attributes importance\n And the result contains\n | category | type | name | place_rank | addresstype |\n | shop | bakery | Dorfbäckerei Herrmann | 30 | shop |\n\n Scenario: Reverse json - Tiger address\n When sending v1/reverse with format jsonv2\n | lat | lon |\n | 32.4752389363 | -86.4810198619 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | osm_type | osm_id | category | type | addresstype |\n | way | 396009653 | place | house | place |\n\n Scenario Outline: Reverse json - Interpolation address\n When sending v1/reverse with format \n | lat | lon |\n | 47.118533 | 9.57056562 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | osm_type | osm_id |\n | way | 1 |\n And the result contains\n | lon | lat | boundingbox!in_box |\n | 9.5705467 | 47.1185454 | 47.118494, 47.118596, 9.570495, 9.570597 |\n And the result contains\n | display_name |\n | 1019, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |\n\n Examples:\n | format |\n | json |\n | jsonv2 |\n\n Scenario Outline: Reverse json - Output of geojson\n When sending v1/reverse with format \n | lat | lon | polygon_geojson |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | geojson+type | geojson+coordinates |\n | LineString | [[9.5039353, 47.0657546], [9.5040437, 47.0657781], [9.5040808, 47.065787], [9.5054298, 47.0661407]] |\n\n Examples:\n | format |\n | json |\n | jsonv2 |\n\n Scenario Outline: Reverse json - Output of WKT\n When sending v1/reverse with format \n | lat | lon | polygon_text |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | geotext!wkt |\n | 9.5039353 47.0657546, 9.5040437 47.0657781, 9.5040808 47.065787, 9.5054298 47.0661407 |\n\n Examples:\n | format |\n | json |\n | jsonv2 |\n\n Scenario Outline: Reverse json - Output of SVG\n When sending v1/reverse with format \n | lat | lon | polygon_svg |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | svg |\n | M 9.5039353 -47.0657546 L 9.5040437 -47.0657781 9.5040808 -47.065787 9.5054298 -47.0661407 |\n\n Examples:\n | format |\n | json |\n | jsonv2 |\n\n Scenario Outline: Reverse json - Output of KML\n When sending v1/reverse with format \n | lat | lon | polygon_kml |\n | 47.06597 | 9.50467 | 1 |\n Then a HTTP 200 is returned\n And the result is valid json\n And the result contains\n | geokml!fm |\n | 9.5039\\d*,47.0657\\d* 9.5040\\d*,47.0657\\d* 9.5040\\d*,47.065\\d* 9.5054\\d*,47.0661\\d* |\n\n Examples:\n | format |\n | json |\n | jsonv2 |\n"
},
{
"path": "test/bdd/features/api/reverse/v1_params.feature",
"content": "Feature: v1/reverse Parameter Tests\n Tests for parameter inputs for the v1 reverse endpoint.\n This file contains mostly bad parameter input. Valid parameters\n are tested in the format tests.\n\n Scenario: Bad format\n When sending v1/reverse\n | lat | lon | format |\n | 47.14122383 | 9.52169581334 | sdf |\n Then a HTTP 400 is returned\n\n Scenario: Missing lon parameter\n When sending v1/reverse\n | lat |\n | 52.52 |\n Then a HTTP 400 is returned\n\n Scenario: Missing lat parameter\n When sending v1/reverse\n | lon |\n | 52.52 |\n Then a HTTP 400 is returned\n\n Scenario Outline: Bad format for lat or lon\n When sending v1/reverse\n | lat | lon |\n | | |\n Then a HTTP 400 is returned\n\n Examples:\n | lat | lon |\n | 48.9660 | 8,4482 |\n | 48,9660 | 8.4482 |\n | 48,9660 | 8,4482 |\n | 48.966.0 | 8.4482 |\n | 48.966 | 8.448.2 |\n | Nan | 8.448 |\n | 48.966 | Nan |\n | Inf | 5.6 |\n | 5.6 | -Inf |\n | | 3.4 |\n | 3.4 | |\n | -45.3 | ; |\n | gkjd | 50 |\n\n Scenario: Non-numerical zoom levels return an error\n When sending v1/reverse\n | lat | lon | zoom |\n | 47.14122383 | 9.52169581334 | adfe |\n Then a HTTP 400 is returned\n\n Scenario Outline: Truthy values for boolean parameters\n When sending v1/reverse\n | lat | lon | addressdetails |\n | 47.14122383 | 9.52169581334 | |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result has attributes address\n\n When sending v1/reverse\n | lat | lon | extratags |\n | 47.14122383 | 9.52169581334 | |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result has attributes extratags\n\n When sending v1/reverse\n | lat | lon | namedetails |\n | 47.14122383 | 9.52169581334 | |\n Then a HTTP 200 is returned\n And the result is valid xml\n And the result has attributes namedetails\n\n Examples:\n | value |\n | yes |\n | no |\n | -1 |\n | 100 |\n | false |\n | 00 |\n\n Scenario: Only one geometry can be requested\n When sending v1/reverse\n | lat | lon | polygon_text | polygon_svg |\n | 47.14122383 | 9.52169581334 | 1 | 1 |\n Then a HTTP 400 is returned\n\n Scenario Outline: Illegal jsonp are not allowed\n When sending v1/reverse with format json\n | lat | lon | json_callback |\n | 47.14122383 | 9.52169581334 | |\n Then a HTTP 400 is returned\n\n Examples:\n | data |\n | 1asd |\n | bar(foo) |\n | XXX['bad'] |\n | foo; evil |\n\n Scenario Outline: Reverse debug mode produces valid HTML\n When sending v1/reverse\n | lat | lon | debug |\n | | | 1 |\n Then a HTTP 200 is returned\n And the result is valid html\n\n Examples:\n | lat | lon |\n | 0.0 | 0.0 |\n | 47.06645 | 9.56601 |\n | 47.14081 | 9.52267 |\n\n Scenario Outline: Full address display for city housenumber-level address with street\n When sending v1/reverse with format \n | lat | lon |\n | 47.1068011 | 9.52810091 |\n Then a HTTP 200 is returned\n And the result is valid \n And the result contains in field address\n | param | value |\n | house_number | 8 |\n | road | Im Winkel |\n | neighbourhood | Oberdorf |\n | village | Triesen |\n | ISO3166-2-lvl8 | LI-09 |\n | county | Oberland |\n | postcode | 9495 |\n | country | Liechtenstein |\n | country_code | li |\n\n Examples:\n | format | outformat |\n | json | json |\n | jsonv2 | json |\n | xml | xml |\n\n Scenario Outline: Results with name details\n When sending v1/reverse with format \n | lat | lon | zoom | namedetails |\n | 47.14052 | 9.52202 | 14 | 1 |\n Then a HTTP 200 is returned\n And the result is valid \n And the result contains in field namedetails\n | name |\n | Ebenholz |\n\n Examples:\n | format | outformat |\n | json | json |\n | jsonv2 | json |\n | xml | xml |\n\n Scenario Outline: Results with extratags\n When sending v1/reverse with format \n | lat | lon | zoom | extratags |\n | 47.14052 | 9.52202 | 14 | 1 |\n Then a HTTP 200 is returned\n And the result is valid \n And the result contains in field extratags\n | wikidata |\n | Q4529531 |\n\n Examples:\n | format | outformat |\n | json | json |\n | jsonv2 | json |\n | xml | xml |\n\n Scenario Outline: Reverse with entrances\n When sending v1/reverse with format \n | lat | lon | entrances | zoom |\n | 47.24942041089678 | 9.52854573737568 | 1 | 18 |\n Then a HTTP 200 is returned\n And the result is valid